diff options
Diffstat (limited to 'erts/emulator/beam')
92 files changed, 11365 insertions, 5213 deletions
diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab index f14b376419..335b121ad8 100644 --- a/erts/emulator/beam/arith_instrs.tab +++ b/erts/emulator/beam/arith_instrs.tab @@ -217,6 +217,11 @@ i_int_div(Fail, Op1, Op2, Dst) { $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2); } else if (ERTS_LIKELY(is_both_small(op1, op2))) { Sint ires = signed_val(op1) / signed_val(op2); + + /* We could skip this check if it weren't for the fact that dividing + * MIN_SMALL by -1 causes an overflow, and we have nothing to gain from + * fixed-point optimizing this instruction since there's no + * __builtin_div_overflow. */ if (ERTS_LIKELY(IS_SSMALL(ires))) { $Dst = make_small(ires); $NEXT0(); @@ -241,7 +246,14 @@ rem.execute(Fail, Dst) { c_p->freason = BADARITH; $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2); } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) { - $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2)); + Sint lhs_untagged, rhs_untagged, untagged_result; + + /* See plus.execute */ + lhs_untagged = (RemOp1 & ~_TAG_IMMED1_MASK); + rhs_untagged = (RemOp2 & ~_TAG_IMMED1_MASK); + untagged_result = lhs_untagged % rhs_untagged; + + $Dst = untagged_result | _TAG_IMMED1_SMALL; $NEXT0(); } else { $OUTLINED_ARITH_2($Fail, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst); @@ -447,10 +459,10 @@ shift.execute(Fail, Dst) { reg[1] = Op2; SWAPOUT; if (IsOpCode(I[0], i_bsl_ssjd)) { - I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[BIF_bsl_2].info.mfa); } else { ASSERT(IsOpCode(I[0], i_bsr_ssjd)); - I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[BIF_bsr_2].info.mfa); } goto post_error_handling; } diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 7046eb5e65..3ea7677b6d 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -68,6 +68,7 @@ atom ErtsSecretAtom='3RT$' atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' +atom abandoned atom abort atom abs_path atom absoluteURI @@ -116,6 +117,7 @@ atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth atom badarg badarith badarity badfile badfun badkey badmap badmatch badsig +atom badopt atom bag atom band atom big @@ -195,6 +197,7 @@ atom current_location atom current_stacktrace atom data atom debug_flags +atom decentralized_counters atom decimals atom default atom delay_trap @@ -216,6 +219,7 @@ atom dist_cmd atom dist_ctrl_put_data atom dist_ctrlr atom dist_data +atom dist_spawn_init atom Div='/' atom div atom dmonitor_node @@ -246,6 +250,7 @@ atom erlang atom erl_signal_server atom error_handler atom error_logger +atom error_only atom erts_code_purger atom erts_debug atom erts_dflags @@ -293,6 +298,7 @@ atom gc_minor_start atom Ge='>=' atom generational atom get_all_trap +atom get_iovec atom get_internal_state_blocked atom get_seq_token atom get_size @@ -345,6 +351,8 @@ atom is_constant atom is_seq_trace atom iterator atom io +atom iodata +atom iovec atom keypos atom kill atom killed @@ -408,6 +416,7 @@ atom min_bin_vheap_size atom minor atom minor_version atom Minus='-' +atom MinusMinus='--' atom module atom module_info atom monitored_by @@ -453,6 +462,7 @@ atom noeol atom noproc atom normal atom nosuspend +atom no_fail atom no_float atom no_integer atom no_network @@ -498,6 +508,7 @@ atom packet atom packet_size atom parallelism atom Plus='+' +atom PlusPlus='++' atom pause atom pending atom pending_driver @@ -550,10 +561,13 @@ atom register atom registered_name atom reload atom rem +atom reply +atom reply_tag atom report_errors atom reset atom reset_seq_trace atom restart +atom resume atom return_from atom return_to atom return_trace @@ -612,6 +626,10 @@ atom silent atom size atom spawn_executable atom spawn_driver +atom spawn_init +atom spawn_reply +atom spawn_request +atom spawn_request_yield atom spawned atom ssl_tls atom stack_size @@ -622,6 +640,7 @@ atom stop atom stream atom strict_monotonic atom strict_monotonic_timestamp +atom success_only atom sunrm atom suspend atom suspended @@ -689,4 +708,4 @@ atom xor atom x86 atom yes atom yield -atom nifs +atom nifs
\ No newline at end of file diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index c530063fa1..e130baa1c0 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -144,7 +144,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], + ERTS_BIF_YIELD3(&bif_trap_export[BIF_code_make_stub_module_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } @@ -279,7 +279,7 @@ finish_loading_1(BIF_ALIST_1) int do_commit = 0; if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1); + ERTS_BIF_YIELD1(&bif_trap_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1); } /* @@ -641,7 +641,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1); + ERTS_BIF_YIELD1(&bif_trap_export[BIF_delete_module_1], BIF_P, BIF_ARG_1); } { @@ -768,7 +768,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2], + ERTS_BIF_YIELD2(&bif_trap_export[BIF_finish_after_on_load_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } @@ -819,21 +819,25 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ num_exps = export_list_size(code_ix); for (i = 0; i < num_exps; i++) { - Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { - continue; - } - if (ep->beam[1] != 0) { - ep->addressv[code_ix] = (void *) ep->beam[1]; - ep->beam[1] = 0; - } else { - if (ep->addressv[code_ix] == ep->beam && - BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); - } + Export *ep = export_list(i, code_ix); + + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { + continue; + } + + DBG_CHECK_EXPORT(ep, code_ix); + + if (ep->trampoline.not_loaded.deferred != 0) { + ep->addressv[code_ix] = (void*)ep->trampoline.not_loaded.deferred; + ep->trampoline.not_loaded.deferred = 0; + } else { + if (ep->bif_number != -1) { + continue; + } + + ep->addressv[code_ix] = ep->trampoline.raw; + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -856,10 +860,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { + if (ep->bif_number != -1) { continue; } - ep->beam[1] = 0; + + ep->trampoline.not_loaded.deferred = 0; } } erts_release_code_write_permission(); @@ -1109,16 +1114,15 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) mod_size = modp->old.code_length; /* - * Check if current instruction or continuation pointer points into module. + * Check if the instruction pointer points into module. */ - if (ErtsInArea(rp->i, mod_start, mod_size) - || ErtsInArea(rp->cp, mod_start, mod_size)) { + if (ErtsInArea(rp->i, mod_start, mod_size)) { return am_true; } - + *redsp += 1; - if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + if (erts_check_nfunc_in_area(rp, mod_start, mod_size)) return am_true; *redsp += (STACK_START(rp) - rp->stop) / 32; @@ -1657,7 +1661,7 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) ASSERT(old_area); ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP0(bif_export[BIF_erts_internal_release_literal_area_switch_0], + BIF_TRAP0(&bif_trap_export[BIF_erts_internal_release_literal_area_switch_0], BIF_P); } @@ -2019,7 +2023,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_purge_module_2], + ERTS_BIF_YIELD2(&bif_trap_export[BIF_erts_internal_purge_module_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } @@ -2043,23 +2047,22 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) ERTS_BIF_PREP_RET(ret, am_false); } else { - /* - * Unload any NIF library - */ - if (modp->old.nif != NULL - || IF_HIPE(hipe_purge_need_blocking(modp))) { - /* ToDo: Do unload nif without blocking */ + if (IF_HIPE(hipe_purge_need_blocking(modp))) { erts_rwunlock_old_code(code_ix); erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - if (modp->old.nif) { - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; - } } + /* + * Unload any NIF library + */ + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + /* * Remove the old code. */ @@ -2191,25 +2194,28 @@ delete_code(Module* modp) for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { - if (ep->addressv[code_ix] == ep->beam) { - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } - else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + if (ep->addressv[code_ix] == ep->trampoline.raw) { + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); DBG_TRACE_MFA_P(&ep->info.mfa, "export trace cleared, code_ix=%d", code_ix); - erts_clear_export_break(modp, &ep->info); + erts_clear_export_break(modp, ep); } else { - ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) || + ASSERT(BeamIsOpCode(ep->trampoline.op, op_call_error_handler) || !erts_initialized); } } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); - ep->beam[1] = 0; + + if (ep->bif_number != -1 && ep->is_bif_traced) { + /* Code unloading kills both global and local call tracing. */ + ep->is_bif_traced = 0; + } + + ep->addressv[code_ix] = ep->trampoline.raw; + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + ep->trampoline.not_loaded.deferred = 0; DBG_TRACE_MFA_P(&ep->info.mfa, "export invalidation, code_ix=%d", code_ix); } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index da43f65780..4144b0e751 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -207,9 +207,6 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) if (erts_is_function_native(ci)) { continue; } - if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ - continue; - } switch (specified) { case 3: if (ci->mfa.arity != mfa->arity) @@ -244,8 +241,10 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) f->matching = (BpFunction *) Alloc(num_exps*sizeof(BpFunction)); ne = 0; for (i = 0; i < num_exps; i++) { - Export* ep = export_list(i, code_ix); - BeamInstr* pc; + BeamInstr *func; + Export* ep; + + ep = export_list(i, code_ix); switch (specified) { case 3: @@ -263,19 +262,20 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) ASSERT(0); } - pc = ep->beam; - if (ep->addressv[code_ix] == pc) { - if (BeamIsOpCode(*pc, op_apply_bif) || - BeamIsOpCode(*pc, op_call_error_handler)) { - continue; - } - ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); - } else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) { - continue; - } + func = ep->addressv[code_ix]; + + if (func == ep->trampoline.raw) { + if (BeamIsOpCode(*func, op_call_error_handler)) { + continue; + } + ASSERT(BeamIsOpCode(*func, op_i_generic_breakpoint)); + } else if (erts_is_function_native(erts_code_to_codeinfo(func))) { + continue; + } f->matching[ne].ci = &ep->info; f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); + ne++; } @@ -305,18 +305,6 @@ erts_consolidate_bp_data(BpFunctions* f, int local) } } -void -erts_consolidate_bif_bp_data(void) -{ - int i; - - ERTS_LC_ASSERT(erts_has_code_write_permission()); - for (i = 0; i < BIF_SIZE; i++) { - Export *ep = bif_export[i]; - consolidate_bp_data(0, &ep->info, 0); - } -} - static void consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { @@ -495,7 +483,7 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) +erts_set_export_trace(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; @@ -503,25 +491,6 @@ erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) } void -erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) -{ - set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); -} - -void -erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) -{ - set_function_break(ci, NULL, - ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, - count_op, erts_tracer_nil); -} - -void -erts_clear_time_trace_bif(ErtsCodeInfo *ci) { - clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); -} - -void erts_set_debug_break(BpFunctions* f) { set_break(f, NULL, ERTS_BPF_DEBUG, 0, erts_tracer_nil); } @@ -547,7 +516,7 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) +erts_clear_export_trace(ErtsCodeInfo *ci, int local) { GenericBp* g = ci->u.gen_bp; @@ -566,12 +535,6 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(ErtsCodeInfo *ci) -{ - clear_function_break(ci, ERTS_BPF_META_TRACE); -} - -void erts_clear_debug_break(BpFunctions* f) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); @@ -630,58 +593,56 @@ erts_clear_module_break(Module *modp) { } void -erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) +erts_clear_export_break(Module* modp, Export *ep) { + ErtsCodeInfo *ci; + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); + ci = &ep->info; + + ASSERT(erts_codeinfo_to_code(ci) == ep->trampoline.raw); + + ASSERT(BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)); + ep->trampoline.op = 0; + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); ASSERT(ci->u.gen_bp == NULL); } /* - * If c_p->cp is a trace return instruction, we set cp - * to be the place where we again start to execute code. + * If the topmost continuation pointer on the stack is a trace return + * instruction, we modify it to be the place where we again start to + * execute code. * - * cp is used by match spec {caller} to get the calling - * function, and if we don't do this fixup it will be - * 'undefined'. This has the odd side effect of {caller} - * not really being which function is the caller, but - * rather which function we are about to return to. + * This continuation pointer is used by match spec {caller} to get the + * calling function, and if we don't do this fixup it will be + * 'undefined'. This has the odd side effect of {caller} not really + * being the function which is the caller, but rather the function + * which we are about to return to. */ static void fixup_cp_before_trace(Process *c_p, int *return_to_trace) { - Eterm *cpp, *E = c_p->stop; - BeamInstr w = *c_p->cp; - if (BeamIsOpCode(w, op_return_trace)) { - cpp = &E[2]; - } else if (BeamIsOpCode(w, op_i_return_to_trace)) { - *return_to_trace = 1; - cpp = &E[0]; - } else if (BeamIsOpCode(w, op_i_return_time_trace)) { - cpp = &E[0]; - } else { - cpp = NULL; - } - if (cpp) { - for (;;) { - BeamInstr w = *cp_val(*cpp); - if (BeamIsOpCode(w, op_return_trace)) { - cpp += 3; - } else if (BeamIsOpCode(w, op_i_return_to_trace)) { - *return_to_trace = 1; - cpp += 1; - } else if (BeamIsOpCode(w, op_i_return_time_trace)) { - cpp += 2; - } else { - break; - } + Eterm *cpp = c_p->stop; + + for (;;) { + BeamInstr w = *cp_val(*cpp); + if (BeamIsOpCode(w, op_return_trace)) { + cpp += 3; + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { + *return_to_trace = 1; + cpp += 1; + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { + cpp += 2; + } else { + break; } - c_p->cp = (BeamInstr *) cp_val(*cpp); - ASSERT(is_CP(*cpp)); } + c_p->stop[0] = (Eterm) cp_val(*cpp); + ASSERT(is_CP(*cpp)); } BeamInstr @@ -742,13 +703,14 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { - Eterm w; + BeamInstr w; + Eterm* E; ErtsCodeInfo* prev_info = erts_trace_time_call(c_p, info, bp->time); - w = (BeamInstr) *c_p->cp; + E = c_p->stop; + w = *(BeamInstr*) E[0]; if (! (BeamIsOpCode(w, op_i_return_time_trace) || BeamIsOpCode(w, op_return_trace) || BeamIsOpCode(w, op_i_return_to_trace)) ) { - Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); @@ -759,9 +721,8 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = prev_info ? make_cp(erts_codeinfo_to_code(prev_info)) : NIL; - E[1] = make_cp(c_p->cp); /* original return address */ - c_p->cp = beam_return_time_trace; + E[1] = prev_info ? make_cp(erts_codeinfo_to_code(prev_info)) : NIL; + E[0] = (Eterm) beam_return_time_trace; c_p->stop = E; } } @@ -773,237 +734,24 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } } -/* - * Entry point called by the trace wrap functions in erl_bif_wrap.c - * - * The trace wrap functions are themselves called through the export - * entries instead of the original BIF functions. - */ -Eterm -erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) -{ - Eterm result; - Eterm (*func)(Process*, Eterm*, BeamInstr*); - Export* ep = bif_export[bif_index]; - Uint32 flags = 0, flags_meta = 0; - ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == ep->beam); /* Yup, the apply code for a bif - * is actually in the - * export entry */ - BeamInstr *cp = p->cp; - GenericBp* g; - GenericBpData* bp = NULL; - Uint bp_flags = 0; - int return_to_trace = 0; - - ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - - g = ep->info.u.gen_bp; - if (g) { - bp = &g->data[erts_active_bp_ix()]; - bp_flags = bp->flags; - } - - /* - * Make continuation pointer OK, it is not during direct BIF calls, - * but it is correct during apply of bif. - */ - if (!applying) { - p->cp = I; - } else { - fixup_cp_before_trace(p, &return_to_trace); - } - if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && - IS_TRACED_FL(p, F_TRACE_CALLS)) { - int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, &ep->info, bp->local_ms, args, - local, &ERTS_TRACER(p)); - } - if (bp_flags & ERTS_BPF_META_TRACE) { - ErtsTracer old_tracer; - - meta_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer); - old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, - 0, &meta_tracer); - - if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { - ErtsTracer new_tracer = erts_tracer_nil; - erts_tracer_update(&new_tracer, meta_tracer); - if (old_tracer == erts_atomic_cmpxchg_acqb( - &bp->meta_tracer->tracer, - (erts_aint_t)new_tracer, - (erts_aint_t)old_tracer)) { - ERTS_TRACER_CLEAR(&old_tracer); - } else { - ERTS_TRACER_CLEAR(&new_tracer); - } - } - } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && - IS_TRACED_FL(p, F_TRACE_CALLS)) { - erts_trace_time_call(p, &ep->info, bp->time); - } - - /* Restore original continuation pointer (if changed). */ - p->cp = cp; - - func = bif_table[bif_index].f; - - result = func(p, args, I); - - if (erts_nif_export_check_save_trace(p, result, - applying, ep, - cp, flags, - flags_meta, I, - meta_tracer)) { - /* - * erts_bif_trace_epilogue() will be called - * later when appropriate via the NIF export - * scheduling functionality... - */ - return result; - } - - return erts_bif_trace_epilogue(p, result, applying, ep, cp, - flags, flags_meta, I, - meta_tracer); -} - -Eterm -erts_bif_trace_epilogue(Process *p, Eterm result, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; - BeamInstr i_return_time_trace = beam_return_time_trace[0]; - Eterm *cpp; - /* Maybe advance cp to skip trace stack frames */ - for (cpp = p->stop; ; cp = cp_val(*cpp++)) { - if (*cp == i_return_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 2; /* Skip return_trace parameters */ - } else if (*cp == i_return_time_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 1; /* Skip return_time_trace parameters */ - } else if (*cp == i_return_to_trace) { - /* A return_to trace message is going to be generated - * by normal means, so we do not have to. - */ - cp = NULL; - break; - } else break; - } - } - - /* Try to get these in the order - * they usually appear in normal code... */ - if (is_non_value(result)) { - Uint reason = p->freason; - if (reason != TRAP) { - Eterm class; - Eterm value = p->fvalue; - /* Expand error value like in handle_error() */ - if (reason & EXF_ARGLIST) { - Eterm *tp; - ASSERT(is_tuple(value)); - tp = tuple_val(value); - value = tp[1]; - } - if ((reason & EXF_THROWN) && (p->catches <= 0)) { - Eterm *hp = HAlloc(p, 3); - value = TUPLE2(hp, am_nocatch, value); - reason = EXC_ERROR; - } - /* Note: expand_error_value() could theoretically - * allocate on the heap, but not for any error - * returned by a BIF, and it would do no harm, - * just be annoying. - */ - value = expand_error_value(p, reason, value); - class = exception_tag[GET_EXC_CLASS(reason)]; - - if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, &ep->info.mfa, class, value, - &meta_tracer); - } - if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, &ep->info.mfa, class, value, - &ERTS_TRACER(p)); - } - if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { - /* can only happen if(local)*/ - Eterm *ptr = p->stop; - ASSERT(is_CP(*ptr)); - ASSERT(ptr <= STACK_START(p)); - /* Search the nearest stack frame for a catch */ - while (++ptr < STACK_START(p)) { - if (is_CP(*ptr)) break; - if (is_catch(*ptr)) { - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into - * calling function */ - erts_trace_return_to(p, I); - } - } - } - } - if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { - erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE; - erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - } else { - if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); - } - /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ - if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); - } - if (flags & MATCH_SET_RETURN_TO_TRACE && - IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { - /* can only happen if(local)*/ - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into calling function */ - erts_trace_return_to(p, I); - } - } - } - ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - return result; -} - static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { int return_to_trace = 0; - BeamInstr *cp_save = c_p->cp; Uint32 flags; Uint need = 0; + Eterm cp_save; Eterm* E = c_p->stop; - fixup_cp_before_trace(c_p, &return_to_trace); + cp_save = E[0]; + fixup_cp_before_trace(c_p, &return_to_trace); ERTS_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_REQ_PROC_MAIN_LOCK(c_p); - /* restore cp after potential fixup */ - c_p->cp = cp_save; + E[0] = cp_save; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { @@ -1023,28 +771,23 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, if (flags & MATCH_SET_RETURN_TO_TRACE && !return_to_trace) { E -= 1; ASSERT(c_p->htop <= E && E <= c_p->hend); - E[0] = make_cp(c_p->cp); - c_p->cp = beam_return_to_trace; + E[0] = (Eterm) beam_return_to_trace; + c_p->stop = E; } - if (flags & MATCH_SET_RX_TRACE) - { + if (flags & MATCH_SET_RX_TRACE) { E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); - E[2] = make_cp(c_p->cp); - E[1] = copy_object(tracer, c_p); - E[0] = make_cp(&info->mfa.module); - /* We ARE at the beginning of an instruction, - the funcinfo is above i. */ - c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? - beam_exception_trace : beam_return_trace; + E[2] = copy_object(tracer, c_p); + E[1] = make_cp(&info->mfa.module); + E[0] = (Eterm) ((flags & MATCH_SET_EXCEPTION_TRACE) ? + beam_exception_trace : beam_return_trace); erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE; erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } else - c_p->stop = E; + } return tracer; } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index a10965191e..827302b9b5 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -119,20 +119,16 @@ void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); void erts_uninstall_breakpoints(BpFunctions* f); void erts_consolidate_bp_data(BpFunctions* f, int local); -void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); -void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); +void erts_set_export_trace(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_export_trace(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, - ErtsTracer tracer); -void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -142,7 +138,7 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); +void erts_clear_export_break(Module *modp, Export *ep); BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, @@ -151,8 +147,6 @@ BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, - ErtsTracer *tracer_ret); int erts_is_native_break(ErtsCodeInfo *ci); int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); @@ -163,10 +157,6 @@ void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); -void erts_clear_time_trace_bif(ErtsCodeInfo *ci); - ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 4d52435139..eeb3a465d0 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -47,7 +47,6 @@ #else # define HEXF "%08bpX" #endif -#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); @@ -158,7 +157,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2], + ERTS_BIF_YIELD2(&bif_trap_export[BIF_erts_debug_breakpoint_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); @@ -332,7 +331,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) "unknown " HEXF "\n", instr); code_ptr++; } - if (i == op_call_nif) { + if (i == op_call_nif_WWW) { /* * The rest of the code will not be executed. Don't disassemble any * more code in this function. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 07c16e3415..1a250470cb 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -109,14 +109,9 @@ do { \ # define CHECK_ARGS(T) #endif -#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0) - -#define GET_BIF_MODULE(p) (p->info.mfa.module) -#define GET_BIF_FUNCTION(p) (p->info.mfa.function) -#define GET_BIF_ARITY(p) (p->info.mfa.arity) -#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) -#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) - +#define GET_EXPORT_MODULE(p) ((p)->info.mfa.module) +#define GET_EXPORT_FUNCTION(p) ((p)->info.mfa.function) +#define GET_EXPORT_ARITY(p) ((p)->info.mfa.arity) /* * We reuse some of fields in the save area in the process structure. @@ -141,10 +136,6 @@ do { \ BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop)) #endif /* NO_JUMP_TABLE */ -#define SET_CP(p, ip) \ - ASSERT(VALID_INSTR(*(ip))); \ - (p)->cp = (ip) - #define SET_I(ip) \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) @@ -165,7 +156,7 @@ BeamInstr beam_continue_exit[1]; /* NOTE These should be the only variables containing trace instructions. -** Sometimes tests are form the instruction value, and sometimes +** Sometimes tests are for the instruction value, and sometimes ** for the referring variable (one of these), and rouge references ** will most likely cause chaos. */ @@ -254,72 +245,6 @@ void** beam_ops; #define Q(N) (N*sizeof(Eterm *)) #define l(N) (freg[N].fd) -/* - * Check that we haven't used the reductions and jump to function pointed to by - * the I register. If we are out of reductions, do a context switch. - */ - -#define DispatchMacro() \ - do { \ - BeamInstr dis_next; \ - dis_next = *I; \ - CHECK_ARGS(I); \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(dis_next); \ - } else { \ - goto context_switch; \ - } \ - } while (0) \ - -#define DispatchMacroFun() \ - do { \ - BeamInstr dis_next; \ - dis_next = *I; \ - CHECK_ARGS(I); \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(dis_next); \ - } else { \ - goto context_switch_fun; \ - } \ - } while (0) - -#define DispatchMacrox() \ - do { \ - if (FCALLS > 0) { \ - BeamInstr dis_next; \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - dis_next = *I; \ - FCALLS--; \ - CHECK_ARGS(I); \ - Goto(dis_next); \ - } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ - && FCALLS > neg_o_reds) { \ - goto save_calls1; \ - } else { \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - CHECK_ARGS(I); \ - goto context_switch; \ - } \ - } while (0) - -#ifdef DEBUG -/* - * To simplify breakpoint setting, put the code in one place only and jump to it. - */ -# define Dispatch() goto do_dispatch -# define Dispatchx() goto do_dispatchx -# define Dispatchfun() goto do_dispatchfun -#else -/* - * Inline for speed. - */ -# define Dispatch() DispatchMacro() -# define Dispatchx() DispatchMacrox() -# define Dispatchfun() DispatchMacroFun() -#endif - #define Arg(N) I[(N)+1] #define GetSource(raw, dst) \ @@ -352,19 +277,6 @@ do { \ } \ } while(0) -#define DispatchReturn \ -do { \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(*I); \ - } \ - else { \ - c_p->current = NULL; \ - c_p->arity = 1; \ - goto context_switch3; \ - } \ -} while (0) - #ifdef DEBUG /* Better static type testing by the C compiler */ # define BEAM_IS_TUPLE(Src) is_tuple(Src) @@ -521,10 +433,10 @@ init_emulator(void) } \ } while(0) -#define DTRACE_RETURN_FROM_PC(p) \ +#define DTRACE_RETURN_FROM_PC(p, i) \ do { \ ErtsCodeMFA* cmfa; \ - if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc(i))) { \ DTRACE_RETURN((p), cmfa); \ } \ } while(0) @@ -534,7 +446,7 @@ init_emulator(void) #define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) #define DTRACE_RETURN(p, mfa) do {} while (0) -#define DTRACE_RETURN_FROM_PC(p) do {} while (0) +#define DTRACE_RETURN_FROM_PC(p, i) do {} while (0) #define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) #define DTRACE_BIF_RETURN(p, mfa) do {} while (0) #define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) @@ -772,27 +684,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #endif #include "beam_hot.h" - -#ifdef DEBUG - /* - * Set a breakpoint here to get control just after a call instruction. - * I points to the first instruction in the called function. - * - * In gdb, use 'call dis(I-5, 1)' to show the name of the function. - */ - do_dispatch: - DispatchMacro(); - - do_dispatchx: - DispatchMacrox(); - - do_dispatchfun: - DispatchMacroFun(); - -#endif - /* - * Jumped to from the Dispatch() macro when the reductions are used up. + * The labels are jumped to from the $DISPATCH() macros when the reductions + * are used up. * * Since the I register points just beyond the FuncBegin instruction, we * can get the module, function, and arity for the function being @@ -986,18 +880,46 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) } #endif return; /* Never executed */ +} - save_calls1: - { - BeamInstr dis_next; +/* + * Enter all BIFs into the export table. + * + * Note that they will all call the error_handler until their modules have been + * loaded, which may prevent the system from booting if BIFs from non-preloaded + * modules are apply/3'd while loading code. Ordinary BIF calls will work fine + * however since they won't go through export entries. + */ +static void install_bifs(void) { + int i; - save_calls(c_p, (Export *) Arg(0)); + for (i = 0; i < BIF_SIZE; i++) { + BifEntry *entry; + Export *ep; + int j; - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); + entry = &bif_table[i]; - dis_next = *I; - FCALLS--; - Goto(dis_next); + ep = erts_export_put(entry->module, entry->name, entry->arity); + + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); + ep->info.mfa.module = entry->module; + ep->info.mfa.function = entry->name; + ep->info.mfa.arity = entry->arity; + ep->bif_number = i; + + memset(&ep->trampoline, 0, sizeof(ep->trampoline)); + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + + for (j = 0; j < ERTS_NUM_CODE_IX; j++) { + ep->addressv[j] = ep->trampoline.raw; + } + + /* Set up a hidden export entry so we can trap to this BIF without + * it being seen when tracing. */ + erts_init_trap_export(&bif_trap_export[i], + entry->module, entry->name, entry->arity, + entry->f); } } @@ -1008,43 +930,30 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) static void init_emulator_finish(void) { - int i; - Export* ep; - #if defined(ARCH_64) && defined(CODE_MODEL_SMALL) - for (i = 0; i < NUMBER_OF_OPCODES; i++) { - BeamInstr instr = BeamOpCodeAddr(i); - if (instr >= (1ull << 32)) { - erts_exit(ERTS_ERROR_EXIT, - "This run-time was supposed be compiled with all code below 2Gb,\n" - "but the instruction '%s' is located at %016lx.\n", - opc[i].name, instr); - } - } + int i; + + for (i = 0; i < NUMBER_OF_OPCODES; i++) { + BeamInstr instr = BeamOpCodeAddr(i); + if (instr >= (1ull << 32)) { + erts_exit(ERTS_ERROR_EXIT, + "This run-time was supposed be compiled with all code below 2Gb,\n" + "but the instruction '%s' is located at %016lx.\n", + opc[i].name, instr); + } + } #endif - beam_apply[0] = BeamOpCodeAddr(op_i_apply); - beam_apply[1] = BeamOpCodeAddr(op_normal_exit); - beam_exit[0] = BeamOpCodeAddr(op_error_action_code); - beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); - beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); - beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); - beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ - beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); + beam_apply[0] = BeamOpCodeAddr(op_i_apply); + beam_apply[1] = BeamOpCodeAddr(op_normal_exit); + beam_exit[0] = BeamOpCodeAddr(op_error_action_code); + beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); + beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); + beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); + beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ + beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); - /* - * Enter all BIFs into the export table. - */ - for (i = 0; i < BIF_SIZE; i++) { - ep = erts_export_put(bif_table[i].module, - bif_table[i].name, - bif_table[i].arity); - bif_export[i] = ep; - ep->beam[0] = BeamOpCodeAddr(op_apply_bif); - ep->beam[1] = (BeamInstr) bif_table[i].f; - /* XXX: set func info for bifs */ - ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); - } + install_bifs(); } /* @@ -1257,7 +1166,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF * - * This layout is determined by the NifExport struct + * This layout is determined by the ErtsNativeFunc struct */ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); @@ -1271,11 +1180,11 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - if (BeamIsOpCode(*I, op_apply_bif)) { + if (BeamIsOpCode(*I, op_call_bif_W)) { exiting = erts_call_dirty_bif(esdp, c_p, I, reg); } else { - ASSERT(BeamIsOpCode(*I, op_call_nif)); + ASSERT(BeamIsOpCode(*I, op_call_nif_WWW)); exiting = erts_call_dirty_nif(esdp, c_p, I, reg); } @@ -1303,7 +1212,7 @@ ubif2mfa(void* uf) int i; for (i = 0; erts_u_bifs[i].bif; i++) { if (erts_u_bifs[i].bif == uf) - return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa; + return &bif_trap_export[erts_u_bifs[i].exp_ix].info.mfa; } erts_exit(ERTS_ERROR_EXIT, "bad u bif: %p\n", uf); return NULL; @@ -1344,6 +1253,33 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { am_badkey, /* 19 */ }; +/* Returns the return address at E[0] in printable form, skipping tracing in + * the same manner as gather_stacktrace. + * + * This is needed to generate correct stacktraces when throwing errors from + * instructions that return like an ordinary function, such as call_nif. */ +BeamInstr *erts_printable_return_address(Process* p, Eterm *E) { + Eterm *ptr = E; + + ASSERT(is_CP(*ptr)); + + while (ptr < STACK_START(p)) { + BeamInstr *cp = cp_val(*ptr); + + if (cp == beam_exception_trace || cp == beam_return_trace) { + ptr += 3; + } else if (cp == beam_return_time_trace) { + ptr += 2; + } else if (cp == beam_return_to_trace) { + ptr += 1; + } else { + return cp; + } + } + + ERTS_ASSERT(!"No continuation pointer on stack"); +} + /* * To fully understand the error handling, one must keep in mind that * when an exception is thrown, the search for a handler can jump back @@ -1373,14 +1309,14 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->freason & EXF_RESTORE_NIF) - erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa); + if (c_p->freason & EXF_RESTORE_NFUNC) + erts_nfunc_restore_error(c_p, &pc, reg, &bif_mfa); #ifdef DEBUG if (bif_mfa) { - /* Verify that bif_mfa does not point into our nif export */ - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); - ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport))); + /* Verify that bif_mfa does not point into our native function wrapper */ + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); + ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(ErtsNativeFunc))); } #endif @@ -1443,7 +1379,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) reg[2] = Value; reg[3] = c_p->ftrace; if ((new_pc = next_catch(c_p, reg))) { - c_p->cp = 0; /* To avoid keeping stale references. */ + c_p->stop[0] = NIL; /* To avoid keeping stale references. */ ERTS_RECV_MARK_CLEAR(c_p); /* No longer safe to use this position */ return new_pc; } @@ -1481,35 +1417,6 @@ next_catch(Process* c_p, Eterm *reg) { return NULL; } - /* - * Better safe than sorry here. In debug builds, produce a core - * dump if the top of the stack doesn't point to a continuation - * pointer. In other builds, ignore a non-CP at the top of stack. - */ - ASSERT(is_CP(*ptr)); - if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace && - *cp_val(*ptr) != i_return_time_trace )) - && c_p->cp) { - /* Can not follow cp here - code may be unloaded */ - BeamInstr *cpp = c_p->cp; - if (cpp == beam_exception_trace) { - ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); - erts_trace_exception(c_p, mfa, - reg[1], reg[2], - ERTS_TRACER_FROM_ETERM(ptr+1)); - /* Skip return_trace parameters */ - ptr += 2; - } else if (cpp == beam_return_trace) { - /* Skip return_trace parameters */ - ptr += 2; - } else if (cpp == beam_return_time_trace) { - /* Skip return_trace parameters */ - ptr += 1; - } else if (cpp == beam_return_to_trace) { - have_return_to_trace = !0; /* Record next cp */ - } - } while (ptr < STACK_START(c_p)) { if (is_catch(*ptr)) { if (active_catches) goto found_catch; @@ -1583,6 +1490,8 @@ terminate_proc(Process* c_p, Eterm Value) if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) { Value = add_stacktrace(c_p, Value, c_p->ftrace); } + c_p->ftrace = NIL; + /* EXF_LOG is a primary exception flag */ if (c_p->freason & EXF_LOG) { int alive = erts_is_alive; @@ -1664,6 +1573,54 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { return Value; } + +static void +gather_stacktrace(Process* p, struct StackTrace* s, int depth) +{ + BeamInstr *prev; + Eterm *ptr; + + if (depth == 0) { + return; + } + + prev = s->depth ? s->trace[s->depth - 1] : s->pc; + ptr = p->stop; + + /* + * Traverse the stack backwards and add all unique continuation + * pointers to the buffer, up to the maximum stack trace size. + * + * Skip trace stack frames. + */ + + ASSERT(ptr >= STACK_TOP(p) && ptr <= STACK_START(p)); + + while (ptr < STACK_START(p) && depth > 0) { + if (is_CP(*ptr)) { + BeamInstr *cp = cp_val(*ptr); + + if (cp == beam_exception_trace || cp == beam_return_trace) { + ptr += 3; + } else if (cp == beam_return_time_trace) { + ptr += 2; + } else if (cp == beam_return_to_trace) { + ptr += 1; + } else { + if (cp != prev) { + /* Record non-duplicates only */ + prev = cp; + s->trace[s->depth++] = cp - 1; + depth--; + } + ptr++; + } + } else { + ptr++; + } + } +} + /* * Quick-saving the stack trace in an internal form on the heap. Note * that c_p->ftrace will point to a cons cell which holds the given args @@ -1750,11 +1707,6 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, s->trace[s->depth++] = pc; depth--; } - /* Save second stack entry if CP is valid and different from pc */ - if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = NULL; args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */ } else { @@ -1762,9 +1714,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, non_bif_stacktrace: s->current = c_p->current; - /* + /* * For a function_clause error, the arguments are in the beam - * registers, c_p->cp is valid, and c_p->current is set. + * registers and c_p->current is set. */ if ( (GET_EXC_INDEX(s->freason)) == (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { @@ -1772,18 +1724,8 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, ASSERT(s->current); a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ - /* Save first stack entry */ - ASSERT(c_p->cp); - if (depth > 0) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = NULL; /* Ignore pc */ } else { - if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = pc; } } @@ -1796,80 +1738,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, } /* Save the actual stack trace */ - erts_save_stacktrace(c_p, s, depth); + gather_stacktrace(c_p, s, depth); } void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) { - if (depth > 0) { - Eterm *ptr; - BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; - - /* - * Traverse the stack backwards and add all unique continuation - * pointers to the buffer, up to the maximum stack trace size. - * - * Skip trace stack frames. - */ - ptr = p->stop; - if (ptr < STACK_START(p) && - (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) && - p->cp) { - /* Cannot follow cp here - code may be unloaded */ - BeamInstr *cpp = p->cp; - int trace_cp; - if (cpp == beam_exception_trace || cpp == beam_return_trace) { - /* Skip return_trace parameters */ - ptr += 2; - trace_cp = 1; - } else if (cpp == beam_return_to_trace) { - /* Skip return_to_trace parameters */ - ptr += 1; - trace_cp = 1; - } - else { - trace_cp = 0; - } - if (trace_cp && s->pc == cpp) { - /* - * If process 'cp' points to a return/exception trace - * instruction and 'cp' has been saved as 'pc' in - * stacktrace, we need to update 'pc' in stacktrace - * with the actual 'cp' located on the top of the - * stack; otherwise, we will lose the top stackframe - * when building the stack trace. - */ - ASSERT(is_CP(p->stop[0])); - s->pc = cp_val(p->stop[0]); - } - } - while (ptr < STACK_START(p) && depth > 0) { - if (is_CP(*ptr)) { - if (*cp_val(*ptr) == i_return_trace) { - /* Skip stack frame variables */ - do ++ptr; while (is_not_CP(*ptr)); - /* Skip return_trace parameters */ - ptr += 2; - } else if (*cp_val(*ptr) == i_return_to_trace) { - /* Skip stack frame variables */ - do ++ptr; while (is_not_CP(*ptr)); - } else { - BeamInstr *cp = cp_val(*ptr); - if (cp != prev) { - /* Record non-duplicates only */ - prev = cp; - s->trace[s->depth++] = cp - 1; - depth--; - } - ptr++; - } - } else ptr++; - } - } + gather_stacktrace(p, s, depth); } /* @@ -2128,95 +2003,66 @@ apply_bif_error_adjustment(Process *p, Export *ep, Eterm *reg, Uint arity, BeamInstr *I, Uint stack_offset) { + int apply_only; + Uint need; + + need = stack_offset /* bytes */ / sizeof(Eterm); + apply_only = stack_offset == 0; + /* * I is only set when the apply is a tail call, i.e., * from the instructions i_apply_only, i_apply_last_P, * and apply_last_IP. */ - if (I - && BeamIsOpCode(ep->beam[0], op_apply_bif) - && (ep == bif_export[BIF_error_1] - || ep == bif_export[BIF_error_2] - || ep == bif_export[BIF_exit_1] - || ep == bif_export[BIF_throw_1])) { - /* - * We are about to tail apply one of the BIFs - * erlang:error/1, erlang:error/2, erlang:exit/1, - * or erlang:throw/1. Error handling of these BIFs is - * special! - * - * We need 'p->cp' to point into the calling - * function when handling the error after the BIF has - * been applied. This in order to get the topmost - * stackframe correct. Without the following adjustment, - * 'p->cp' will point into the function that called - * current function when handling the error. We add a - * dummy stackframe in order to achieve this. - * - * Note that these BIFs unconditionally will cause - * an exception to be raised. That is, our modifications - * of 'p->cp' as well as the stack will be corrected by - * the error handling code. - * - * If we find an exception/return-to trace continuation - * pointer as the topmost continuation pointer, we do not - * need to do anything since the information already will - * be available for generation of the stacktrace. - */ - int apply_only = stack_offset == 0; - BeamInstr *cpp; + if (!(I && (ep->bif_number == BIF_error_1 || + ep->bif_number == BIF_error_2 || + ep->bif_number == BIF_exit_1 || + ep->bif_number == BIF_throw_1))) { + return; + } - if (apply_only) { - ASSERT(p->cp != NULL); - cpp = p->cp; - } - else { - ASSERT(is_CP(p->stop[0])); - cpp = cp_val(p->stop[0]); - } + /* + * We are about to tail apply one of the BIFs erlang:error/1, + * erlang:error/2, erlang:exit/1, or erlang:throw/1. Error handling of + * these BIFs is special! + * + * We need the topmost continuation pointer to point into the calling + * function when handling the error after the BIF has been applied. This in + * order to get the topmost stackframe correct. + * + * Note that these BIFs will unconditionally cause an exception to be + * raised. That is, our modifications of the stack will be corrected by the + * error handling code. + */ + if (need == 0) { + need = 1; /* i_apply_only */ + } - if (cpp != beam_exception_trace - && cpp != beam_return_trace - && cpp != beam_return_to_trace) { - Uint need = stack_offset /* bytes */ / sizeof(Eterm); - if (need == 0) - need = 1; /* i_apply_only */ - if (p->stop - p->htop < need) - erts_garbage_collect(p, (int) need, reg, arity+1); - p->stop -= need; - - if (apply_only) { - /* - * Called from the i_apply_only instruction. - * - * 'p->cp' contains continuation pointer pointing - * into the function that called current function. - * We push that continuation pointer onto the stack, - * and set 'p->cp' to point into current function. - */ + if (p->stop - p->htop < need) { + erts_garbage_collect(p, (int) need, reg, arity+1); + } - p->stop[0] = make_cp(p->cp); - p->cp = I; - } - else { - /* - * Called from an i_apply_last_p, or apply_last_IP, - * instruction. - * - * Calling instruction will after we return read - * a continuation pointer from the stack and write - * it to 'p->cp', and then remove the topmost - * stackframe of size 'stack_offset'. - * - * We have sized the dummy-stackframe so that it - * will be removed by the instruction we currently - * are executing, and leave the stackframe that - * normally would have been removed intact. - * - */ - p->stop[0] = make_cp(I); - } - } + if (apply_only) { + /* + * Called from the i_apply_only instruction. + * + * Push the continuation pointer for the current function to the stack. + */ + p->stop -= need; + p->stop[0] = make_cp(I); + } else { + /* + * Called from an i_apply_last_* instruction. + * + * The calling instruction will deallocate a stack frame of size + * 'stack_offset'. + * + * Push the continuation pointer for the current function to the stack, + * and then add a dummy stackframe for the i_apply_last* instruction + * to discard. + */ + p->stop[0] = make_cp(I); + p->stop -= need; } } @@ -2437,10 +2283,10 @@ erts_hibernate(Process* c_p, Eterm* reg) c_p->arg_reg[0] = module; c_p->arg_reg[1] = function; c_p->arg_reg[2] = args; - c_p->stop = STACK_START(c_p); + c_p->stop = c_p->hend - 1; /* Keep first continuation pointer */ + ASSERT(c_p->stop[0] == make_cp(beam_apply+1)); c_p->catches = 0; c_p->i = beam_apply; - c_p->cp = (BeamInstr *) beam_apply+1; /* * If there are no waiting messages, garbage collect and @@ -2460,7 +2306,7 @@ erts_hibernate(Process* c_p, Eterm* reg) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; + c_p->current = &bif_trap_export[BIF_hibernate_3].info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -3268,10 +3114,10 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { - return 0; + return 0; } - return ep->addressv[erts_active_code_ix()] == ep->beam && - BeamIsOpCode(ep->beam[0], op_apply_bif); + + return ep->bif_number != -1; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 35f2ea6688..3fc3b8168e 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -141,7 +141,7 @@ typedef struct { * eventually patch with a pointer into * the export entry. */ - BifFunction bf; /* Pointer to BIF function if BIF; + Export *bif; /* Pointer to export entry if BIF; * NULL otherwise. */ } ImportEntry; @@ -315,6 +315,7 @@ typedef struct LoaderState { * (or 0 if there is no on_load function) */ int otp_20_or_higher; /* Compiled with OTP 20 or higher */ + unsigned max_opcode; /* Highest opcode used in module */ /* * Atom table. @@ -844,17 +845,23 @@ erts_finish_loading(Binary* magic, Process* c_p, if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->beam) { - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + + DBG_CHECK_EXPORT(ep, code_ix); + + if (ep->addressv[code_ix] == ep->trampoline.raw) { + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, &ep->info); - ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; - ep->beam[1] = 0; + + erts_clear_export_break(mod_tab_p, ep); + + ep->addressv[code_ix] = + (BeamInstr*)ep->trampoline.breakpoint.address; + ep->trampoline.breakpoint.address = 0; + + ASSERT(ep->addressv[code_ix] != ep->trampoline.raw); } - ASSERT(ep->beam[1] == 0); + ASSERT(ep->trampoline.breakpoint.address == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -1470,15 +1477,14 @@ load_import_table(LoaderState* stp) } stp->import[i].arity = arity; stp->import[i].patches = 0; - stp->import[i].bf = NULL; + stp->import[i].bif = NULL; /* - * If the export entry refers to a BIF, get the pointer to - * the BIF function. + * If the export entry refers to a BIF, save a pointer to the BIF entry. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (BeamIsOpCode(e->beam[0], op_apply_bif)) { - stp->import[i].bf = (BifFunction) e->beam[1]; + if (e->bif_number != -1) { + stp->import[i].bif = e; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1529,33 +1535,6 @@ read_export_table(LoaderState* stp) LoadError2(stp, "export table entry %u: label %u not resolved", i, n); } stp->export[i].address = address = stp->codev + value; - - /* - * Find out if there is a BIF with the same name. - */ - - if (!is_bif(stp->module, func, arity)) { - continue; - } - - /* - * This is a stub for a BIF. - * - * It should not be exported, and the information in its - * func_info instruction should be invalidated so that it - * can be filtered out by module_info(functions) and by - * any other functions that walk through all local functions. - */ - - if (stp->labels[n].num_patches > 0) { - LoadError3(stp, "there are local calls to the stub for " - "the BIF %T:%T/%d", - stp->module, func, arity); - } - stp->export[i].address = NULL; - address[-1] = 0; - address[-2] = NIL; - address[-3] = NIL; } return 1; @@ -1563,31 +1542,33 @@ read_export_table(LoaderState* stp) return 0; } - static int is_bif(Eterm mod, Eterm func, unsigned arity) { - Export* e = erts_active_export_entry(mod, func, arity); - if (e == NULL) { - return 0; - } - if (! BeamIsOpCode(e->beam[0], op_apply_bif)) { - return 0; - } - if (mod == am_erlang && func == am_apply && arity == 3) { - /* - * erlang:apply/3 is a special case -- it is implemented - * as an instruction and it is OK to redefine it. - */ - return 0; + Export *e = erts_active_export_entry(mod, func, arity); + + if (e != NULL) { + return e->bif_number != -1; } - return 1; + + return 0; } static int read_lambda_table(LoaderState* stp) { unsigned int i; + unsigned int otp_22_or_lower; + + /* + * Determine whether this module was compiled with OTP 22 or lower + * by looking at the max opcode number. The compiler in OTP 23 will + * always set the max opcode to the opcode for `swap` (whether + * actually used or not) so that a module compiled for OTP 23 + * cannot be loaded in earlier versions. + */ + + otp_22_or_lower = stp->max_opcode < genop_swap_2; GetInt(stp, 4, stp->num_lambdas); if (stp->num_lambdas > stp->lambdas_allocated) { @@ -1619,6 +1600,29 @@ read_lambda_table(LoaderState* stp) GetInt(stp, 4, Index); GetInt(stp, 4, stp->lambdas[i].num_free); GetInt(stp, 4, OldUniq); + + /* + * Fun entries are now keyed by the explicit ("new") index in + * the fun entry. That allows multiple make_fun2 instructions + * to share the same fun entry (when the `fun F/A` syntax is + * used). Before OTP 23, fun entries were keyed by the old + * index, which is the order of the entries in the fun + * chunk. Each make_fun2 needed to refer to its own fun entry. + * + * Modules compiled before OTP 23 can safely be loaded if the + * old index and the new index are equal. That is true for all + * modules compiled with OTP R15 and later. + */ + if (otp_22_or_lower && i != Index) { + /* + * Compiled with a compiler before OTP R15B. The new indices + * are not reliable, so it is not safe to load this module. + */ + LoadError2(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler " + "(old-style fun with indices: %d/%d)", + i, Index); + } fe = erts_put_fun_entry2(stp->module, OldUniq, i, stp->mod_md5, Index, arity-stp->lambdas[i].num_free); stp->lambdas[i].fe = fe; @@ -1839,7 +1843,6 @@ read_code_header(LoaderState* stp) { unsigned head_size; unsigned version; - unsigned opcode_max; int i; /* @@ -1871,8 +1874,8 @@ read_code_header(LoaderState* stp) /* * Verify the number of the highest opcode used. */ - GetInt(stp, 4, opcode_max); - if (opcode_max > MAX_GENERIC_OPCODE) { + GetInt(stp, 4, stp->max_opcode); + if (stp->max_opcode > MAX_GENERIC_OPCODE) { LoadError2(stp, "This BEAM file was compiled for a later version" " of the run-time system than " ERLANG_OTP_RELEASE ".\n" @@ -1880,7 +1883,7 @@ read_code_header(LoaderState* stp) ERLANG_OTP_RELEASE " compiler.\n" " (Use of opcode %d; this emulator supports " "only up to %d.)", - opcode_max, MAX_GENERIC_OPCODE); + stp->max_opcode, MAX_GENERIC_OPCODE); } GetInt(stp, 4, stp->num_labels); @@ -1919,8 +1922,6 @@ read_code_header(LoaderState* stp) code = stp->codev = (BeamInstr*) &stp->hdr->functions; \ } \ } while (0) - -#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) static void init_label(Label* lp) { @@ -2500,10 +2501,14 @@ load_code(LoaderState* stp) if (i >= stp->num_imports) { LoadError1(stp, "invalid import table index %d", i); } - if (stp->import[i].bf == NULL) { + if (stp->import[i].bif == NULL) { LoadError1(stp, "not a BIF: import table index %d", i); } - code[ci++] = (BeamInstr) stp->import[i].bf; + { + int bif_index = stp->import[i].bif->bif_number; + BifEntry *bif_entry = &bif_table[bif_index]; + code[ci++] = (BeamInstr) bif_entry->f; + } break; case 'P': /* Byte offset into tuple or stack */ case 'Q': /* Like 'P', but packable */ @@ -2711,36 +2716,30 @@ load_code(LoaderState* stp) num_trailing_f = 0; } #endif + CodeNeed(1); switch (tmp_op->a[arg].type) { case TAG_i: - CodeNeed(1); code[ci++] = make_small(tmp_op->a[arg].val); break; case TAG_u: case TAG_a: case TAG_v: - CodeNeed(1); code[ci++] = tmp_op->a[arg].val; break; case TAG_f: - CodeNeed(1); register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); ci++; break; case TAG_x: - CodeNeed(1); code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: - CodeNeed(1); code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_n: - CodeNeed(1); code[ci++] = NIL; break; case TAG_q: - CodeNeed(1); new_literal_patch(stp, ci); code[ci++] = tmp_op->a[arg].val; break; @@ -2811,18 +2810,43 @@ load_code(LoaderState* stp) switch (stp->specific_op) { case op_i_func_info_IaaI: { + int padding_required; Sint offset; + if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %u)", stp->num_functions); } - if (stp->may_load_nif) { + /* Native function calls may be larger than their stubs, so + * we'll need to make sure any potentially-native function stub + * is padded with enough room. + * + * Note that the padding is applied for the previous function, + * not the current one, so we check whether the old F/A is + * a BIF. */ + padding_required = last_func_start && (stp->may_load_nif || + is_bif(stp->module, stp->function, stp->arity)); + + /* + * Save context for error messages. + */ + stp->function = code[ci-2]; + stp->arity = code[ci-1]; + + /* + * Save current offset of into the line instruction array. + */ + if (stp->func_line) { + stp->func_line[function_number] = stp->current_li; + } + + if (padding_required) { const int finfo_ix = ci - FUNC_INFO_SZ; - if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) { + if (finfo_ix - last_func_start < BEAM_NATIVE_MIN_FUNC_SZ) { /* Must make room for call_nif op */ - int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start); - ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ); + int pad = BEAM_NATIVE_MIN_FUNC_SZ - (finfo_ix - last_func_start); + ASSERT(pad > 0 && pad < BEAM_NATIVE_MIN_FUNC_SZ); CodeNeed(pad); sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FUNC_INFO_SZ*sizeof(BeamInstr)); @@ -2833,20 +2857,6 @@ load_code(LoaderState* stp) } last_func_start = ci; - /* - * Save current offset of into the line instruction array. - */ - - if (stp->func_line) { - stp->func_line[function_number] = stp->current_li; - } - - /* - * Save context for error messages. - */ - stp->function = code[ci-2]; - stp->arity = code[ci-1]; - /* When this assert is triggered, it is normally a sign that the size of the ops.tab i_func_info instruction is not the same as FUNC_INFO_SZ */ @@ -2876,7 +2886,6 @@ load_code(LoaderState* stp) case op_i_bs_match_string_yfWW: new_string_patch(stp, ci-1); break; - case op_catch_yf: /* code[ci-3] &&lb_catch_yf * code[ci-2] y-register offset in E @@ -2977,6 +2986,7 @@ load_code(LoaderState* stp) #define succ3(St, X, Y) ((X).type == (Y).type && (X).val + 3 == (Y).val) #define succ4(St, X, Y) ((X).type == (Y).type && (X).val + 4 == (Y).val) +#define offset(St, X, Y, Offset) ((X).type == (Y).type && (X).val + Offset == (Y).val) #ifdef NO_FPE_SIGNALS #define no_fpe_signals(St) 1 @@ -3131,27 +3141,6 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) return 0; } -static int -is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val+2 <= Reg.val; -} - -static int -is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val <= Reg.val; -} - -static int -is_killed_by_call_fun(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val+1 <= Reg.val; -} - /* * Test whether register Reg is killed by make_fun instruction that * creates the fun given by index idx. @@ -3171,14 +3160,23 @@ is_killed_by_make_fun(LoaderState* stp, GenOpArg Reg, GenOpArg idx) } } -/* - * Test whether register Reg is killed by the send instruction that follows. - */ - +/* Test whether Bif is "heavy" and should always go through its export entry */ static int -is_killed_by_send(LoaderState* stp, GenOpArg Reg) +is_heavy_bif(LoaderState* stp, GenOpArg Bif) { - return Reg.type == TAG_x && 2 <= Reg.val; + Export *ep; + + if (Bif.type != TAG_u || Bif.val >= stp->num_imports) { + return 0; + } + + ep = stp->import[Bif.val].bif; + + if (ep) { + return bif_table[ep->bif_number].kind == BIF_KIND_HEAVY; + } + + return 0; } /* @@ -3324,8 +3322,8 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } goto generic; } - } else { - GENOP_NAME_ARITY(op, i_bs_get_integer, 6); + } else if (Size.type == TAG_x || Size.type == TAG_y) { + GENOP_NAME_ARITY(op, i_bs_get_integer, 6); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Live; @@ -3335,6 +3333,9 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[5] = Dst; op->next = NULL; return op; + } else { + /* Invalid literal size. */ + goto error; } op->next = NULL; return op; @@ -3391,7 +3392,7 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[4] = Flags; op->a[5] = Dst; } - } else { + } else if (Size.type == TAG_x || Size.type == TAG_y) { GENOP_NAME_ARITY(op, i_bs_get_binary2, 6); op->a[0] = Ms; op->a[1] = Fail; @@ -3400,6 +3401,9 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[4].type = TAG_u; op->a[4].val = (Unit.val << 3) | Flags.val; op->a[5] = Dst; + } else { + /* Invalid literal size. */ + goto error; } op->next = NULL; return op; @@ -3636,12 +3640,20 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, goto error; } } - } else { + } else if (Size.type == TAG_x || Size.type == TAG_y) { GENOP_NAME_ARITY(op, i_bs_skip_bits2, 4); op->a[0] = Ms; op->a[1] = Size; op->a[2] = Fail; op->a[3] = Unit; + } else { + /* + * Invalid literal size. Can only happen if compiler + * optimizations are selectively disabled. For example, + * at the time of writing, [no_copt, no_type_opt] will allow + * skip instructions with invalid sizes to slip through. + */ + goto error; } op->next = NULL; return op; @@ -5208,27 +5220,52 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ for (i = 0; i < stp->num_exps; i++) { - Export* ep; - BeamInstr* address = stp->export[i].address; + Export* ep; + BeamInstr* address = stp->export[i].address; - if (address == NULL) { - /* Skip stub for a BIF */ - continue; - } - ep = erts_export_put(stp->module, stp->export[i].function, - stp->export[i].arity); - if (on_load) { - /* - * on_load: Don't make any of the exported functions - * callable yet. Keep any function in the current - * code callable. - */ - ep->beam[1] = (BeamInstr) address; - } - else + ep = erts_export_put(stp->module, + stp->export[i].function, + stp->export[i].arity); + + /* Fill in BIF stubs with a proper call to said BIF. */ + if (ep->bif_number != -1) { + erts_write_bif_wrapper(ep, address); + } + + if (on_load) { + /* + * on_load: Don't make any of the exported functions + * callable yet. Keep any function in the current + * code callable. + */ + ep->trampoline.not_loaded.deferred = (BeamInstr) address; + } else { ep->addressv[erts_staging_code_ix()] = address; + } } +#ifdef DEBUG + /* Ensure that we've loaded stubs for all BIFs in this module. */ + for (i = 0; i < BIF_SIZE; i++) { + BifEntry *entry = &bif_table[i]; + + if (stp->module == entry->module) { + Export *ep = erts_export_put(entry->module, + entry->name, + entry->arity); + BeamInstr *addr = ep->addressv[erts_staging_code_ix()]; + + if (!ErtsInArea(addr, stp->codev, stp->ci * sizeof(BeamInstr))) { + erts_exit(ERTS_ABORT_EXIT, + "Module %T doesn't export BIF %T/%i\n", + entry->module, + entry->name, + entry->arity); + } + } + } +#endif + /* * Import functions and patch all callers. */ @@ -5333,6 +5370,16 @@ transform_engine(LoaderState* st) if (((1 << instr->a[ap].type) & mask) == 0) goto restart; break; +#if defined(TOP_is_type_next_arg) + case TOP_is_type_next_arg: + mask = *pc++; + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + ap++; + break; +#endif case TOP_pred: i = *pc++; switch (i) { @@ -5362,6 +5409,18 @@ transform_engine(LoaderState* st) if (*pc++ != instr->a[ap].val) goto restart; break; +#if defined(TOP_is_type_eq_next_arg) + case TOP_is_type_eq_next_arg: + mask = *pc++; + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + if (*pc++ != instr->a[ap].val) + goto restart; + ap++; + break; +#endif case TOP_is_same_var: ASSERT(ap < instr->arity); i = *pc++; @@ -5400,15 +5459,16 @@ transform_engine(LoaderState* st) i = instr->a[ap].val; ASSERT(i < st->num_imports); - if (i >= st->num_imports || st->import[i].bf == NULL) - goto restart; - if (bif_number != -1 && - bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { + if (i >= st->num_imports || st->import[i].bif == NULL) goto restart; - } + if (bif_number != -1) { + Export *bif = st->import[i].bif; + if (bif->bif_number != bif_number) { + goto restart; + } + } } break; - #endif #if defined(TOP_is_not_bif) case TOP_is_not_bif: @@ -5438,7 +5498,7 @@ transform_engine(LoaderState* st) * they are special. */ if (i < st->num_imports) { - if (st->import[i].bf != NULL || + if (st->import[i].bif != NULL || (st->import[i].module == am_erlang && st->import[i].function == am_apply && (st->import[i].arity == 2 || st->import[i].arity == 3))) { @@ -5478,7 +5538,42 @@ transform_engine(LoaderState* st) var[i].val = instr->a[ap].val; ap++; break; - +#if defined(TOP_is_type_set_var_next_arg) + case TOP_is_type_set_var_next_arg: + mask = pc[0]; + i = pc[1]; + ASSERT(i < TE_MAX_VARS); + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + ASSERT(i < TE_MAX_VARS); + var[i] = instr->a[ap]; + ap++; + pc += 2; + break; +#endif +#if defined(TOP_is_type_eq_set_var_next_arg) + case TOP_is_type_eq_set_var_next_arg: + { + Eterm val; + mask = pc[0]; + val = pc[1]; + i = pc[2]; + ASSERT(i < TE_MAX_VARS); + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + if (val != instr->a[ap].val) + goto restart; + ASSERT(i < TE_MAX_VARS); + var[i] = instr->a[ap]; + ap++; + pc += 3; + } + break; +#endif #if defined(TOP_rest_args) case TOP_rest_args: { @@ -5494,19 +5589,27 @@ transform_engine(LoaderState* st) case TOP_commit: instr = instr->next; /* The next_instr was optimized away. */ keep = instr; - st->genop = instr; -#ifdef DEBUG - instr = 0; -#endif break; +#if defined(TOP_commit_new_instr) + case TOP_commit_new_instr: + /* + * Reuse the last instruction on the left side instead of + * allocating a new instruction. Note that this is not + * safe if TOP_rest_args has been executed; therefore, + * this combined instruction is never used when that is + * the case. + */ + ASSERT(instr->a == instr->def_args); + keep = instr; + instr->op = op = *pc++; + instr->arity = gen_opc[op].arity; + ap = 0; + break; +#endif #if defined(TOP_keep) case TOP_keep: /* Keep the current instruction unchanged. */ keep = instr; - st->genop = instr; -#ifdef DEBUG - instr = 0; -#endif break; #endif #if defined(TOP_call_end) @@ -5535,11 +5638,12 @@ transform_engine(LoaderState* st) keep = instr->next; /* The next_instr was optimized away. */ *lastp = keep; - st->genop = new_instr; + instr = new_instr; } /* FALLTHROUGH */ #endif case TOP_end: + st->genop = instr; while (first != keep) { GenOp* next = first->next; FREE_GENOP(st, first); @@ -5550,28 +5654,28 @@ transform_engine(LoaderState* st) /* * Note that the instructions are generated in reverse order. */ - NEW_GENOP(st, instr); - instr->next = st->genop; - st->genop = instr; - instr->op = op = *pc++; - instr->arity = gen_opc[op].arity; - ap = 0; - break; + { + GenOp* new_instr; + NEW_GENOP(st, new_instr); + new_instr->next = instr; + instr = new_instr; + instr->op = op = *pc++; + instr->arity = gen_opc[op].arity; + ap = 0; + } + break; #ifdef TOP_rename case TOP_rename: instr->op = op = *pc++; instr->arity = gen_opc[op].arity; return TE_OK; #endif - case TOP_store_type: - i = *pc++; - instr->a[ap].type = i; - instr->a[ap].val = 0; - break; - case TOP_store_val: - i = *pc++; - instr->a[ap].val = i; - break; + case TOP_store_val_next_arg: + instr->a[ap].type = pc[0]; + instr->a[ap].val = pc[1]; + ap++; + pc += 2; + break; case TOP_store_var_next_arg: i = *pc++; ASSERT(i < TE_MAX_VARS); @@ -5599,6 +5703,23 @@ transform_engine(LoaderState* st) break; case TOP_fail: return TE_FAIL; +#if defined(TOP_skip_unless) + case TOP_skip_unless: + /* + * Note that the caller of transform_engine() guarantees that + * there is always a second instruction available. + */ + ASSERT(instr); + if (instr->next->op != pc[0]) { + /* The second instruction is wrong. Skip ahead. */ + pc += pc[1] + 2; + ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ + } else { + /* Correct second instruction. */ + pc += 2; + } + break; +#endif default: ASSERT(0); } @@ -6283,12 +6404,12 @@ exported_from_module(Process* p, /* Process whose heap to use. */ if (ep->info.mfa.module == mod) { Eterm tuple; - - if (ep->addressv[code_ix] == ep->beam && - BeamIsOpCode(ep->beam[0], op_call_error_handler)) { - /* There is a call to the function, but it does not exist. */ - continue; - } + + if (ep->addressv[code_ix] == ep->trampoline.raw && + BeamIsOpCode(ep->trampoline.op, op_call_error_handler)) { + /* There is a call to the function, but it does not exist. */ + continue; + } if (hp == hend) { int need = 10 * 5; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 156c3c45e2..e7127c5b08 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -106,7 +106,7 @@ typedef struct beam_code_header { }BeamCodeHeader; -# define BEAM_NIF_MIN_FUNC_SZ 4 +# define BEAM_NATIVE_MIN_FUNC_SZ 4 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 04e9db1f8e..9fc6ba4ac6 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -77,6 +77,8 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3) Eterm pid; so.flags = erts_default_spo_flags; + so.opts = NIL; + so.tag = am_spawn_reply; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -309,6 +311,15 @@ demonitor(Process *c_p, Eterm ref, Eterm *multip) int deleted; ErtsDSigSendContext ctx; + if (mon->flags & ERTS_ML_FLG_SPAWN_PENDING) { + /* + * Not allowed to remove this until spawn + * operation has succeeded; restore monitor... + */ + erts_monitor_tree_insert(&ERTS_P_MONITORS(c_p), mon); + return am_false; + } + ASSERT(is_external_pid(to) || is_node_name_atom(to)); if (is_external_pid(to)) @@ -701,8 +712,11 @@ BIF_RETTYPE spawn_link_3(BIF_ALIST_3) { ErlSpawnOpts so; Eterm pid; + Eterm tmp_heap[2]; so.flags = erts_default_spo_flags|SPO_LINK; + so.opts = CONS(&tmp_heap[0], am_link, NIL); + so.tag = am_spawn_reply; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -716,132 +730,39 @@ BIF_RETTYPE spawn_link_3(BIF_ALIST_3) /**********************************************************************/ -BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) +BIF_RETTYPE spawn_opt_4(BIF_ALIST_4) { ErlSpawnOpts so; Eterm pid; - Eterm* tp; - Eterm ap; - Eterm arg; Eterm res; + int opts_error; /* - * Check that the first argument is a tuple of four elements. + * Fail order: + * - Bad types + * - Bad options */ - if (is_not_tuple(BIF_ARG_1)) { - error: - BIF_ERROR(BIF_P, BADARG); - } - tp = tuple_val(BIF_ARG_1); - if (*tp != make_arityval(4)) - goto error; - - /* - * Store default values for options. - */ - so.flags = erts_default_spo_flags|SPO_USE_ARGS; - so.min_heap_size = H_MIN_SIZE; - so.min_vheap_size = BIN_VH_MIN_SIZE; - so.max_heap_size = H_MAX_SIZE; - so.max_heap_flags = H_MAX_FLAGS; - so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); - so.scheduler = 0; - - /* - * Walk through the option list. - */ - ap = tp[4]; - while (is_list(ap)) { - arg = CAR(list_val(ap)); - if (arg == am_link) { - so.flags |= SPO_LINK; - } else if (arg == am_monitor) { - so.flags |= SPO_MONITOR; - } else if (is_tuple(arg)) { - Eterm* tp2 = tuple_val(arg); - Eterm val; - if (*tp2 != make_arityval(2)) - goto error; - arg = tp2[1]; - val = tp2[2]; - if (arg == am_priority) { - if (val == am_max) - so.priority = PRIORITY_MAX; - else if (val == am_high) - so.priority = PRIORITY_HIGH; - else if (val == am_normal) - so.priority = PRIORITY_NORMAL; - else if (val == am_low) - so.priority = PRIORITY_LOW; - else - goto error; - } else if (arg == am_message_queue_data) { - switch (val) { - case am_on_heap: - so.flags &= ~SPO_OFF_HEAP_MSGQ; - so.flags |= SPO_ON_HEAP_MSGQ; - break; - case am_off_heap: - so.flags &= ~SPO_ON_HEAP_MSGQ; - so.flags |= SPO_OFF_HEAP_MSGQ; - break; - default: - goto error; - } - } else if (arg == am_min_heap_size && is_small(val)) { - Sint min_heap_size = signed_val(val); - if (min_heap_size < 0) { - goto error; - } else if (min_heap_size < H_MIN_SIZE) { - so.min_heap_size = H_MIN_SIZE; - } else { - so.min_heap_size = erts_next_heap_size(min_heap_size, 0); - } - } else if (arg == am_max_heap_size) { - if (!erts_max_heap_size(val, &so.max_heap_size, &so.max_heap_flags)) - goto error; - } else if (arg == am_min_bin_vheap_size && is_small(val)) { - Sint min_vheap_size = signed_val(val); - if (min_vheap_size < 0) { - goto error; - } else if (min_vheap_size < BIN_VH_MIN_SIZE) { - so.min_vheap_size = BIN_VH_MIN_SIZE; - } else { - so.min_vheap_size = erts_next_heap_size(min_vheap_size, 0); - } - } else if (arg == am_fullsweep_after && is_small(val)) { - Sint max_gen_gcs = signed_val(val); - if (max_gen_gcs < 0) { - goto error; - } else { - so.max_gen_gcs = max_gen_gcs; - } - } else if (arg == am_scheduler && is_small(val)) { - Sint scheduler = signed_val(val); - if (scheduler < 0 || erts_no_schedulers < scheduler) - goto error; - so.scheduler = (int) scheduler; - } else { - goto error; - } - } else { - goto error; - } - ap = CDR(list_val(ap)); - } - if (is_not_nil(ap)) { - goto error; - } - - if (so.max_heap_size != 0 && so.max_heap_size < so.min_heap_size) { - goto error; + opts_error = erts_parse_spawn_opts(&so, BIF_ARG_4, NULL, 0); + if (opts_error) { + Sint arity; + if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + arity = erts_list_length(BIF_ARG_3); + if (arity < 0) + BIF_ERROR(BIF_P, BADARG); + if (arity > MAX_SMALL) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + if (opts_error > 0) + BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(BIF_P, BADARG); } - + /* * Spawn the process. */ - pid = erl_create_process(BIF_P, tp[1], tp[2], tp[3], &so); + so.opts = BIF_ARG_4; + so.tag = am_spawn_reply; + pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); } else if (so.flags & SPO_MONITOR) { @@ -859,6 +780,130 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) } } +/**********************************************************************/ + +BIF_RETTYPE erts_internal_spawn_request_4(BIF_ALIST_4) +{ + ErlSpawnOpts so; + Eterm tmp_heap_mfna[4]; + Eterm tmp_heap_alist[4 + 2]; + Sint arity; + int opts_error; + Eterm tag, tmp, error; + + if (!is_atom(BIF_ARG_1)) + goto badarg; + if (!is_atom(BIF_ARG_2)) + goto badarg; + arity = erts_list_length(BIF_ARG_3); + if (arity < 0) + goto badarg; + + /* + * Fail order: + * - Bad types + * - Bad options + */ + opts_error = erts_parse_spawn_opts(&so, BIF_ARG_4, &tag, !0); + if (arity > MAX_SMALL) + goto system_limit; + if (opts_error) { + if (opts_error > 0) + goto badarg; + goto badopt; + } + + /* Make argument list for erts_internal:spawn_init/1 */ + tmp = TUPLE3(&tmp_heap_alist[0], BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + tmp = CONS(&tmp_heap_alist[4], tmp, NIL); + + so.mfa = TUPLE3(&tmp_heap_mfna[0], BIF_ARG_1, BIF_ARG_2, make_small(arity)); + so.flags |= SPO_ASYNC; + so.mref = THE_NON_VALUE; + so.tag = tag; + so.opts = BIF_ARG_4; + + /* + * Spawn the process. + */ + tmp = erl_create_process(BIF_P, am_erts_internal, am_spawn_init, tmp, &so); + if (is_non_value(tmp)) { + switch (so.error_code) { + case SYSTEM_LIMIT: + goto system_limit; + case BADARG: + default: + ERTS_INTERNAL_ERROR("Unexpected error from erl_create_process()"); + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + } + } + + ASSERT(is_internal_pid(tmp)); + + if (ERTS_USE_MODIFIED_TIMING()) { + BIF_TRAP2(erts_delay_trap, BIF_P, so.mref, ERTS_MODIFIED_TIMING_DELAY); + } + else { + BIF_RET(so.mref); + } + +badarg: + BIF_RET(am_badarg); +system_limit: + error = am_system_limit; + goto send_error; +badopt: + error = am_badopt; + /* fall through... */ +send_error: { + Eterm ref = erts_make_ref(BIF_P); + if (!(so.flags & SPO_NO_EMSG)) + erts_send_local_spawn_reply(BIF_P, ERTS_PROC_LOCK_MAIN, NULL, + tag, ref, error, am_undefined); + BIF_RET(ref); + } + +} + +BIF_RETTYPE spawn_request_abandon_1(BIF_ALIST_1) +{ + ErtsMonitor *omon; + + if (is_not_internal_ref(BIF_ARG_1)) { + if (is_not_ref(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + /* Not an outstanding spawn_request of this process... */ + BIF_RET(am_false); + } + + omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(BIF_P), BIF_ARG_1); + if (!omon + || ((omon->flags & (ERTS_ML_FLG_SPAWN_PENDING + | ERTS_ML_FLG_SPAWN_ABANDONED)) + != ERTS_ML_FLG_SPAWN_PENDING)) { + /* Not an outstanding spawn_request of this process... */ + BIF_RET(am_false); + } + + ASSERT(erts_monitor_is_origin(omon)); + + if (omon->flags & ERTS_ML_FLG_SPAWN_LINK) { + /* Leave it for reply... */ + omon->flags |= ERTS_ML_FLG_SPAWN_ABANDONED; + } + else { + /* We don't need it anymore; remove it... */ + ErtsMonitorData *mdp; + erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), omon); + mdp = erts_monitor_to_data(omon); + if (erts_monitor_dist_delete(&mdp->target)) + erts_monitor_release_both(mdp); + else + erts_monitor_release(omon); + } + BIF_RET(am_true); +} + /**********************************************************************/ /* remove a link from a process */ @@ -990,8 +1035,7 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) BIF_RETTYPE get_stacktrace_0(BIF_ALIST_0) { - Eterm t = build_stacktrace(BIF_P, BIF_P->ftrace); - BIF_RET(t); + BIF_RET(NIL); } /**********************************************************************/ @@ -1816,7 +1860,7 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_INTERNAL_ERROR (-6) #define SEND_AWAIT_RESULT (-7) #define SEND_YIELD_CONTINUE (-8) -#define SEND_SYSTEM_LIMIT (-9) +#define SEND_SYSTEM_LIMIT (-9) static Sint remote_send(Process *p, DistEntry *dep, @@ -1915,7 +1959,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " - "incarnation (%d) of this node (%d)\n", + "incarnation (%u) of this node (%u)\n", msg, p->common.id, to, @@ -1959,7 +2003,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " - "incarnation (%d) of this node (%d)\n", + "incarnation (%u) of this node (%u)\n", msg, p->common.id, to, @@ -1987,7 +2031,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, trace_send(p, portid, msg); if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { - seq_trace_update_send(p); + seq_trace_update_serial(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); } @@ -2160,7 +2204,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) break; case SEND_YIELD: if (suspend) { - ERTS_BIF_PREP_YIELD3(retval, bif_export[BIF_send_3], p, to, msg, opts); + ERTS_BIF_PREP_YIELD3(retval, &bif_trap_export[BIF_send_3], p, to, msg, opts); } else { ERTS_BIF_PREP_RET(retval, am_nosuspend); } @@ -2277,7 +2321,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ERTS_BIF_PREP_RET(retval, msg); break; case SEND_YIELD: - ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg); + ERTS_BIF_PREP_YIELD2(retval, &bif_trap_export[BIF_send_2], p, to, msg); break; case SEND_YIELD_RETURN: yield_return: @@ -2584,7 +2628,7 @@ BIF_RETTYPE iolist_size_1(BIF_ALIST_1) } else { ERTS_BIF_ERROR_TRAPPED1(BIF_P, BADARG, - bif_export[BIF_iolist_size_1], + &bif_trap_export[BIF_iolist_size_1], input_list); } @@ -2604,7 +2648,7 @@ BIF_RETTYPE iolist_size_1(BIF_ALIST_1) ESTACK_SAVE(s, &context->stack); erts_set_gc_state(BIF_P, 0); BUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_iolist_size_1], BIF_P, state_mref); + BIF_TRAP1(&bif_trap_export[BIF_iolist_size_1], BIF_P, state_mref); } /**********************************************************************/ @@ -3942,7 +3986,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { erts_halt(pos_int_code); - ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined); + ERTS_BIF_YIELD2(&bif_trap_export[BIF_halt_2], BIF_P, am_undefined, am_undefined); } else { erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -4531,7 +4575,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(am_enabled); case ERTS_SCHDLR_SSPND_YIELD_RESTART: ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_system_flag_2], + BIF_TRAP2(&bif_trap_export[BIF_system_flag_2], BIF_P, BIF_ARG_1, BIF_ARG_2); case ERTS_SCHDLR_SSPND_YIELD_DONE: ERTS_BIF_YIELD_RETURN_X(BIF_P, am_enabled, @@ -4556,7 +4600,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(make_small(old_no)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_system_flag_2], + BIF_TRAP2(&bif_trap_export[BIF_system_flag_2], BIF_P, BIF_ARG_1, BIF_ARG_2); case ERTS_SCHDLR_SSPND_YIELD_DONE: ERTS_BIF_YIELD_RETURN_X(BIF_P, make_small(old_no), @@ -4720,7 +4764,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(make_small(old_no)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_system_flag_2], + BIF_TRAP2(&bif_trap_export[BIF_system_flag_2], BIF_P, BIF_ARG_1, BIF_ARG_2); case ERTS_SCHDLR_SSPND_YIELD_DONE: ERTS_BIF_YIELD_RETURN_X(BIF_P, make_small(old_no), @@ -4870,9 +4914,13 @@ BIF_RETTYPE phash_2(BIF_ALIST_2) BIF_RETTYPE phash2_1(BIF_ALIST_1) { Uint32 hash; - - hash = make_hash2(BIF_ARG_1); - BIF_RET(make_small(hash & ((1L << 27) - 1))); + Eterm trap_state = THE_NON_VALUE; + hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P); + if (trap_state == THE_NON_VALUE) { + BIF_RET(make_small(hash & ((1L << 27) - 1))); + } else { + BIF_TRAP1(&bif_trap_export[BIF_phash2_1], BIF_P, trap_state); + } } BIF_RETTYPE phash2_2(BIF_ALIST_2) @@ -4880,6 +4928,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) Uint32 hash; Uint32 final_hash; Uint32 range; + Eterm trap_state = THE_NON_VALUE; /* Check for special case 2^32 */ if (term_equals_2pow32(BIF_ARG_2)) { @@ -4891,7 +4940,10 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2(BIF_ARG_1); + hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P); + if (trap_state != THE_NON_VALUE) { + BIF_TRAP2(&bif_trap_export[BIF_phash2_2], BIF_P, trap_state, BIF_ARG_2); + } if (range) { final_hash = hash % range; /* [0..range-1] */ } else { @@ -4967,15 +5019,32 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, Eterm (*bif)(BIF_ALIST)) { int i; + sys_memset((void *) ep, 0, sizeof(Export)); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->addressv[i] = ep->beam; + ep->addressv[i] = ep->trampoline.raw; } + + ep->bif_number = -1; + + ep->info.op = op_i_func_info_IaaI; ep->info.mfa.module = m; ep->info.mfa.function = f; ep->info.mfa.arity = a; - ep->beam[0] = BeamOpCodeAddr(op_apply_bif); - ep->beam[1] = (BeamInstr) bif; + + ep->trampoline.op = BeamOpCodeAddr(op_call_bif_W); + ep->trampoline.raw[1] = (BeamInstr)bif; +} + +/* + * Writes a BIF call wrapper to the given address. + */ +void erts_write_bif_wrapper(Export *export, BeamInstr *address) { + BifEntry *entry = &bif_table[export->bif_number]; + + address[0] = BeamOpCodeAddr(op_call_bif_W); + address[1] = (BeamInstr)entry->f; } void erts_init_bif(void) @@ -5027,7 +5096,7 @@ void erts_init_bif(void) } /* - * Scheduling of BIFs via NifExport... + * Scheduling of BIFs via ErtsNativeFunc... */ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -5042,8 +5111,8 @@ schedule(Process *c_p, Process *dirty_shadow_proc, int argc, Eterm *argv) { ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, - mfa, pc, BeamOpCodeAddr(op_apply_bif), + (void) erts_nfunc_schedule(c_p, dirty_shadow_proc, + mfa, pc, BeamOpCodeAddr(op_call_bif_W), dfunc, ifunc, module, function, argc, argv); @@ -5052,23 +5121,23 @@ schedule(Process *c_p, Process *dirty_shadow_proc, static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); - erts_nif_export_restore(BIF_P, nep, BIF_ARG_1); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(BIF_P); + erts_nfunc_restore(BIF_P, nep, BIF_ARG_1); BIF_RET(BIF_ARG_1); } static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(BIF_P); /* * Arity and argument registers already set * correct by call to dirty_bif_trap()... */ - ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); + ASSERT(BIF_P->arity == nep->trampoline.info.mfa.arity); - erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE); + erts_nfunc_restore(BIF_P, nep, THE_NON_VALUE); BIF_P->i = (BeamInstr *) nep->func; BIF_P->freason = TRAP; @@ -5083,8 +5152,8 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) freason = signed_val(BIF_ARG_1); - /* Restore orig info for error and clear nif export in handle_error() */ - freason |= EXF_RESTORE_NIF; + /* Restore orig info for error and clear nif wrapper in handle_error() */ + freason |= EXF_RESTORE_NFUNC; BIF_P->fvalue = BIF_ARG_2; @@ -5122,6 +5191,7 @@ erts_schedule_bif(Process *proc, if (!ERTS_PROC_IS_EXITING(c_p)) { Export *exp; BifFunction dbif, ibif; + BeamInstr call_instr; BeamInstr *pc; /* @@ -5156,29 +5226,41 @@ erts_schedule_bif(Process *proc, if (i == NULL) { ERTS_INTERNAL_ERROR("Missing instruction pointer"); } + + if (BeamIsOpCode(*i, op_i_generic_breakpoint)) { + ErtsCodeInfo *ci; + GenericBp *bp; + + ci = erts_code_to_codeinfo(i); + bp = ci->u.gen_bp; + + call_instr = bp->orig_instr; + } else { + call_instr = *i; + } + #ifdef HIPE - else if (proc->flags & F_HIPE_MODE) { + if (proc->flags & F_HIPE_MODE) { /* Pointer to bif export in i */ exp = (Export *) i; - pc = c_p->cp; + pc = cp_val(c_p->stop[0]); mfa = &exp->info.mfa; - } + } else /* !! This is part of the if clause below !! */ #endif - else if (BeamIsOpCode(*i, op_call_bif_e)) { - /* Pointer to bif export in i+1 */ - exp = (Export *) i[1]; + if (BeamIsOpCode(call_instr, op_call_light_bif_be)) { + /* Pointer to bif export in i+2 */ + exp = (Export *) i[2]; pc = i; mfa = &exp->info.mfa; } - else if (BeamIsOpCode(*i, op_call_bif_only_e)) { - /* Pointer to bif export in i+1 */ - exp = (Export *) i[1]; + else if (BeamIsOpCode(call_instr, op_call_light_bif_only_be)) { + /* Pointer to bif export in i+2 */ + exp = (Export *) i[2]; pc = i; mfa = &exp->info.mfa; } - else if (BeamIsOpCode(*i, op_apply_bif)) { - /* Pointer to bif in i+1, and mfa in i-3 */ - pc = c_p->cp; + else if (BeamIsOpCode(call_instr, op_call_bif_W)) { + pc = cp_val(c_p->stop[0]); mfa = erts_code_to_codemfa(i); } else { @@ -5206,7 +5288,7 @@ erts_schedule_bif(Process *proc, static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I) { - NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsNativeFunc *nep = ERTS_I_BEAM_OP_TO_NFUNC(I); ErtsBifFunc bif = (ErtsBifFunc) nep->func; BIF_RETTYPE ret; @@ -5219,12 +5301,12 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I) ret = (*bif)(c_p, reg, I); if (is_value(ret)) - erts_nif_export_restore(c_p, nep, ret); + erts_nfunc_restore(c_p, nep, ret); else if (c_p->freason != TRAP) - c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ + c_p->freason |= EXF_RESTORE_NFUNC; /* restore in handle_error() */ else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { /* BIF did an ordinary trap... */ - erts_nif_export_restore(c_p, nep, ret); + erts_nfunc_restore(c_p, nep, ret); } /* else: * BIF rescheduled itself using erts_schedule_bif(). @@ -5241,7 +5323,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * int exiting; Process *dirty_shadow_proc; ErtsBifFunc bf; - NifExport *nep; + ErtsNativeFunc *nep; #ifdef DEBUG Eterm *c_p_htop; erts_aint32_t state; @@ -5254,8 +5336,8 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * #endif - nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); - ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + nep = ERTS_I_BEAM_OP_TO_NFUNC(I); + ASSERT(nep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p)); nep->func = ERTS_SCHED_BIF_TRAP_MARKER; @@ -5269,7 +5351,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * dirty_shadow_proc->freason = c_p->freason; dirty_shadow_proc->fvalue = c_p->fvalue; dirty_shadow_proc->ftrace = c_p->ftrace; - dirty_shadow_proc->cp = c_p->cp; dirty_shadow_proc->i = c_p->i; #ifdef DEBUG @@ -5316,7 +5397,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * c_p->freason = dirty_shadow_proc->freason; c_p->fvalue = dirty_shadow_proc->fvalue; c_p->ftrace = dirty_shadow_proc->ftrace; - c_p->cp = dirty_shadow_proc->cp; c_p->i = dirty_shadow_proc->i; c_p->arity = dirty_shadow_proc->arity; } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index a5f46c4264..63c9fc23d5 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -25,13 +25,14 @@ # # <bif-decl> ::= "bif" <bif> <C-name>* | # "ubif" <bif> <C-name>* | -# "gcbif" <bif> <C-name>* +# "hbif" <bif> <C-name>* # <bif> ::= <module> ":" <name> "/" <arity> # -# ubif: Use for operators and guard BIFs that never build anything -# on the heap (such as tuple_size/1) and operators. +# ubif: Use for operators and guard BIFs. # -# gcbif: Use for guard BIFs that may build on the heap (such as abs/1). +# hbif: Use for BIFs that perform garbage collection or need up-to-date +# information on where they were called from. These must be called +# through the export entry. # # bif: Use for all other BIFs. # @@ -60,7 +61,7 @@ bif erlang:display_string/1 bif erlang:display_nl/0 ubif erlang:element/2 bif erlang:erase/0 -bif erlang:erase/1 +hbif erlang:erase/1 bif erlang:exit/1 bif erlang:exit/2 bif erlang:exit_signal/2 @@ -70,7 +71,7 @@ ubif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 -bif erts_internal:garbage_collect/1 +hbif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 @@ -127,10 +128,10 @@ bif erlang:ports/0 bif erlang:pre_loaded/0 bif erlang:process_flag/2 bif erts_internal:process_flag/3 -bif erlang:process_info/1 -bif erlang:process_info/2 +hbif erlang:process_info/1 +hbif erlang:process_info/2 bif erlang:processes/0 -bif erlang:put/2 +hbif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 ubif erlang:round/1 @@ -143,6 +144,8 @@ bif erlang:split_binary/2 bif erlang:statistics/1 bif erlang:term_to_binary/1 bif erlang:term_to_binary/2 +bif erlang:term_to_iovec/1 +bif erlang:term_to_iovec/2 bif erlang:throw/1 bif erlang:time/0 ubif erlang:tl/1 @@ -153,7 +156,7 @@ bif erlang:universaltime_to_localtime/1 bif erlang:unlink/1 bif erlang:unregister/1 bif erlang:whereis/1 -bif erlang:spawn_opt/1 +bif erlang:spawn_opt/4 bif erlang:setnode/2 bif erlang:dist_get_stat/1 bif erlang:dist_ctrl_input_handler/2 @@ -174,7 +177,7 @@ bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 bif erts_internal:request_system_task/4 -bif erts_internal:check_process_code/1 +hbif erts_internal:check_process_code/1 bif erts_internal:map_to_tuple_keys/1 bif erts_internal:term_type/1 @@ -193,10 +196,14 @@ bif erts_internal:scheduler_wall_time/1 bif erts_internal:dirty_process_handle_signals/1 -bif erts_internal:create_dist_channel/4 +bif erts_internal:create_dist_channel/3 bif erts_internal:ets_super_user/1 +bif erts_internal:spawn_request/4 +bif erts_internal:dist_spawn_request/4 +bif erlang:spawn_request_abandon/1 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -466,7 +473,7 @@ bif code:is_module_native/1 # New Bifs in R9C. # -bif erlang:hibernate/3 +hbif erlang:hibernate/3 bif error_logger:warning_map/0 # @@ -758,3 +765,9 @@ bif erts_internal:ets_raw_next/2 bif erts_internal:abort_pending_connection/2 + +# +# New in 23 +# + +bif erts_internal:get_creation/0 diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab index 8e0caa38a3..9c5f9e0c51 100644 --- a/erts/emulator/beam/bif_instrs.tab +++ b/erts/emulator/beam/bif_instrs.tab @@ -212,26 +212,32 @@ i_length.execute(Fail, Live, Dst) { // Call a BIF, store the result in x(0) and transfer control to the // next instruction. // -call_bif(Exp) { +call_light_bif(Bif, Exp) { + Export *export; ErtsBifFunc bf; + Eterm result; ErlHeapFragment *live_hf_end; - Export *export = (Export*) $Exp; + + bf = (ErtsBifFunc) $Bif; + export = (Export*) $Exp; if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* * If we have run out of reductions, do a context * switch before calling the BIF. */ - c_p->arity = GET_BIF_ARITY(export); + c_p->arity = GET_EXPORT_ARITY(export); c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), - GET_BIF_ADDRESS(export)); + if (ERTS_UNLIKELY(export->is_bif_traced)) { + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT(export); + } - bf = GET_BIF_ADDRESS(export); + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_EXPORT_MODULE(export), bf); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); @@ -243,21 +249,26 @@ call_bif(Exp) { ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + + /* Only heavy BIFs may GC. */ + ASSERT(E == c_p->stop); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); + Uint arity = GET_EXPORT_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } - PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); /* @@ -280,10 +291,9 @@ call_bif(Exp) { * erlang code or by nif_bif.epilogue() when the BIF * is done). */ - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); SET_I(c_p->i); - SWAPIN; - Dispatch(); + $DISPATCH(); } /* @@ -297,29 +307,38 @@ call_bif(Exp) { // // Call a BIF tail-recursively, storing the result in x(0) and doing -// a return to the continuation poiner (c_p->cp). +// a return to the continuation poiner. // - -call_bif_only(Exp) { +call_light_bif_only(Bif, Exp) { + ErlHeapFragment *live_hf_end; ErtsBifFunc bf; + Export *export; Eterm result; - ErlHeapFragment *live_hf_end; - Export *export = (Export*) $Exp; + + bf = (ErtsBifFunc) $Bif; + export = (Export*) $Exp; if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* * If we have run out of reductions, do a context * switch before calling the BIF. */ - c_p->arity = GET_BIF_ARITY(export); + c_p->arity = GET_EXPORT_ARITY(export); c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), - GET_BIF_ADDRESS(export)); + if (ERTS_UNLIKELY(export->is_bif_traced)) { + /* Set up a dummy stack frame so we can perform a normal call. Loader + * transformations ensure that the next instruction after this is + * 'deallocate_return 0'. */ + $AH(0, 0, GET_EXPORT_ARITY(export)); + + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT(export); + } - bf = GET_BIF_ADDRESS(export); + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_EXPORT_MODULE(export), bf); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); @@ -331,21 +350,26 @@ call_bif_only(Exp) { ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + + /* Only heavy BIFs may GC. */ + ASSERT(E == c_p->stop); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); + Uint arity = GET_EXPORT_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } - PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); /* @@ -367,11 +391,10 @@ call_bif_only(Exp) { } else if (c_p->freason == TRAP) { /* * Dispatch to a trap. When the trap is done, a jump - * to the continuation pointer (c_p->cp) will be done. + * to the continuation pointer on the stack will be done. */ SET_I(c_p->i); - SWAPIN; - Dispatch(); + $DISPATCH(); } /* @@ -413,17 +436,26 @@ send() { r(0) = result; CHECK_TERM(r(0)); } else if (c_p->freason == TRAP) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); SET_I(c_p->i); SWAPIN; - Dispatch(); + $DISPATCH(); } else { goto find_func_info; } } +call_nif_early() { + HEAVY_SWAPOUT; + I = erts_call_nif_early(c_p, erts_code_to_codeinfo(I)); + HEAVY_SWAPIN; + ASSERT(VALID_INSTR(*I)); + Goto(*I); + //| -no_next +} + call_nif := nif_bif.call_nif.epilogue; -apply_bif := nif_bif.apply_bif.epilogue; +call_bif := nif_bif.call_bif.epilogue; nif_bif.head() { Eterm nif_bif_result; @@ -433,7 +465,7 @@ nif_bif.head() { ErtsCodeMFA *codemfa; } -nif_bif.call_nif() { +nif_bif.call_nif(Func, NifMod, DirtyFunc) { /* * call_nif is always first instruction in function: * @@ -443,11 +475,14 @@ nif_bif.call_nif() { * I[0]: &&call_nif * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF + * I[3]: Function pointer to dirty NIF. This is not used in this + * instruction, but dirty schedulers look at it. * - * This layout is determined by the NifExport struct + * This layout is determined by the ErtsNativeFunc struct */ + (void)$DirtyFunc; + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); codemfa = erts_code_to_codemfa(I); @@ -465,12 +500,12 @@ nif_bif.call_nif() { ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; + NifF* fp = vbf = (NifF*) $Func; struct enif_environment_t env; ASSERT(c_p->scheduler_data); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); + erts_pre_nif(&env, c_p, (struct erl_module_nif*)$NifMod, NULL); ASSERT((c_p->scheduler_data)->current_nif == NULL); (c_p->scheduler_data)->current_nif = &env; @@ -495,15 +530,15 @@ nif_bif.call_nif() { DTRACE_NIF_RETURN(c_p, codemfa); } -nif_bif.apply_bif() { +nif_bif.call_bif(Func) { /* - * At this point, I points to the code[0] in the export entry for - * the BIF: + * At this point, I points to the code[0] in the native function wrapper + * for the BIF: * * code[-3]: Module * code[-2]: Function * code[-1]: Arity - * code[0]: &&apply_bif + * code[0]: &&call_bif * code[1]: Function pointer to BIF function */ @@ -515,21 +550,19 @@ nif_bif.apply_bif() { codemfa = erts_code_to_codemfa(I); - ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)$Func); /* In case we apply process_info/1,2 or load_nif/1 */ c_p->current = codemfa; $SET_CP_I_ABS(I); /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ + * (check_process_code/2, put/2, etc). */ DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); + vbf = (BifFunction)$Func; PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); @@ -557,6 +590,7 @@ nif_bif.apply_bif() { } nif_bif.epilogue() { + //| -no_next ERTS_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { @@ -570,8 +604,7 @@ nif_bif.epilogue() { if (ERTS_LIKELY(is_value(nif_bif_result))) { r(0) = nif_bif_result; CHECK_TERM(r(0)); - SET_I(c_p->cp); - c_p->cp = 0; + $RETURN(); Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); @@ -579,8 +612,42 @@ nif_bif.epilogue() { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } - Dispatch(); + $DISPATCH(); + } + { + BeamInstr *cp = erts_printable_return_address(c_p, E); + ASSERT(VALID_INSTR(*cp)); + I = handle_error(c_p, cp, reg, c_p->current); } - I = handle_error(c_p, c_p->cp, reg, c_p->current); goto post_error_handling; } + +i_load_nif() { + //| -no_next + if (erts_try_seize_code_write_permission(c_p)) { + Eterm result; + + PRE_BIF_SWAPOUT(c_p); + result = erts_load_nif(c_p, I, r(0), r(1)); + erts_release_code_write_permission(); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + SWAPIN; + + if (ERTS_LIKELY(is_value(result))) { + r(0) = result; + $NEXT0(); + } else { + static ErtsCodeMFA mfa = {am_erlang, am_load_nif, 2}; + c_p->freason = BADARG; + I = handle_error(c_p, I, reg, &mfa); + goto post_error_handling; + } + } else { + /* Yield and try again */ + $SET_CP_I_ABS(I); + SWAPOUT; + c_p->current = NULL; + c_p->arity = 2; + goto do_schedule; + } +} diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 522f50287a..7666f23a4f 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2176,6 +2176,24 @@ term_to_Uint64(Eterm term, Uint64 *up) #endif } +int +term_to_Uint32(Eterm term, Uint32 *up) +{ +#if ERTS_SIZEOF_ETERM == 4 + return term_to_Uint(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i >= 0) { + *up = (Uint32) i; + return 1; + } + } + *up = BADARG; + return 0; +#endif +} + int term_to_Sint(Eterm term, Sint *sp) { diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index ad19cce395..3fed076419 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -168,6 +168,8 @@ Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *); int term_to_Uint64(Eterm, Uint64*); int term_to_Sint64(Eterm, Sint64*); #endif +int term_to_Uint32(Eterm, Uint32*); + Uint32 big_to_uint32(Eterm b); int term_equals_2pow32(Eterm); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 4ddf59092a..0ae4bc1e60 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -567,7 +567,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { if (reds_left <= L2B_B2L_RESCHED_REDS) { /* Yield and do it with full context reds... */ - ERTS_BIF_YIELD1(bif_export[BIF_binary_to_list_1], + ERTS_BIF_YIELD1(&bif_trap_export[BIF_binary_to_list_1], BIF_P, BIF_ARG_1); } /* Allow a bit more reductions... */ @@ -621,7 +621,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { if (reds_left <= L2B_B2L_RESCHED_REDS) { /* Yield and do it with full context reds... */ - ERTS_BIF_YIELD3(bif_export[BIF_binary_to_list_3], + ERTS_BIF_YIELD3(&bif_trap_export[BIF_binary_to_list_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } /* Allow a bit more reductions... */ @@ -668,7 +668,7 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { if (reds_left <= L2B_B2L_RESCHED_REDS) { /* Yield and do it with full context reds... */ - ERTS_BIF_YIELD1(bif_export[BIF_bitstring_to_list_1], + ERTS_BIF_YIELD1(&bif_trap_export[BIF_bitstring_to_list_1], BIF_P, BIF_ARG_1); } /* Allow a bit more reductions... */ @@ -1041,7 +1041,7 @@ HIPE_WRAPPER_BIF_DISABLE_GC(list_to_binary, 1) BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_list_to_binary_1]); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, &bif_trap_export[BIF_list_to_binary_1]); } HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) @@ -1054,7 +1054,7 @@ BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) } BIF_ERROR(BIF_P, BADARG); } - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, &bif_trap_export[BIF_iolist_to_binary_1]); } static int bitstr_list_len(ErtsIOListState *); @@ -1081,7 +1081,7 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) else { ErtsL2BState state = ERTS_L2B_STATE_INITER(BIF_P, BIF_ARG_1, - bif_export[BIF_list_to_bitstring_1], + &bif_trap_export[BIF_list_to_bitstring_1], bitstr_list_len, list_to_bitstr_buf_yielding); int orig_reds_left = ERTS_BIF_REDS_LEFT(BIF_P); diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index 50352b4084..eab9d51b5e 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -37,9 +37,12 @@ erts_atomic32_t the_active_code_index; erts_atomic32_t the_staging_code_index; +static int code_writing_seized = 0; static Process* code_writing_process = NULL; struct code_write_queue_item { Process *p; + void (*aux_func)(void *); + void *aux_arg; struct code_write_queue_item* next; }; static struct code_write_queue_item* code_write_queue = NULL; @@ -108,19 +111,37 @@ void erts_abort_staging_code_ix(void) CIX_TRACE("abort"); } +static int try_seize_cwp(Process* c_p, + void (*aux_func)(void *), + void *aux_arg); /* * Calller _must_ yield if we return 0 */ int erts_try_seize_code_write_permission(Process* c_p) { + ASSERT(c_p != NULL); + return try_seize_cwp(c_p, NULL, NULL); +} + +int erts_try_seize_code_write_permission_aux(void (*aux_func)(void *), + void *aux_arg) +{ + ASSERT(aux_func != NULL); + return try_seize_cwp(NULL, aux_func, aux_arg); +} + +static int try_seize_cwp(Process* c_p, + void (*aux_func)(void *), + void *aux_arg) +{ int success; ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */ - ASSERT(c_p != NULL); erts_mtx_lock(&code_write_permission_mtx); - success = (code_writing_process == NULL); + success = !code_writing_seized; if (success) { + code_writing_seized = 1; code_writing_process = c_p; #ifdef ERTS_ENABLE_LOCK_CHECK erts_tsd_set(has_code_write_permission, (void *) 1); @@ -128,13 +149,22 @@ int erts_try_seize_code_write_permission(Process* c_p) } else { /* Already locked */ struct code_write_queue_item* qitem; - ASSERT(code_writing_process != c_p); - qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); - qitem->p = c_p; - erts_proc_inc_refc(c_p); - qitem->next = code_write_queue; - code_write_queue = qitem; - erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); + if (c_p) { + ASSERT(code_writing_process != c_p); + qitem->p = c_p; + qitem->aux_func = NULL; + qitem->aux_arg = NULL; + erts_proc_inc_refc(c_p); + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + } + else { + qitem->p = NULL; + qitem->aux_func = aux_func; + qitem->aux_arg = aux_arg; + } + qitem->next = code_write_queue; + code_write_queue = qitem; } erts_mtx_unlock(&code_write_permission_mtx); return success; @@ -146,15 +176,25 @@ void erts_release_code_write_permission(void) ERTS_LC_ASSERT(erts_has_code_write_permission()); while (code_write_queue != NULL) { /* unleash the entire herd */ struct code_write_queue_item* qitem = code_write_queue; - erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); - if (!ERTS_PROC_IS_EXITING(qitem->p)) { - erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); - } - erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); - code_write_queue = qitem->next; - erts_proc_dec_refc(qitem->p); + if (qitem->p) { + erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(qitem->p)) { + erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + erts_proc_dec_refc(qitem->p); + } + else { /* aux work*/ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); + erts_schedule_misc_aux_work((int) esdp->no, + qitem->aux_func, + qitem->aux_arg); + } + code_write_queue = qitem->next; erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); } + code_writing_seized = 0; code_writing_process = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK erts_tsd_set(has_code_write_permission, (void *) 0); @@ -165,6 +205,6 @@ void erts_release_code_write_permission(void) #ifdef ERTS_ENABLE_LOCK_CHECK int erts_has_code_write_permission(void) { - return (code_writing_process != NULL) && erts_tsd_get(has_code_write_permission); + return code_writing_seized && erts_tsd_get(has_code_write_permission); } #endif diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 42976d2301..64a9c95979 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -133,6 +133,15 @@ ErtsCodeIndex erts_staging_code_ix(void); */ int erts_try_seize_code_write_permission(struct process* c_p); +/* Try seize exclusive code write permission for aux work. + * System thread progress must not be blocked. + * On success return true. + * On failure return false and aux work func(arg) will be scheduled when + * permission is released. . + */ +int erts_try_seize_code_write_permission_aux(void (*func)(void *), + void *arg); + /* Release code write permission. * Will resume any suspended waiters. */ diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 1293ad2d83..3091e322bc 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -149,6 +149,12 @@ static char *erts_dop_to_string(enum dop dop) { return "PAYLOAD_EXIT2_TT"; if (dop == DOP_PAYLOAD_MONITOR_P_EXIT) return "PAYLOAD_MONITOR_P_EXIT"; + if (dop == DOP_SPAWN_REQUEST) + return "SPAWN_REQUEST"; + if (dop == DOP_SPAWN_REQUEST_TT) + return "SPAWN_REQUEST_TT"; + if (dop == DOP_SPAWN_REPLY) + return "SPAWN_REPLY"; ASSERT(0); return "UNKNOWN"; } @@ -168,6 +174,9 @@ static char *erts_dop_to_string(enum dop dop) { int erts_is_alive; /* System must be blocked on change */ int erts_dist_buf_busy_limit; +int erts_dflags_test_remove_hopefull_flags; + +Export spawn_request_yield_export; /* distribution trap functions */ Export* dmonitor_node_trap = NULL; @@ -262,6 +271,12 @@ static int monitor_connection_down(ErtsMonitor *mon, void *unused, Sint reds) return ERTS_MON_LNK_FIRE_REDS; } +static int dist_pend_spawn_exit_connection_down(ErtsMonitor *mon, void *unused, Sint reds) +{ + erts_monitor_release(mon); + return 1; +} + static int link_connection_down(ErtsLink *lnk, void *vdist, Sint reds) { erts_proc_sig_send_link_exit(NULL, THE_NON_VALUE, lnk, @@ -273,12 +288,14 @@ typedef enum { ERTS_CML_CLEANUP_STATE_LINKS, ERTS_CML_CLEANUP_STATE_MONITORS, ERTS_CML_CLEANUP_STATE_ONAME_MONITORS, + ERTS_CML_CLEANUP_STATE_PEND_SPAWN_EXIT_MONITORS, ERTS_CML_CLEANUP_STATE_SEQUENCES, ERTS_CML_CLEANUP_STATE_NODE_MONITORS } ErtsConMonLnkSeqCleanupState; typedef struct { ErtsConMonLnkSeqCleanupState state; + DistEntry* dep; ErtsMonLnkDist *dist; DistSeqNode *seq; void *yield_state; @@ -327,6 +344,16 @@ con_monitor_link_seq_cleanup(void *vcmlcp) if (reds <= 0) break; + ASSERT(!cmlcp->yield_state); + cmlcp->state = ERTS_CML_CLEANUP_STATE_PEND_SPAWN_EXIT_MONITORS; + case ERTS_CML_CLEANUP_STATE_PEND_SPAWN_EXIT_MONITORS: + reds = erts_monitor_tree_foreach_delete_yielding(&dist->dist_pend_spawn_exit, + dist_pend_spawn_exit_connection_down, + NULL, &cmlcp->yield_state, + reds); + if (reds <= 0) + break; + cmlcp->dist = NULL; erts_mon_link_dist_dec_refc(dist); @@ -343,11 +370,30 @@ con_monitor_link_seq_cleanup(void *vcmlcp) cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS; case ERTS_CML_CLEANUP_STATE_NODE_MONITORS: if (cmlcp->trigger_node_monitors) { + Process* waiter; send_nodes_mon_msgs(NULL, am_nodedown, cmlcp->nodename, cmlcp->visability, cmlcp->reason); + erts_de_rwlock(cmlcp->dep); + ASSERT(cmlcp->dep->state == ERTS_DE_STATE_IDLE || + cmlcp->dep->state == ERTS_DE_STATE_PENDING); + ASSERT(cmlcp->dep->pending_nodedown); + waiter = cmlcp->dep->suspended_nodeup; + cmlcp->dep->suspended_nodeup = NULL; + cmlcp->dep->pending_nodedown = 0; + erts_de_rwunlock(cmlcp->dep); + erts_deref_dist_entry(cmlcp->dep); + + if (waiter) { + erts_proc_lock(waiter, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(waiter)) { + erts_resume(waiter, ERTS_PROC_LOCK_STATUS); + } + erts_proc_unlock(waiter, ERTS_PROC_LOCK_STATUS); + erts_proc_dec_refc(waiter); + } } erts_cleanup_offheap(&cmlcp->oh); erts_free(ERTS_ALC_T_CML_CLEANUP, vcmlcp); @@ -364,7 +410,8 @@ con_monitor_link_seq_cleanup(void *vcmlcp) } static void -schedule_con_monitor_link_seq_cleanup(ErtsMonLnkDist *dist, +schedule_con_monitor_link_seq_cleanup(DistEntry* dep, + ErtsMonLnkDist *dist, DistSeqNode *seq, Eterm nodename, Eterm visability, @@ -404,7 +451,16 @@ schedule_con_monitor_link_seq_cleanup(ErtsMonLnkDist *dist, cmlcp->seq = seq; - cmlcp->trigger_node_monitors = is_value(nodename); + if (is_value(nodename)) { + ASSERT(dep); + cmlcp->trigger_node_monitors = 1; + cmlcp->dep = dep; + erts_ref_dist_entry(dep); + } + else { + cmlcp->trigger_node_monitors = 0; + cmlcp->dep = NULL; + } cmlcp->nodename = nodename; cmlcp->visability = visability; if (rsz == 0) @@ -422,6 +478,255 @@ schedule_con_monitor_link_seq_cleanup(ErtsMonLnkDist *dist, } } +static void +dist_pend_spawn_exit_save_child_result(Eterm result, Eterm ref, ErtsMonLnkDist *dist) +{ + ErtsMonitorData *new_mdp = NULL; + Process *proc = NULL; + int done = 0; + + while (1) { + erts_mtx_lock(&dist->mtx); + + if (!dist->alive) + done = !0; + else { + ErtsMonitor *mon; + mon = erts_monitor_tree_lookup(dist->dist_pend_spawn_exit, ref); + if (!mon) { + if (new_mdp) { + /* + * Save info so parent can get child pid when handling + * links during termination + */ + erts_monitor_tree_insert(&dist->dist_pend_spawn_exit, + &new_mdp->target); + done = !0; + new_mdp = NULL; + } + } + else { + ErtsMonitorDataExtended *mdep; + /* + * The terminating parent is waiting for this signal. + * Store childs pid and resume parent if it has + * suspended (i.e, mon->other.ptr != NULL)... + */ + proc = mon->other.ptr; + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + if (is_atom(result)) + mdep->md.origin.other.item = result; + else { + Eterm *hp; + ErlOffHeap oh; +#ifdef DEBUG + int i; + for (i = 0; i < EXTERNAL_THING_HEAD_SIZE + 1; i++) { + ASSERT(is_non_value(mdep->heap[i])); + } +#endif + hp = &(mdep)->heap[0]; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = mdep->uptr.ohhp; + mdep->md.origin.other.item + = copy_struct(result, + EXTERNAL_THING_HEAD_SIZE + 1, + &hp, &oh); + mdep->uptr.ohhp = oh.first; + } + done = !0; + } + } + + erts_mtx_unlock(&dist->mtx); + + if (done) + break; + + /* + * No monitor previously saved by parent; create one + * and store child pid... + */ + new_mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, ref, + am_undefined, result, NIL); + ASSERT(new_mdp->target.other.item == am_undefined); + new_mdp->target.other.ptr = NULL; + + ((ErtsMonitorDataExtended *) new_mdp)->dist = dist; + erts_mon_link_dist_inc_refc(dist); + erts_monitor_release(&new_mdp->origin); + } + + if (proc) + erts_resume(proc, 0); + + if (new_mdp) + erts_monitor_release(&new_mdp->target); + +} + +int +erts_dist_pend_spawn_exit_delete(ErtsMonitor *mon) +{ + ErtsMonitorDataExtended *mdep; + int delete; + ErtsMonLnkDist *dist; + Uint16 flags; + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + dist = mdep->dist; + + erts_mtx_lock(&dist->mtx); + + flags = mon->flags; + delete = !!dist->alive & !!(flags & ERTS_ML_FLG_IN_TABLE); + + if (delete) + erts_monitor_tree_delete(&dist->dist_pend_spawn_exit, mon); + + erts_mtx_unlock(&dist->mtx); + + return delete; +} + +int +erts_dist_pend_spawn_exit_parent_setup(ErtsMonitor *mon) +{ + /* + * Return: + * 0 -> connection is closing + * !0 -> moved target part of monitor to dist_pend_spawn_exit + */ + ErtsMonitorData *mdp; + ErtsMonLnkDist *dist; + int res; + + ASSERT(erts_monitor_is_origin(mon)); + + mdp = (ErtsMonitorData *) erts_monitor_to_data(mon); + + if (!erts_monitor_dist_delete(&mdp->target)) + return 0; + + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + + while (1) { + ErtsMonitor *tmp_mon; + + erts_mtx_lock(&dist->mtx); + mdp->target.other.ptr = NULL; + + if (!dist->alive) { + res = 0; + tmp_mon = NULL; + } + else { + res = !0; + tmp_mon = erts_monitor_tree_lookup(dist->dist_pend_spawn_exit, + mdp->ref); + if (!tmp_mon) + erts_monitor_tree_insert(&dist->dist_pend_spawn_exit, + &mdp->target); + else + erts_monitor_tree_delete(&dist->dist_pend_spawn_exit, tmp_mon); + } + + erts_mtx_unlock(&dist->mtx); + + if (!tmp_mon) { + if (!res) + erts_monitor_release(&mdp->target); + return res; + } + else { + /* + * Child had responded; copy its pid then store + * original target end in 'dist_pend_spawn_exit' + */ + ErtsMonitorData *tmp_mdp = erts_monitor_to_data(tmp_mon); + + if (is_atom(tmp_mdp->origin.other.item)) { + erts_monitor_release(tmp_mon); + erts_monitor_release(&mdp->target); + return 0; /* Spawn failed; drop it... */ + } + + /* + * If mdp->origin.other.item != am_pending, the other + * end is behaving badly by sending multiple + * responses... + */ + if (mdp->origin.other.item == am_pending) { + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + ErlOffHeap oh; + Eterm *hp; +#ifdef DEBUG + int i; + + ASSERT(is_external_pid(tmp_mdp->origin.other.item)); + for (i = 0; i < EXTERNAL_THING_HEAD_SIZE + 1; i++) { + ASSERT(is_non_value(mdep->heap[i])); + } +#endif + hp = &(mdep)->heap[0]; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = mdep->uptr.ohhp; + mdep->md.origin.other.item + = copy_struct(tmp_mdp->origin.other.item, + EXTERNAL_THING_HEAD_SIZE + 1, + &hp, &oh); + mdep->uptr.ohhp = oh.first; + } + erts_monitor_release(tmp_mon); + } + } +} + +int +erts_dist_pend_spawn_exit_parent_wait(Process *c_p, + ErtsProcLocks locks, + ErtsMonitor *mon) +{ + /* + * return value of + * > 0 --> Child pid can now be found in monitor + * 0 --> Connection not alive; drop it + * < 0 --> Setup completed; later need to wait for child pid + */ + ErtsMonitorData *mdp; + ErtsMonLnkDist *dist; + int res; + + ASSERT(erts_monitor_is_origin(mon)); + + mdp = (ErtsMonitorData *) erts_monitor_to_data(mon); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + + erts_mtx_lock(&dist->mtx); + + if (!dist->alive) { + res = 0; + } + else { + ASSERT(&mdp->target == + erts_monitor_tree_lookup(dist->dist_pend_spawn_exit, mdp->ref)); + if (mdp->origin.other.item != am_pending) { + erts_monitor_tree_delete(&dist->dist_pend_spawn_exit, &mdp->target); + res = 1; + } + else { + /* We need to suspend wait and wait for the result... */ + mdp->target.other.ptr = (void *) c_p; + erts_suspend(c_p, locks, NULL); + res = -1; + } + } + + erts_mtx_unlock(&dist->mtx); + + return res; +} + /* ** A full node name consists of a "n@h" ** @@ -644,7 +949,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ErtsAtomCache *cache; ErtsProcList *suspendees; ErtsDistOutputBuf *obuf; - Uint32 flags; + Uint64 flags; erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1); erts_de_rwlock(dep); @@ -674,7 +979,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) dep->sequences = NULL; nodename = dep->sysname; - flags = dep->flags; + flags = dep->dflags; erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) NIL); cache = dep->cache; @@ -693,10 +998,11 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) dep->send = NULL; erts_set_dist_entry_not_connected(dep); - + dep->pending_nodedown = 1; erts_de_rwunlock(dep); - schedule_con_monitor_link_seq_cleanup(mld, + schedule_con_monitor_link_seq_cleanup(dep, + mld, sequences, nodename, (flags & DFLAG_PUBLISHED @@ -711,8 +1017,6 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) delete_cache(cache); free_de_out_queues(dep, obuf); - if (dep->transcode_ctx) - transcode_free_ctx(dep); } dec_no_nodes(); @@ -734,6 +1038,8 @@ trap_function(Eterm func, int arity) */ static Eterm erts_dflags_record; +static BIF_RETTYPE spawn_request_yield_3(BIF_ALIST_3); + void init_dist(void) { init_nodes_monitors(); @@ -759,61 +1065,114 @@ void init_dist(void) dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, am_dist_ctrl_put_data, 2); + erts_init_trap_export(&spawn_request_yield_export, + am_erts_internal, am_spawn_request_yield, + 3, spawn_request_yield_3); { - Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+6)*sizeof(Eterm)); - erts_dflags_record = TUPLE6(hp, am_erts_dflags, - make_small(DFLAG_DIST_DEFAULT), - make_small(DFLAG_DIST_MANDATORY), - make_small(DFLAG_DIST_ADDABLE), - make_small(DFLAG_DIST_REJECTABLE), - make_small(DFLAG_DIST_STRICT_ORDER)); - erts_set_literal_tag(&erts_dflags_record, hp, (1+6)); + Eterm *hp_start, *hp, **hpp = NULL; + Uint sz = 0, *szp = &sz; + while (1) { + erts_dflags_record = + erts_bld_tuple(hpp, szp, 6, + am_erts_dflags, + erts_bld_uint64(hpp, szp, DFLAG_DIST_DEFAULT), + erts_bld_uint64(hpp, szp, DFLAG_DIST_MANDATORY), + erts_bld_uint64(hpp, szp, DFLAG_DIST_ADDABLE), + erts_bld_uint64(hpp, szp, DFLAG_DIST_REJECTABLE), + erts_bld_uint64(hpp, szp, DFLAG_DIST_STRICT_ORDER)); + if (hpp) { + ASSERT(is_value(erts_dflags_record)); + ASSERT(hp == hp_start + sz); + erts_set_literal_tag(&erts_dflags_record, hp_start, sz); + break; + } + hp = hp_start = erts_alloc(ERTS_ALC_T_LITERAL, sz*sizeof(Eterm)); + hpp = &hp; + szp = NULL; + } } } -#define ErtsDistOutputBuf2Binary(OB) OB->bin - static ERTS_INLINE ErtsDistOutputBuf * -alloc_dist_obuf(Uint size, Uint headers) +alloc_dist_obufs(byte **extp, TTBEncodeContext *ctx, + Uint data_size, Uint fragments, Uint vlen) { - Uint obuf_size = sizeof(ErtsDistOutputBuf)*(headers); + int ix; ErtsDistOutputBuf *obuf; + char *ptr; + Uint iov_sz, obsz; Binary *bin; - byte *extp; - int i; + ErlIOVec **feiov; + Uint fragment_size; - bin = erts_bin_drv_alloc(obuf_size + size); - erts_refc_add(&bin->intern.refc, headers - 1, 1); + obsz = sizeof(ErtsDistOutputBuf)*fragments; + if (obsz % sizeof(void *) != 0) + obsz += sizeof(void *) - (obsz % sizeof(void *)); - obuf = (ErtsDistOutputBuf *)&bin->orig_bytes[0]; - extp = (byte *)&bin->orig_bytes[obuf_size]; + iov_sz = erts_ttb_iov_size(0, vlen, fragments); + + bin = erts_bin_drv_alloc(obsz + iov_sz + data_size); + ctx->result_bin = bin; + ptr = (char *) &bin->orig_bytes[0]; + + obuf = (ErtsDistOutputBuf *) ptr; + ptr += obsz; + + if (fragments > 1) + fragment_size = ERTS_DIST_FRAGMENT_SIZE; + else + fragment_size = ~((Uint) 0); + + feiov = erts_ttb_iov_init(ctx, 0, ptr, vlen, + fragments, fragment_size); + ptr += iov_sz; - for (i = 0; i < headers; i++) { - obuf[i].bin = bin; - obuf[i].extp = extp; + erts_refc_add(&bin->intern.refc, fragments - 1, 1); + + for (ix = 0; ix < fragments; ix++) { + obuf[ix].bin = bin; + obuf[ix].eiov = feiov[ix]; #ifdef DEBUG - obuf[i].dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; - obuf[i].ext_startp = extp; - obuf[i].alloc_endp = &extp[size]; - ASSERT(bin == ErtsDistOutputBuf2Binary(obuf)); + obuf[ix].dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; #endif } + *extp = (byte *) ptr; return obuf; } static ERTS_INLINE void -free_dist_obuf(ErtsDistOutputBuf *obuf) +free_dist_obuf(ErtsDistOutputBuf *obuf, int free_binv) { - Binary *bin = ErtsDistOutputBuf2Binary(obuf); ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN); - erts_bin_release(bin); + + if (free_binv) { + int i; + int vlen = obuf->eiov->vsize; + ErlDrvBinary **binv = obuf->eiov->binv; + for (i = 0; i < vlen; i++) { + if (binv[i]) + driver_free_binary(binv[i]); + } + } + erts_bin_release(obuf->bin); } static ERTS_INLINE Sint size_obuf(ErtsDistOutputBuf *obuf) { - return sizeof(ErtsDistOutputBuf) + (obuf->ext_endp - obuf->ext_start) - + (obuf->hdr_endp - obuf->hdrp); + Sint vlen = obuf->eiov->vsize; + Sint sz; +#ifdef DEBUG + Sint i; + for (i = 0, sz = 0; i < vlen; i++) + sz += obuf->eiov->iov[i].iov_len; + ASSERT(sz == obuf->eiov->size); +#endif + sz = sizeof(ErtsDistOutputBuf) + sizeof(ErlIOVec); + sz += obuf->eiov->size; + sz += sizeof(SysIOVec)*vlen; + sz += sizeof(ErlDrvBinary*)*vlen; + return sz; } static ErtsDistOutputBuf* clear_de_out_queues(DistEntry* dep) @@ -853,7 +1212,7 @@ static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf) fobuf = obuf; obuf = obuf->next; obufsize += size_obuf(fobuf); - free_dist_obuf(fobuf); + free_dist_obuf(fobuf, !0); } if (obufsize) { @@ -880,7 +1239,7 @@ int erts_dsend_context_dtor(Binary* ctx_bin) if (ctx->phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->obuf) { int i; for (i = 0; i < ctx->fragments; i++) - free_dist_obuf(&ctx->obuf[i]); + free_dist_obuf(&ctx->obuf[i], !0); } if (ctx->deref_dep) erts_deref_dist_entry(ctx->dep); @@ -947,14 +1306,14 @@ erts_dsig_send_m_exit(ErtsDSigSendContext *ctx, Eterm watcher, Eterm watched, { Eterm ctl, msg; - if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor) */ return ERTS_DSIG_SEND_OK; } - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) { ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_MONITOR_P_EXIT), watched, watcher, ref); msg = reason; @@ -976,7 +1335,7 @@ erts_dsig_send_monitor(ErtsDSigSendContext *ctx, Eterm watcher, Eterm watched, { Eterm ctl; - if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_MONITOR_P. * Just avoid sending it and by doing that reduce this monitor @@ -1002,7 +1361,7 @@ erts_dsig_send_demonitor(ErtsDSigSendContext *ctx, Eterm watcher, { Eterm ctl; - if (~ctx->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + if (~ctx->dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { /* * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor) */ @@ -1019,7 +1378,7 @@ erts_dsig_send_demonitor(ErtsDSigSendContext *ctx, Eterm watcher, static int can_send_seqtrace_token(ErtsDSigSendContext* ctx, Eterm token) { Eterm label; - if (ctx->flags & DFLAG_BIG_SEQTRACE_LABELS) { + if (ctx->dflags & DFLAG_BIG_SEQTRACE_LABELS) { /* The other end is capable of handling arbitrary seq_trace labels. */ return 1; } @@ -1052,7 +1411,7 @@ erts_dsig_send_msg(ErtsDSigSendContext* ctx, Eterm remote, Eterm message) #endif if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); } @@ -1080,7 +1439,7 @@ erts_dsig_send_msg(ErtsDSigSendContext* ctx, Eterm remote, Eterm message) send_token = (token != NIL && can_send_seqtrace_token(ctx, token)); - if (ctx->flags & DFLAG_SEND_SENDER) { + if (ctx->dflags & DFLAG_SEND_SENDER) { dist_op = make_small(send_token ? DOP_SEND_SENDER_TT : DOP_SEND_SENDER); @@ -1127,7 +1486,7 @@ erts_dsig_send_reg_msg(ErtsDSigSendContext* ctx, Eterm remote_name, #endif if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, full_to, sender); } @@ -1182,20 +1541,20 @@ erts_dsig_send_exit_tt(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, DTRACE_CHARBUF(reason_str, 128); #endif - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) msg = reason; if (have_seqtrace(token)) { - seq_trace_update_send(ctx->c_p); + seq_trace_update_serial(ctx->c_p); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) { ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT_TT), local, remote, token); } else ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_EXIT_TT), local, remote, token, reason); } else { - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT), local, remote); else ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); @@ -1226,9 +1585,9 @@ erts_dsig_send_exit_tt(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, int erts_dsig_send_exit(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Eterm reason) { - Eterm ctl, msg = ctx->dep->flags & DFLAG_EXIT_PAYLOAD ? reason : THE_NON_VALUE; + Eterm ctl, msg = ctx->dep->dflags & DFLAG_EXIT_PAYLOAD ? reason : THE_NON_VALUE; - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) { ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT), local, remote); msg = reason; } else { @@ -1243,7 +1602,7 @@ erts_dsig_send_exit2(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Eterm { Eterm ctl, msg; - if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { + if (ctx->dep->dflags & DFLAG_EXIT_PAYLOAD) { ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_PAYLOAD_EXIT2), local, remote); msg = reason; @@ -1268,6 +1627,65 @@ erts_dsig_send_group_leader(ErtsDSigSendContext *ctx, Eterm leader, Eterm remote return dsig_send_ctl(ctx, ctl); } +static int +dsig_send_spawn_request(ErtsDSigSendContext *ctx, Eterm ref, Eterm from, + Eterm gl, Eterm mfa, Eterm alist, Eterm opts) +{ + Process *sender = ctx->c_p; + if (!have_seqtrace(SEQ_TRACE_TOKEN(sender))) { + ctx->ctl = TUPLE6(&ctx->ctl_heap[0], make_small(DOP_SPAWN_REQUEST), + ref, from, gl, mfa, opts); + } + else { + Eterm tmp_heap[8]; + Eterm node = ctx->dep ? ctx->dep->sysname : ctx->node; + Eterm msg; + Eterm token; + /* + * Present this as two messages for the sequence tracing. + * All data except the argument list in the first message + * and then the argument list as second message (which + * willl become an actual message). For more info see + * handling of seq-trace token in erl_create_process(). + */ + + seq_trace_update_serial(sender); + token = SEQ_TRACE_TOKEN(sender); + msg = TUPLE6(&tmp_heap[0], am_spawn_request, + ref, from, gl, mfa, opts); + seq_trace_output(token, msg, SEQ_TRACE_SEND, node, sender); + + seq_trace_update_serial(sender); + token = SEQ_TRACE_TOKEN(sender); + seq_trace_output(token, alist, SEQ_TRACE_SEND, node, sender); + + ctx->ctl = TUPLE7(&ctx->ctl_heap[0], make_small(DOP_SPAWN_REQUEST_TT), + ref, from, gl, mfa, opts, token); + } + ctx->msg = alist; + return erts_dsig_send(ctx); +} + +int +erts_dsig_send_spawn_reply(ErtsDSigSendContext *ctx, + Eterm ref, + Eterm to, + Eterm flags, + Eterm result, + Eterm token) +{ + if (!have_seqtrace(token)) { + ctx->ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_SPAWN_REPLY), + ref, to, flags, result); + } + else { + ctx->ctl = TUPLE6(&ctx->ctl_heap[0], make_small(DOP_SPAWN_REPLY_TT), + ref, to, flags, result, token); + } + ctx->msg = THE_NON_VALUE; + return erts_dsig_send(ctx); +} + #define ERTS_RBT_PREFIX dist_seq #define ERTS_RBT_T DistSeqNode #define ERTS_RBT_KEY_T Uint @@ -1832,7 +2250,7 @@ int erts_net_message(Port *prt, * the atom '' (empty cookie). */ ASSERT((type == DOP_SEND_SENDER || type == DOP_SEND_SENDER_TT) - ? (is_pid(tuple[2]) && (dep->flags & DFLAG_SEND_SENDER)) + ? (is_pid(tuple[2]) && (dep->dflags & DFLAG_SEND_SENDER)) : tuple[2] == am_Empty); #ifdef ERTS_DIST_MSG_DBG @@ -2054,6 +2472,247 @@ int erts_net_message(Port *prt, (void) erts_proc_sig_send_group_leader(NULL, to, from, NIL); break; + case DOP_SPAWN_REQUEST_TT: { + Eterm tmp_heap[2]; + ErlSpawnOpts so; + int code, opts_error; + Eterm pid, error, ref, from, gl, mfa, opts, token, args; + /* {DOP_SPAWN_REQUEST_TT, Ref, From, GL, MFA, Opts, Token} */ + + if (tuple_arity != 7) + goto invalid_message; + + token = tuple[7]; + + if (0) { + + case DOP_SPAWN_REQUEST: + /* {DOP_SPAWN_REQUEST, Ref, From, GL, MFA, Opts} */ + if (tuple_arity != 6) + goto invalid_message; + + token = NIL; + } + + ref = tuple[2]; + from = tuple[3]; + gl = tuple[4]; + mfa = tuple[5]; + opts = tuple[6]; + + if (is_not_external_ref(ref)) + goto invalid_message; + if (is_not_external_pid(from)) + goto invalid_message; + if (is_not_pid(gl)) + goto invalid_message; + if (is_not_tuple_arity(mfa, 3)) + goto invalid_message; + else { + Eterm *tp = tuple_val(tuple[5]); + if (is_not_atom(tp[1])) + goto invalid_message; + if (is_not_atom(tp[2])) + goto invalid_message; + if (is_not_small(tp[3])) + goto invalid_message; + } + + opts_error = erts_parse_spawn_opts(&so, opts, NULL, 0); + if (opts_error) { + ErtsDSigSendContext ctx; + if (opts_error > 1) + goto invalid_message; + error = am_badopt; + dist_spawn_error: + if (ede_hfrag) { + erts_free_dist_ext_copy(erts_get_dist_ext(ede_hfrag)); + free_message_buffer(ede_hfrag); + } + if (have_seqtrace(token)) { + /* + * See erl_create_process() for why we do this + * serial trickery... + */ + Eterm tmp_heap[7]; + Eterm seq_msg; + Eterm serial; + Uint serial_num; + /* Receiver spawn_request... */ + serial = SEQ_TRACE_T_SERIAL(token); + serial_num = unsigned_val(serial); + serial_num--; + serial = make_small(serial_num); + SEQ_TRACE_T_SERIAL(token) = serial; + seq_msg = TUPLE6(&tmp_heap[0], am_spawn_request, + ref, from, gl, mfa, opts); + seq_trace_output(token, seq_msg, SEQ_TRACE_RECEIVE, + erts_this_dist_entry->sysname, NULL); + SEQ_TRACE_T_LASTCNT(token) = serial; + /* Send spawn_reply... */ + erts_seq_trace_update_node_token(token); + seq_msg = TUPLE4(&tmp_heap[0], + am_spawn_reply, ref, am_error, error); + seq_trace_output(token, seq_msg, SEQ_TRACE_SEND, from, NULL); + } + code = erts_dsig_prepare(&ctx, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_spawn_reply(&ctx, + tuple[2], + tuple[3], + make_small(0), + error, + token); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + break; + } + + so.mref = ref; + so.tag = am_spawn_reply; + so.parent_id = from; + so.group_leader = gl; + so.mfa = mfa; + so.dist_entry = dep; + so.edep = edep; + so.ede_hfrag = ede_hfrag; + so.token = token; + so.opts = opts; + + args = CONS(&tmp_heap[0], mfa, NIL); + pid = erl_create_process(NULL, + am_erts_internal, + am_dist_spawn_init, + args, + &so); + if (is_non_value(pid)) { + if (so.error_code == SYSTEM_LIMIT) + error = am_system_limit; + else + goto invalid_message; /* should not happen */ + goto dist_spawn_error; + } + + break; + } + + case DOP_SPAWN_REPLY_TT: { + ErtsLinkData *ldp; + ErtsLink *lnk; + int monitor; + Eterm ref, result, flags_term, parent, token; + Uint flags; + + /* {DOP_SPAWN_REPLY_TT, Ref, To, Flags, From, Token} */ + if (tuple_arity != 6) + goto invalid_message; + + token = tuple[6]; + + if (0) { + case DOP_SPAWN_REPLY: + + /* {DOP_SPAWN_REPLY, Ref, To, Flags, From} */ + if (tuple_arity != 5) + goto invalid_message; + + token = NIL; + } + + ldp = NULL; + lnk = NULL; + monitor = 0; + + ref = tuple[2]; + parent = tuple[3]; + flags_term = tuple[4]; + result = tuple[5]; + + if (is_not_internal_pid(parent)) { + if (is_external_pid(parent)) { + DistEntry *dep = external_pid_dist_entry(parent); + if (dep == erts_this_dist_entry) + break; /* Old incarnation of this node... */ + } + goto invalid_message; + } + + if (is_not_internal_ref(ref)) { + if (is_external_ref(ref)) { + DistEntry *dep = external_ref_dist_entry(ref); + if (dep == erts_this_dist_entry) + break; /* Old incarnation of this node... */ + } + goto invalid_message; + } + + if (is_not_small(flags_term)) + goto invalid_message; + + flags = unsigned_val(flags_term); + if (flags >= (1 << 27)) + goto invalid_message; + + if (is_not_external_pid(result) && is_not_atom(result)) + goto invalid_message; + + if (is_external_pid(result)) { + + monitor = !!(flags & ERTS_DIST_SPAWN_FLAG_MONITOR); + + if (flags & ERTS_DIST_SPAWN_FLAG_LINK) { + /* Successful spawn-link... */ + int code; + + ldp = erts_link_create(ERTS_LNK_TYPE_DIST_PROC, + result, parent); + ASSERT(ldp->a.other.item == parent); + ASSERT(eq(ldp->b.other.item, result)); + code = erts_link_dist_insert(&ldp->a, dep->mld); + ASSERT(code); (void)code; + + lnk = &ldp->b; + } + } + + if (!erts_proc_sig_send_dist_spawn_reply(dep->sysname, ref, + parent, lnk, result, + token)) { + ErtsDSigSendContext ctx; + int code; + + if (monitor) { + code = erts_dsig_prepare(&ctx, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_demonitor(&ctx, parent, + result, ref); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + } + + if (lnk) { + + code = erts_link_dist_delete(&ldp->a); + ASSERT(code); + erts_link_release_both(ldp); + } + + if (flags & ERTS_DIST_SPAWN_FLAG_LINK) { + /* + * Save info so the terminating parent can send us + * an exit signal with the correct exit reason... + */ + dist_pend_spawn_exit_save_child_result(result, + ref, + dep->mld); + } + } + + break; + } + default: goto invalid_message; } @@ -2201,7 +2860,7 @@ retry: ctx->connection_id = dep->connection_id; ctx->no_suspend = no_suspend; ctx->no_trap = no_trap; - ctx->flags = dep->flags; + ctx->dflags = dep->dflags; ctx->return_term = am_true; ctx->phase = ERTS_DSIG_SEND_PHASE_INIT; ctx->from = proc ? proc->common.id : am_undefined; @@ -2255,8 +2914,6 @@ erts_dsig_send(ErtsDSigSendContext *ctx) while (1) { switch (ctx->phase) { case ERTS_DSIG_SEND_PHASE_INIT: - ctx->flags = ctx->flags; - ctx->c_p = ctx->c_p; if (!ctx->c_p) { ctx->no_trap = 1; @@ -2270,13 +2927,11 @@ erts_dsig_send(ErtsDSigSendContext *ctx) if (!erts_is_alive) return ERTS_DSIG_SEND_OK; - if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) { + if (ctx->dflags & DFLAG_DIST_HDR_ATOM_CACHE) { ctx->acmp = erts_get_atom_cache_map(ctx->c_p); - ctx->max_finalize_prepend = 0; } else { ctx->acmp = NULL; - ctx->max_finalize_prepend = 3; } #ifdef ERTS_DIST_MSG_DBG @@ -2287,176 +2942,212 @@ erts_dsig_send(ErtsDSigSendContext *ctx) erts_fprintf(dbg_file, " MSG: %.160T\n", ctx->msg); #endif - ctx->data_size = ctx->max_finalize_prepend; + ctx->data_size = 0; erts_reset_atom_cache_map(ctx->acmp); - switch (erts_encode_dist_ext_size(ctx->ctl, ctx->flags, - ctx->acmp, &ctx->data_size)) { - case ERTS_EXT_SZ_OK: - break; - case ERTS_EXT_SZ_SYSTEM_LIMIT: - retval = ERTS_DSIG_SEND_TOO_LRG; - goto done; - case ERTS_EXT_SZ_YIELD: - ERTS_INTERNAL_ERROR("Unexpected yield result"); - break; + ERTS_INIT_TTBSizeContext(&ctx->u.sc, ctx->dflags); + + while (1) { + ErtsExtSzRes sz_res; + Sint reds = CONTEXT_REDS; + sz_res = erts_encode_dist_ext_size(ctx->ctl, + ctx->acmp, + &ctx->u.sc, + &ctx->data_size, + &reds, + &ctx->vlen, + &ctx->fragments); + ctx->reds -= CONTEXT_REDS - reds; + if (sz_res == ERTS_EXT_SZ_OK) + break; + if (sz_res == ERTS_EXT_SZ_SYSTEM_LIMIT) { + retval = ERTS_DSIG_SEND_TOO_LRG; + goto done; + } } + if (is_non_value(ctx->msg)) { ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; break; } - ctx->u.sc.wstack.wstart = NULL; - ctx->u.sc.flags = ctx->flags; - ctx->u.sc.level = 0; ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; case ERTS_DSIG_SEND_PHASE_MSG_SIZE: { - ErtsExtSzRes sz_res; - sz_res = (!ctx->no_trap - ? erts_encode_dist_ext_size_ctx(ctx->msg, - ctx, - &ctx->data_size) - : erts_encode_dist_ext_size(ctx->msg, - ctx->flags, - ctx->acmp, - &ctx->data_size)); - switch (sz_res) { - case ERTS_EXT_SZ_OK: - break; - case ERTS_EXT_SZ_SYSTEM_LIMIT: - retval = ERTS_DSIG_SEND_TOO_LRG; - goto done; - case ERTS_EXT_SZ_YIELD: - if (ctx->no_trap) - ERTS_INTERNAL_ERROR("Unexpected yield result"); + Sint reds, *redsp; + if (!ctx->no_trap) + redsp = &ctx->reds; + else { + reds = CONTEXT_REDS; + redsp = &reds; + } + while (1) { + ErtsExtSzRes sz_res; + sz_res = erts_encode_dist_ext_size(ctx->msg, + ctx->acmp, + &ctx->u.sc, + &ctx->data_size, + redsp, + &ctx->vlen, + &ctx->fragments); + if (ctx->no_trap) { + ctx->reds -= CONTEXT_REDS - reds; + if (sz_res == ERTS_EXT_SZ_YIELD) { + reds = CONTEXT_REDS; + continue; + } + } + if (sz_res == ERTS_EXT_SZ_OK) + break; + if (sz_res == ERTS_EXT_SZ_SYSTEM_LIMIT) { + retval = ERTS_DSIG_SEND_TOO_LRG; + goto done; + } + ASSERT(sz_res == ERTS_EXT_SZ_YIELD); retval = ERTS_DSIG_SEND_CONTINUE; goto done; } ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; } - case ERTS_DSIG_SEND_PHASE_ALLOC: - erts_finalize_atom_cache_map(ctx->acmp, ctx->flags); - - if (ctx->flags & DFLAG_FRAGMENTS && is_value(ctx->msg) && is_not_immed(ctx->msg)) { - /* Calculate the max number of fragments that are needed */ - ASSERT(is_pid(ctx->from) && - "from has to be a pid because it is used as sequence id"); - ctx->fragments = ctx->data_size / ERTS_DIST_FRAGMENT_SIZE + 1; - } else - ctx->fragments = 1; - - ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp, ctx->fragments); - - ctx->obuf = alloc_dist_obuf( - ctx->dhdr_ext_size + ctx->data_size + - (ctx->fragments-1) * ERTS_DIST_FRAGMENT_HEADER_SIZE, - ctx->fragments); - ctx->obuf->ext_start = &ctx->obuf->extp[0]; - ctx->obuf->ext_endp = &ctx->obuf->extp[0] + ctx->max_finalize_prepend - + ctx->dhdr_ext_size; - + case ERTS_DSIG_SEND_PHASE_ALLOC: { + + erts_finalize_atom_cache_map(ctx->acmp, ctx->dflags); + + ERTS_INIT_TTBEncodeContext(&ctx->u.ec, ctx->dflags); + ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(&ctx->u.ec, + ctx->acmp, + ctx->fragments); + ctx->obuf = alloc_dist_obufs(&ctx->extp, + &ctx->u.ec, + ctx->dhdr_ext_size + ctx->data_size + + ((ctx->fragments - 1) + * ERTS_DIST_FRAGMENT_HEADER_SIZE), + ctx->fragments, + ctx->vlen); + ctx->alloced_fragments = ctx->fragments; /* Encode internal version of dist header */ - ctx->obuf->extp = erts_encode_ext_dist_header_setup( - ctx->obuf->ext_endp, ctx->acmp, ctx->fragments, ctx->from); - /* Encode control message */ - erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL); - - ctx->obuf->hdrp = NULL; - ctx->obuf->hdr_endp = NULL; + ctx->dhdrp = ctx->extp; + ctx->extp += ctx->dhdr_ext_size; + ctx->dhdrp = erts_encode_ext_dist_header_setup(&ctx->u.ec, + ctx->extp, + ctx->acmp, + ctx->fragments, + ctx->from); + ctx->dhdr_ext_size = ctx->extp - ctx->dhdrp; + while (1) { + Sint reds = CONTEXT_REDS; + /* Encode control message */ + int res = erts_encode_dist_ext(ctx->ctl, &ctx->extp, + ctx->dflags, ctx->acmp, + &ctx->u.ec, &ctx->fragments, + &reds); + ctx->reds -= CONTEXT_REDS - reds; + if (res == 0) + break; + } if (is_non_value(ctx->msg)) { - ctx->obuf->msg_start = NULL; ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; break; } - ctx->u.ec.flags = ctx->flags; - ctx->u.ec.hopefull_flags = 0; - ctx->u.ec.level = 0; - ctx->u.ec.wstack.wstart = NULL; - ctx->obuf->msg_start = ctx->obuf->ext_endp; ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; - case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: - if (!ctx->no_trap) { - if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, - ctx->acmp, &ctx->u.ec, &ctx->reds)) { + } + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: { + Sint reds, *redsp; + if (!ctx->no_trap) + redsp = &ctx->reds; + else { + reds = CONTEXT_REDS; + redsp = &reds; + } + while (1) { + int res = erts_encode_dist_ext(ctx->msg, &ctx->extp, + ctx->dflags, ctx->acmp, + &ctx->u.ec, + &ctx->fragments, + redsp); + if (!ctx->no_trap) { + if (res == 0) + break; retval = ERTS_DSIG_SEND_CONTINUE; goto done; } - } else { - erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, - ctx->acmp, NULL, NULL); + else { + ctx->reds -= CONTEXT_REDS - reds; + if (res == 0) + break; + reds = CONTEXT_REDS; + } } ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + } case ERTS_DSIG_SEND_PHASE_FIN: { - - ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); - ASSERT(ctx->obuf->ext_startp <= ctx->obuf->extp - ctx->max_finalize_prepend); - ASSERT(ctx->obuf->ext_endp <= (byte*)ctx->obuf->ext_startp + ctx->data_size + ctx->dhdr_ext_size); - - ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; - - ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags; - - if (ctx->fragments > 1) { - int fin_fragments; - int i; - byte *msg = ctx->obuf->msg_start, - *msg_end = ctx->obuf->ext_endp, - *hdrp = msg_end; - - ASSERT((ctx->obuf->hopefull_flags & ctx->flags) == ctx->obuf->hopefull_flags); - ASSERT(get_int64(ctx->obuf->extp + 1 + 1 + 8) == ctx->fragments); - - /* Now that encoding is done we know how large the term will - be so we adjust the number of fragments to send. Note that - this can mean that only 1 fragment is sent. */ - fin_fragments = (ctx->obuf->ext_endp - ctx->obuf->msg_start + ERTS_DIST_FRAGMENT_SIZE-1) / - ERTS_DIST_FRAGMENT_SIZE - 1; - + Uint fid = ctx->fragments; + ErtsDistOutputBuf *obuf = ctx->obuf; + ErlIOVec *eiov; + Sint fix; + + ASSERT(fid >= 1); + ASSERT(ctx->alloced_fragments >= ctx->fragments); + + eiov = obuf[0].eiov; + ASSERT(eiov); + ASSERT(eiov->vsize >= 3); + ASSERT(!eiov->iov[0].iov_base); + ASSERT(!eiov->iov[0].iov_len); + ASSERT(!eiov->binv[0]); + ASSERT(!eiov->iov[1].iov_base); + ASSERT(!eiov->iov[1].iov_len); + ASSERT(!eiov->binv[1]); + + if (ctx->alloced_fragments > 1) { + ASSERT(get_int64(ctx->dhdrp + 1 + 1 + 8) == ctx->alloced_fragments); /* Update the frag_id in the DIST_FRAG_HEADER */ - put_int64(fin_fragments+1, ctx->obuf->extp + 1 + 1 + 8); - - if (fin_fragments > 0) - msg += ERTS_DIST_FRAGMENT_SIZE; - else - msg = msg_end; - ctx->obuf->next = &ctx->obuf[1]; - ctx->obuf->ext_endp = msg; - - /* Loop through all fragments, updating the output buffers - to be correct and also writing the DIST_FRAG_CONT header. */ - for (i = 1; i < fin_fragments + 1; i++) { - ctx->obuf[i].hopefull_flags = 0; - ctx->obuf[i].extp = msg; - ctx->obuf[i].ext_start = msg; - if (msg + ERTS_DIST_FRAGMENT_SIZE > msg_end) - ctx->obuf[i].ext_endp = msg_end; - else { - msg += ERTS_DIST_FRAGMENT_SIZE; - ctx->obuf[i].ext_endp = msg; - } - ASSERT(ctx->obuf[i].ext_endp > ctx->obuf[i].extp); - ctx->obuf[i].hdrp = erts_encode_ext_dist_header_fragment( - &hdrp, fin_fragments - i + 1, ctx->from); - ctx->obuf[i].hdr_endp = hdrp; - ctx->obuf[i].next = &ctx->obuf[i+1]; - } - /* If the initial fragment calculation was incorrect we free the - remaining output buffers. */ - for (; i < ctx->fragments; i++) { - free_dist_obuf(&ctx->obuf[i]); - } - if (!ctx->no_trap && !ctx->no_suspend) - ctx->reds -= ctx->fragments; - ctx->fragments = fin_fragments + 1; + put_int64(ctx->fragments, ctx->dhdrp + 1 + 1 + 8); + } + + eiov->size += ctx->dhdr_ext_size; + eiov->iov[1].iov_base = ctx->dhdrp; + eiov->iov[1].iov_len = ctx->dhdr_ext_size; + erts_refc_inc(&obuf[0].bin->intern.refc, 2); + eiov->binv[1] = Binary2ErlDrvBinary(obuf[0].bin); + obuf[0].next = &obuf[1]; + + for (fix = 1; fix < ctx->fragments; fix++) { + byte *hdr = erts_encode_ext_dist_header_fragment(&ctx->extp, + --fid, ctx->from); + Uint sz = ctx->extp - hdr; + + eiov = obuf[fix].eiov; + ASSERT(eiov); + ASSERT(eiov->vsize >= 3); + ASSERT(!eiov->iov[0].iov_base); + ASSERT(!eiov->iov[0].iov_len); + ASSERT(!eiov->binv[0]); + ASSERT(!eiov->iov[1].iov_base); + ASSERT(!eiov->iov[1].iov_len); + ASSERT(!eiov->binv[1]); + + eiov->size += sz; + eiov->iov[1].iov_base = hdr; + eiov->iov[1].iov_len = sz; + erts_refc_inc(&obuf[fix].bin->intern.refc, 2); + eiov->binv[1] = Binary2ErlDrvBinary(obuf[fix].bin); + obuf[fix].next = &obuf[fix+1]; + } + obuf[fix-1].next = NULL; + ASSERT(fid == 1); + /* If the initial fragment calculation was incorrect we free the + remaining output buffers. */ + for (; fix < ctx->alloced_fragments; fix++) { + free_dist_obuf(&ctx->obuf[fix], 0); } ctx->phase = ERTS_DSIG_SEND_PHASE_SEND; - if (ctx->reds <= 0) { + if (ctx->reds <= 0 && !ctx->no_trap) { retval = ERTS_DSIG_SEND_CONTINUE; goto done; } @@ -2478,7 +3169,7 @@ erts_dsig_send(ErtsDSigSendContext *ctx) /* Not the same connection as when we started; drop message... */ erts_de_runlock(dep); for (i = 0; i < ctx->fragments; i++) - free_dist_obuf(&ctx->obuf[i]); + free_dist_obuf(&ctx->obuf[i], !0); ctx->fragments = 0; } else { @@ -2538,13 +3229,9 @@ erts_dsig_send(ErtsDSigSendContext *ctx) erts_mtx_lock(&dep->qlock); } - if (fragments > 1) { - if (!ctx->obuf->hdrp) { - ASSERT(get_int64(ctx->obuf->extp + 10) == ctx->fragments); - } else { - ASSERT(get_int64(ctx->obuf->hdrp + 10) == ctx->fragments); - } - } + ASSERT(fragments < 2 + || (get_int64(ctx->obuf->eiov->iov[1].iov_base + 10) + == ctx->fragments)); if (fragments) { ctx->obuf[fragments-1].next = NULL; @@ -2666,8 +3353,19 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) bufp = NULL; } else { - size = obuf->ext_endp - obuf->extp; - bufp = (char*) obuf->extp; + char *ptr; + Sint i, vlen = obuf->eiov->vsize; + SysIOVec *iov = obuf->eiov->iov; + size = obuf->eiov->size; + bufp = ptr = erts_alloc(ERTS_ALC_T_TMP, size); + for (i = 0; i < vlen; i++) { + Uint len = iov[i].iov_len; + if (len) { + sys_memcpy((void *) ptr, (void *) iov[i].iov_base, len); + ptr += len; + } + } + ASSERT(size == ptr - bufp); } #ifdef USE_VM_PROBES @@ -2689,6 +3387,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) fpe_was_unmasked = erts_block_fpe(); (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, bufp, size); erts_unblock_fpe(fpe_was_unmasked); + erts_free(ERTS_ALC_T_TMP, bufp); return size; } @@ -2696,59 +3395,53 @@ static Uint dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - ErlDrvSizeT size = 0; - SysIOVec iov[3]; - ErlDrvBinary* bv[3]; - ErlIOVec eiov; + SysIOVec iov[1]; + Uint size; + ErlDrvBinary* bv[1]; + ErlIOVec eiov, *eiovp; ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - iov[0].iov_base = NULL; - iov[0].iov_len = 0; - bv[0] = NULL; - - if (!obuf) { - size = 0; + if (obuf) + eiovp = obuf->eiov; + else { + iov[0].iov_base = NULL; + iov[0].iov_len = 0; + bv[0] = NULL; + eiov.size = 0; eiov.vsize = 1; + eiov.iov = iov; + eiov.binv = bv; + eiovp = &eiov; } - else { - int i = 1; - eiov.vsize = 2; - - if (obuf->hdrp) { - eiov.vsize = 3; - iov[i].iov_base = obuf->hdrp; - iov[i].iov_len = obuf->hdr_endp - obuf->hdrp; - size += iov[i].iov_len; - bv[i] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); -#ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(dbg_file, "SEND: "); - bw(iov[i].iov_base, iov[i].iov_len); -#endif - i++; - } +#ifdef DEBUG + { + Uint sz; + Sint i; + for (i = 0, sz = 0; i < eiovp->vsize; i++) + sz += eiovp->iov[i].iov_len; + ASSERT(sz == eiovp->size); + } +#endif - iov[i].iov_base = obuf->extp; - iov[i].iov_len = obuf->ext_endp - obuf->extp; #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(dbg_file, "SEND: "); - bw(iov[i].iov_base, iov[i].iov_len); -#endif - size += iov[i].iov_len; - bv[i] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + { + Sint i; + erts_fprintf(dbg_file, "SEND: "); + for (i = 0; i < eiovp->vsize; i++) { + if (eiovp->iov[i].iov_len) + bw(eiovp->iov[i].iov_base, eiovp->iov[i].iov_len); + } } +#endif - eiov.size = size; - eiov.iov = iov; - eiov.binv = bv; - - if (size > (Uint) INT_MAX) + if (eiovp->size > (Uint) INT_MAX) erts_exit(ERTS_DUMP_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", - size); + eiovp->size); ASSERT(prt->drv_ptr->outputv); @@ -2768,9 +3461,14 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) #endif prt->caller = NIL; fpe_was_unmasked = erts_block_fpe(); - (*prt->drv_ptr->outputv)((ErlDrvData) prt->drv_data, &eiov); + (*prt->drv_ptr->outputv)((ErlDrvData) prt->drv_data, eiovp); erts_unblock_fpe(fpe_was_unmasked); + size = (Uint) eiovp->size; + /* Remove header used by driver... */ + eiovp->size -= eiovp->iov[0].iov_len; + eiovp->iov[0].iov_base = NULL; + eiovp->iov[0].iov_len = 0; return size; } @@ -2796,7 +3494,7 @@ erts_dist_command(Port *prt, int initial_reds) { Sint reds = initial_reds - ERTS_PORT_REDS_DIST_CMD_START; enum dist_entry_state state; - Uint32 flags; + Uint64 flags; Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY); @@ -2809,7 +3507,7 @@ erts_dist_command(Port *prt, int initial_reds) erts_atomic_set_mb(&dep->dist_cmd_scheduled, 0); erts_de_rlock(dep); - flags = dep->flags; + flags = dep->dflags; state = dep->state; send = dep->send; erts_de_runlock(dep); @@ -2871,14 +3569,14 @@ erts_dist_command(Port *prt, int initial_reds) do { Uint size; ErtsDistOutputBuf *fob; + obufsize += size_obuf(foq.first); size = (*send)(prt, foq.first); erts_atomic64_inc_nob(&dep->out); esdp->io.out += (Uint64) size; reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); fob = foq.first; - obufsize += size_obuf(fob); foq.first = foq.first->next; - free_dist_obuf(fob); + free_dist_obuf(fob, !0); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds < 0 || (sched_flags & ERTS_PTS_FLG_EXIT); if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) @@ -2902,7 +3600,7 @@ erts_dist_command(Port *prt, int initial_reds) reds = erts_encode_ext_dist_header_finalize(ob, dep, flags, reds); obufsize -= size_obuf(ob); if (reds < 0) - break; + break; /* finalize needs to be restarted... */ last_finalized = ob; ob = ob->next; } while (ob); @@ -2938,24 +3636,23 @@ erts_dist_command(Port *prt, int initial_reds) int preempt = 0; while (oq.first && !preempt) { ErtsDistOutputBuf *fob; - Uint size; + Uint size, obsz; obufsize += size_obuf(oq.first); reds = erts_encode_ext_dist_header_finalize(oq.first, dep, flags, reds); - obufsize -= size_obuf(oq.first); - if (reds < 0) { + obsz = size_obuf(oq.first); + obufsize -= obsz; + if (reds < 0) { /* finalize needs to be restarted... */ preempt = 1; break; } - ASSERT(oq.first->bin->orig_bytes <= (char*)oq.first->extp - && oq.first->extp <= oq.first->ext_endp); size = (*send)(prt, oq.first); erts_atomic64_inc_nob(&dep->out); esdp->io.out += (Uint64) size; reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); fob = oq.first; - obufsize += size_obuf(fob); + obufsize += obsz; oq.first = oq.first->next; - free_dist_obuf(fob); + free_dist_obuf(fob, !0); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds <= 0 || (sched_flags & ERTS_PTS_FLG_EXIT); if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt) @@ -3008,7 +3705,6 @@ erts_dist_command(Port *prt, int initial_reds) done: if (obufsize != 0) { - ASSERT(obufsize > 0); erts_mtx_lock(&dep->qlock); #ifdef DEBUG qsize = (Sint) erts_atomic_add_read_nob(&dep->qsize, @@ -3060,7 +3756,7 @@ erts_dist_command(Port *prt, int initial_reds) ErtsDistOutputBuf *fob = oq.first; oq.first = oq.first->next; obufsize += size_obuf(fob); - free_dist_obuf(fob); + free_dist_obuf(fob, !0); } foq.first = NULL; @@ -3374,13 +4070,17 @@ dist_ctrl_get_data_1(BIF_ALIST_1) { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); const Sint initial_reds = ERTS_BIF_REDS_LEFT(BIF_P); - Sint reds = initial_reds, obufsize = 0; + Sint reds = initial_reds, obufsize = 0, ix, vlen; ErtsDistOutputBuf *obuf; Eterm *hp, res; - ProcBin *pb; erts_aint_t qsize; Uint32 conn_id, get_size; - Uint hsz = 0, bin_sz; + Uint hsz = 0, data_sz; + SysIOVec *iov; + ErlDrvBinary **binv; +#ifdef DEBUG + Eterm *hendp; +#endif if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); @@ -3412,7 +4112,6 @@ dist_ctrl_get_data_1(BIF_ALIST_1) { if (!dep->tmp_out_queue.first) { ASSERT(!dep->tmp_out_queue.last); - ASSERT(!dep->transcode_ctx); qsize = erts_atomic_read_acqb(&dep->qsize); if (qsize > 0) { erts_mtx_lock(&dep->qlock); @@ -3433,13 +4132,13 @@ dist_ctrl_get_data_1(BIF_ALIST_1) obuf = dep->tmp_out_queue.first; obufsize += size_obuf(obuf); - reds = erts_encode_ext_dist_header_finalize(obuf, dep, dep->flags, reds); + reds = erts_encode_ext_dist_header_finalize(obuf, dep, dep->dflags, reds); obufsize -= size_obuf(obuf); - if (reds < 0) { + if (reds < 0) { /* finalize needs to be restarted... */ erts_de_runlock(dep); if (obufsize) erts_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize); - ERTS_BIF_YIELD1(bif_export[BIF_dist_ctrl_get_data_1], + ERTS_BIF_YIELD1(&bif_trap_export[BIF_dist_ctrl_get_data_1], BIF_P, BIF_ARG_1); } @@ -3452,53 +4151,70 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_de_runlock(dep); - bin_sz = obuf->ext_endp - obuf->extp + obuf->hdr_endp - obuf->hdrp; + vlen = obuf->eiov->vsize; + data_sz = obuf->eiov->size; + iov = obuf->eiov->iov; + binv = obuf->eiov->binv; + +#ifdef DEBUG + { + Uint dbg_sz; + for (ix = 0, dbg_sz = 0; ix < vlen; ix++) + dbg_sz += iov[ix].iov_len; + ASSERT(data_sz == dbg_sz); + } +#endif + + ASSERT(vlen >= 1); + ASSERT(iov[0].iov_len == 0); + ASSERT(!binv[0]); + + hsz = 2 /* cons */ + PROC_BIN_SIZE; + hsz *= vlen - 1; get_size = dep->opts & ERTS_DIST_CTRL_OPT_GET_SIZE; if (get_size) { hsz += 3; /* 2 tuple */ - if (!IS_USMALL(0, bin_sz)) + if (!IS_USMALL(0, data_sz)) hsz += BIG_UINT_HEAP_SIZE; } - if (!obuf->hdrp) { - hp = HAlloc(BIF_P, PROC_BIN_SIZE + hsz); - pb = (ProcBin *) (char *) hp; - pb->thing_word = HEADER_PROC_BIN; - pb->size = obuf->ext_endp - obuf->extp; - pb->next = MSO(BIF_P).first; - MSO(BIF_P).first = (struct erl_off_heap_header*) pb; - pb->val = ErtsDistOutputBuf2Binary(obuf); - pb->bytes = (byte*) obuf->extp; - pb->flags = 0; - res = make_binary(pb); - hp += PROC_BIN_SIZE; - } else { - hp = HAlloc(BIF_P, PROC_BIN_SIZE * 2 + 4 + hsz); - pb = (ProcBin *) (char *) hp; - pb->thing_word = HEADER_PROC_BIN; - pb->size = obuf->ext_endp - obuf->extp; - pb->next = MSO(BIF_P).first; - MSO(BIF_P).first = (struct erl_off_heap_header*) pb; - pb->val = ErtsDistOutputBuf2Binary(obuf); - pb->bytes = (byte*) obuf->extp; - pb->flags = 0; - hp += PROC_BIN_SIZE; + hp = HAlloc(BIF_P, hsz); +#ifdef DEBUG + hendp = hp + hsz; +#endif - res = CONS(hp, make_binary(pb), NIL); - hp += 2; + res = NIL; + + for (ix = vlen - 1; ix > 0; ix--) { + Binary *bin; + ProcBin *pb; + Eterm bin_term; + ASSERT(binv[ix]); + + /* + * We intentionally avoid using sub binaries + * since the GC might convert those to heap + * binaries and by this ruin the nice preparation + * for usage of this data as I/O vector in + * nifs/drivers. + */ + + bin = ErlDrvBinary2Binary(binv[ix]); pb = (ProcBin *) (char *) hp; + hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; - pb->size = obuf->hdr_endp - obuf->hdrp; + pb->size = (Uint) iov[ix].iov_len; pb->next = MSO(BIF_P).first; MSO(BIF_P).first = (struct erl_off_heap_header*) pb; - pb->val = ErtsDistOutputBuf2Binary(obuf); - erts_refc_inc(&pb->val->intern.refc, 1); - pb->bytes = (byte*) obuf->hdrp; + pb->val = bin; + pb->bytes = (byte*) iov[ix].iov_base; pb->flags = 0; - hp += PROC_BIN_SIZE; - res = CONS(hp, make_binary(pb), res); + OH_OVERHEAD(&MSO(BIF_P), pb->size / sizeof(Eterm)); + bin_term = make_binary(pb); + + res = CONS(hp, bin_term, res); hp += 2; } @@ -3522,15 +4238,20 @@ dist_ctrl_get_data_1(BIF_ALIST_1) if (get_size) { Eterm sz_term; - if (IS_USMALL(0, bin_sz)) - sz_term = make_small(bin_sz); + if (IS_USMALL(0, data_sz)) + sz_term = make_small(data_sz); else { - sz_term = uint_to_big(bin_sz, hp); + sz_term = uint_to_big(data_sz, hp); hp += BIG_UINT_HEAP_SIZE; } res = TUPLE2(hp, sz_term, res); + hp += 3; } + ASSERT(hendp == hp); + + free_dist_obuf(obuf, 0); + BIF_RET2(res, (initial_reds - reds)); } @@ -3764,13 +4485,11 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ BIF_RETTYPE setnode_2(BIF_ALIST_2) { Process *net_kernel = NULL; - Uint creation; + Uint32 creation; int success; /* valid creation ? */ - if(!term_to_Uint(BIF_ARG_2, &creation)) - goto error; - if(creation > 3) + if(!term_to_Uint32(BIF_ARG_2, &creation)) goto error; /* valid node name ? */ @@ -3810,6 +4529,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) if (success) { inc_no_nodes(); erts_set_this_node(BIF_ARG_1, (Uint32) creation); + erts_this_dist_entry->creation = creation; erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); erts_proc_lock(net_kernel, ERTS_PROC_LOCKS_ALL); @@ -3850,7 +4570,9 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) typedef struct { DistEntry *dep; - Uint flags; + int de_locked; + Uint64 dflags; + Uint32 creation; Uint version; Eterm setup_pid; Process *net_kernel; @@ -3858,24 +4580,26 @@ typedef struct { static int setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep, - Eterm ctrlr, Uint flags, - Uint version, Eterm setup_pid, + Eterm ctrlr, Uint64 flags, + Uint32 creation, Eterm setup_pid, Process *net_kernel); static Eterm setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment **bpp); -BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) +BIF_RETTYPE erts_internal_create_dist_channel_3(BIF_ALIST_3) { BIF_RETTYPE ret; - Uint flags; + Uint64 flags; Uint version; + Uint32 creation; Eterm *hp, res_tag = THE_NON_VALUE, res = THE_NON_VALUE; DistEntry *dep = NULL; int de_locked = 0; Port *pp = NULL; int true_nk; + Eterm *tpl; Process *net_kernel = erts_whereis_process(BIF_P, ERTS_PROC_LOCK_MAIN, am_net_kernel, ERTS_PROC_LOCK_STATUS, @@ -3901,19 +4625,27 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) if (!is_internal_port(BIF_ARG_2) && !is_internal_pid(BIF_ARG_2)) goto badarg; + if (!is_tuple_arity(BIF_ARG_3, 3)) + goto badarg; + + tpl = tuple_val(BIF_ARG_3); + /* Dist flags... */ - if (!is_small(BIF_ARG_3)) + if (!term_to_Uint64(tpl[1], &flags)) goto badarg; - flags = unsigned_val(BIF_ARG_3); /* Version... */ - if (!is_small(BIF_ARG_4)) + if (!is_small(tpl[2])) goto badarg; - version = unsigned_val(BIF_ARG_4); + version = unsigned_val(tpl[2]); if (version == 0) goto badarg; + /* Creation... */ + if (!term_to_Uint32(tpl[3], &creation)) + goto badarg; + if (~flags & DFLAG_DIST_MANDATORY) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "%T", BIF_P->common.id); @@ -3944,11 +4676,18 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) goto system_limit; /* Should never happen!!! */ if (is_internal_pid(BIF_ARG_2)) { + erts_de_rwlock(dep); + de_locked = 1; + if (dep->pending_nodedown) + goto suspend; + if (BIF_P->common.id == BIF_ARG_2) { ErtsSetupConnDistCtrl scdc; scdc.dep = dep; - scdc.flags = flags; + scdc.de_locked = 1; + scdc.dflags = flags; + scdc.creation = creation; scdc.version = version; scdc.setup_pid = BIF_P->common.id; scdc.net_kernel = net_kernel; @@ -3956,6 +4695,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) res = setup_connection_distctrl(BIF_P, &scdc, NULL, NULL); /* Dec of refc on net_kernel by setup_connection_distctrl() */ net_kernel = NULL; + de_locked = 0; BUMP_REDS(BIF_P, 5); dep = NULL; @@ -3968,11 +4708,16 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) else { ErtsSetupConnDistCtrl *scdcp; + erts_de_rwunlock(dep); + de_locked = 0; + scdcp = erts_alloc(ERTS_ALC_T_SETUP_CONN_ARG, sizeof(ErtsSetupConnDistCtrl)); scdcp->dep = dep; - scdcp->flags = flags; + scdcp->de_locked = 0; + scdcp->dflags = flags; + scdcp->creation = creation; scdcp->version = version; scdcp->setup_pid = BIF_P->common.id; scdcp->net_kernel = net_kernel; @@ -4021,6 +4766,9 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) || is_not_nil(dep->cid)) goto badarg; + if(dep->pending_nodedown) + goto suspend; + erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); erts_prtsd_set(pp, ERTS_PRTSD_DIST_ENTRY, dep); @@ -4044,7 +4792,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) conn_id = dep->connection_id; set_res = setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, - version, BIF_P->common.id, + creation, BIF_P->common.id, net_kernel); /* Dec of refc on net_kernel by setup_connection_epiloge_rwunlock() */ net_kernel = NULL; @@ -4084,6 +4832,17 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) return ret; + suspend: + ASSERT(de_locked); + ASSERT(!dep->suspended_nodeup); + dep->suspended_nodeup = BIF_P; + erts_proc_inc_refc(BIF_P); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_PREP_YIELD3(ret, + &bif_trap_export[BIF_erts_internal_create_dist_channel_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + goto done; + badarg: ERTS_BIF_PREP_RET(ret, am_badarg); goto done; @@ -4095,8 +4854,8 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4) static int setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep, - Eterm ctrlr, Uint flags, - Uint version, Eterm setup_pid, + Eterm ctrlr, Uint64 flags, + Uint32 creation, Eterm setup_pid, Process *net_kernel) { Eterm notify_proc = NIL; @@ -4125,8 +4884,7 @@ setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep, if (!success) return 0; - dep->version = version; - dep->creation = 0; + dep->creation = creation; ASSERT(is_internal_port(ctrlr) || is_internal_pid(ctrlr)); ASSERT(dep->state == ERTS_DE_STATE_PENDING); @@ -4175,7 +4933,6 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment * { ErtsSetupConnDistCtrl *scdcp = (ErtsSetupConnDistCtrl *) arg; DistEntry *dep = scdcp->dep; - int dep_locked = 0; Eterm *hp; Uint32 conn_id; int dec_net_kernel_on_error = !0; @@ -4186,8 +4943,10 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment * if (ERTS_PROC_IS_EXITING(c_p)) goto badarg; - erts_de_rwlock(dep); - dep_locked = !0; + if (!scdcp->de_locked) { + erts_de_rwlock(dep); + scdcp->de_locked = !0; + } if (dep->state != ERTS_DE_STATE_PENDING) goto badarg; @@ -4211,7 +4970,7 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment * *redsp = 5; if (!setup_connection_epiloge_rwunlock(c_p, dep, c_p->common.id, - scdcp->flags, scdcp->version, + scdcp->dflags, scdcp->creation, scdcp->setup_pid, scdcp->net_kernel)) { erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); @@ -4241,7 +5000,7 @@ badarg: if (bpp) /* not called directly */ erts_free(ERTS_ALC_T_SETUP_CONN_ARG, arg); - if (dep_locked) + if (scdcp->de_locked) erts_de_rwunlock(dep); erts_deref_dist_entry(dep); @@ -4252,7 +5011,40 @@ badarg: BIF_RETTYPE erts_internal_get_dflags_0(BIF_ALIST_0) { + if (erts_dflags_test_remove_hopefull_flags) { + /* For internal emulator tests only! */ + Eterm *hp, **hpp = NULL; + Uint sz = 0, *szp = &sz; + Eterm res; + while (1) { + res = erts_bld_tuple(hpp, szp, 6, + am_erts_dflags, + erts_bld_uint64(hpp, szp, DFLAG_DIST_DEFAULT & ~DFLAG_DIST_HOPEFULLY), + erts_bld_uint64(hpp, szp, DFLAG_DIST_MANDATORY & ~DFLAG_DIST_HOPEFULLY), + erts_bld_uint64(hpp, szp, DFLAG_DIST_ADDABLE & ~DFLAG_DIST_HOPEFULLY), + erts_bld_uint64(hpp, szp, DFLAG_DIST_REJECTABLE & ~DFLAG_DIST_HOPEFULLY), + erts_bld_uint64(hpp, szp, DFLAG_DIST_STRICT_ORDER & ~DFLAG_DIST_HOPEFULLY)); + if (hpp) { + ASSERT(is_value(res)); + return res; + } + hp = HAlloc(BIF_P, sz); + hpp = &hp; + szp = NULL; + } + } return erts_dflags_record; + +} + +BIF_RETTYPE erts_internal_get_creation_0(BIF_ALIST_0) +{ + Eterm *hp; + Uint hsz = 0; + + erts_bld_uint(NULL, &hsz, erts_this_dist_entry->creation); + hp = HAlloc(BIF_P, hsz); + return erts_bld_uint(&hp, NULL, erts_this_dist_entry->creation); } BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1) @@ -4331,7 +5123,7 @@ Sint erts_abort_pending_connection_rwunlock(DistEntry* dep, erts_de_rwunlock(dep); schedule_con_monitor_link_seq_cleanup( - mld, NULL, THE_NON_VALUE, + NULL, mld, NULL, THE_NON_VALUE, THE_NON_VALUE, THE_NON_VALUE); if (resume_procs) { @@ -4421,6 +5213,398 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) return 1; } +static BIF_RETTYPE spawn_request_yield_3(BIF_ALIST_3) +{ + Binary* bin = erts_magic_ref2bin(BIF_ARG_1); + ErtsDSigSendContext *ctx = (ErtsDSigSendContext*) ERTS_MAGIC_BIN_DATA(bin); + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); + int code; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dsend_context_dtor); + + ctx->reds = initial_reds; + code = erts_dsig_send(ctx); + + switch (code) { + case ERTS_DSIG_SEND_OK: + erts_set_gc_state(BIF_P, 1); + BIF_RET(BIF_ARG_2); + + case ERTS_DSIG_SEND_YIELD: + erts_set_gc_state(BIF_P, 1); + ERTS_BIF_YIELD_RETURN(BIF_P, BIF_ARG_2); + + case ERTS_DSIG_SEND_CONTINUE: { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&spawn_request_yield_export, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + + case ERTS_DSIG_SEND_TOO_LRG: { + ErtsMonitor *mon; + ErtsMonitorData *mdp; + Eterm ref; + + erts_set_gc_state(BIF_P, 1); + + if (is_internal_ordinary_ref(BIF_ARG_2)) + ref = BIF_ARG_2; + else { + Eterm *tp; + ASSERT(is_tuple_arity(BIF_ARG_2, 2)); + tp = tuple_val(BIF_ARG_2); + ref = tp[1]; + ASSERT(is_internal_ordinary_ref(ref)); + } + + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(BIF_P), ref); + ASSERT(mon); + erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), mon); + mdp = erts_monitor_to_data(mon); + if (erts_monitor_dist_delete(&mdp->target)) + erts_monitor_release_both(mdp); + else + erts_monitor_release(mon); + + erts_send_local_spawn_reply(BIF_P, ERTS_PROC_LOCK_MAIN, NULL, + BIF_ARG_3, ref, am_system_limit, + am_undefined); + BIF_RET(BIF_ARG_2); + } + + default: + ERTS_INTERNAL_ERROR("Invalid spawn request result"); + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + } +} + +BIF_RETTYPE erts_internal_dist_spawn_request_4(BIF_ALIST_4) +{ + BIF_RETTYPE ret_val; + Eterm node = BIF_ARG_1; + Eterm mfa = BIF_ARG_2; + Eterm opts = BIF_ARG_3; + Eterm tag = am_spawn_reply; + Eterm mod, func, alist, new_opts, error, ref, + ok_result; + Uint nargs, nopts, rm_opts, rebuild_opts; + DistEntry *dep = NULL; + Eterm list; + ErtsDSigSendContext ctx; + int code, link = 0, monitor = 0, success_message, error_message; + + ok_result = THE_NON_VALUE; + success_message = error_message = !0; + + if (!is_node_name_atom(node)) + goto badarg; + dep = erts_find_or_insert_dist_entry(node); + if (dep == erts_this_dist_entry) + goto badarg; + if (!is_tuple_arity(mfa, 3)) + goto badarg; + else { + Eterm *tp = tuple_val(mfa); + mod = tp[1]; + func = tp[2]; + alist = tp[3]; + if (!is_atom(mod)) + goto badarg; + if (!is_atom(func)) + goto badarg; + nargs = 0; + list = alist; + while (is_list(list)) { + Eterm *cp = list_val(list); + list = CDR(cp); + nargs++; + } + if (!is_nil(list)) + goto badarg; + } + + new_opts = list = opts; + nopts = 0; + rm_opts = 0; + rebuild_opts = 0; + + while (is_list(list)) { + Eterm *cp = list_val(list); + Eterm car = CAR(cp); + list = CDR(cp); + nopts++; + switch (car) { + case am_link: + link = !0; + break; + case am_monitor: + monitor = !0; + break; + default: + if (is_tuple_arity(car, 2)) { + Eterm *tp = tuple_val(car); + switch (tp[1]) { + + case am_reply_tag: + tag = tp[2]; + + if (0) { + case am_reply: + switch (tp[2]) { + case am_error_only: + success_message = 0; + error_message = !0; + break; + case am_success_only: + success_message = !0; + error_message = 0; + break; + case am_no: + success_message = 0; + error_message = 0; + break; + case am_yes: + success_message = !0; + error_message = !0; + break; + default: + goto badarg; + } + } + + if (BIF_ARG_4 != am_spawn_request) + goto badarg; + + rm_opts++; + new_opts = list; + rebuild_opts = nopts - rm_opts; + break; + + default: + break; + } + } + break; + } + } + if (!is_nil(list)) + goto badarg; + + if (rm_opts) { + /* + * If no 'rebuild_opts', all options to drop were in + * the head of the 'opts' list. 'new_opts' now contain + * the tail of original option list without the dropped + * options. + */ + if (rebuild_opts) { +#ifdef DEBUG + Eterm reusable_tail, *hp_start; +#endif + Uint rm_cnt; + Eterm *hp, *prev_cp; + /* + * Remove 'reply_tag' option in option list. + * This options are mixed with other options. + * + * We build the list backwards and reuse tail + * without options to remove, if such exist. + */ + + hp = HAlloc(BIF_P, 2*rebuild_opts); + +#ifdef DEBUG + hp_start = hp; + reusable_tail = new_opts; +#endif + + hp += 2*(rebuild_opts - 1); + new_opts = make_list(hp); + prev_cp = NULL; + list = opts; + rm_cnt = 0; + + while (is_list(list)) { + Eterm *cp = list_val(list); + Eterm car = CAR(cp); + list = CDR(cp); + if (is_tuple_arity(car, 2)) { + Eterm *tp = tuple_val(car); + if (am_reply_tag == tp[1] + || am_reply == tp[1]) { + rm_cnt++; + /* skip option */ + if (rm_cnt == rm_opts) { + ASSERT(prev_cp); + ASSERT(list == reusable_tail); + CDR(prev_cp) = list; + break; /* last option to skip */ + } + continue; + } + } +#ifdef DEBUG + rebuild_opts--; +#endif + CAR(hp) = car; + prev_cp = hp; + hp -= 2; + CDR(prev_cp) = make_list(hp); + } + ASSERT(hp == hp_start - 2); + ASSERT(rebuild_opts == 0); + } + + opts = new_opts; + } + + /* Arguments checked; do it... */ + + ref = erts_make_ref(BIF_P); + if (BIF_ARG_4 == am_spawn_request) + ok_result = ref; + else { + Eterm *hp = HAlloc(BIF_P, 3); + ASSERT(BIF_ARG_4 == am_spawn_opt); + ok_result = TUPLE2(hp, ref, monitor ? am_true : am_false); + } + + code = erts_dsig_prepare(&ctx, dep, + BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 0, 1); + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + goto noconnection; + + case ERTS_DSIG_PREP_CONNECTED: + if (!(dep->dflags & DFLAG_SPAWN)) { + erts_de_runlock(dep); + goto notsup; + } + /* Fall through... */ + case ERTS_DSIG_PREP_PENDING: { + int inserted; + ErtsMonitorData *mdp; + Eterm nargs_term, mfna, *hp; + + if (IS_USMALL(0, nargs)) { + hp = HAlloc(BIF_P, 4); + nargs_term = make_small(nargs); + } + else { + hp = HAlloc(BIF_P, 4+BIG_UINT_HEAP_SIZE); + nargs_term = uint_to_big(nargs, hp); + hp += BIG_UINT_HEAP_SIZE; + } + + mfna = TUPLE3(hp, mod, func, nargs_term); + + mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, ref, + BIF_P->common.id, am_pending, + tag); + if (monitor) + mdp->origin.flags |= ERTS_ML_FLG_SPAWN_MONITOR; + if (link) + mdp->origin.flags |= ERTS_ML_FLG_SPAWN_LINK; + if (!success_message) + mdp->origin.flags |= ERTS_ML_FLG_SPAWN_NO_SMSG; + if (!error_message) + mdp->origin.flags |= ERTS_ML_FLG_SPAWN_NO_EMSG; + + erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), + &mdp->origin); + inserted = erts_monitor_dist_insert(&mdp->target, dep->mld); + ASSERT(inserted); (void)inserted; + + erts_de_runlock(dep); + + ctx.reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); + + code = dsig_send_spawn_request(&ctx, ref, BIF_P->common.id, + BIF_P->group_leader, mfna, + alist, opts); + switch (code) { + case ERTS_DSIG_SEND_OK: + ERTS_BIF_PREP_RET(ret_val, ok_result); + break; + + case ERTS_DSIG_SEND_YIELD: + ERTS_BIF_PREP_YIELD_RETURN(ret_val, BIF_P, ok_result); + break; + + case ERTS_DSIG_SEND_CONTINUE: { + Eterm ctx_term; + /* Keep dist entry alive over trap... */ + ctx.deref_dep = 1; + dep = NULL; + + erts_set_gc_state(BIF_P, 0); + ctx_term = erts_dsend_export_trap_context(BIF_P, &ctx); + BUMP_ALL_REDS(BIF_P); + ERTS_BIF_PREP_TRAP3(ret_val, &spawn_request_yield_export, + BIF_P, ctx_term, ok_result, tag); + break; + } + + case ERTS_DSIG_SEND_TOO_LRG: { + ErtsMonitor *mon; + ErtsMonitorData *mdp; + + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(BIF_P), ref); + ASSERT(mon); + erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), mon); + mdp = erts_monitor_to_data(mon); + if (erts_monitor_dist_delete(&mdp->target)) + erts_monitor_release_both(mdp); + else + erts_monitor_release(mon); + + goto system_limit; + } + + default: + ERTS_INTERNAL_ERROR("Invalid spawn request result"); + ERTS_BIF_PREP_RET(ret_val, am_internal_error); + } + break; + } + default: + ERTS_INTERNAL_ERROR("Invalid dsig prepare result"); + ERTS_BIF_PREP_RET(ret_val, am_internal_error); + break; + } + +do_return: + + if (dep) + erts_deref_dist_entry(dep); + + return ret_val; + +badarg: + ERTS_BIF_PREP_RET(ret_val, am_badarg); + goto do_return; + +system_limit: + error = am_system_limit; + goto send_error; +noconnection: + error = am_noconnection; + goto send_error; +notsup: + error = am_notsup; + /* fall through... */ +send_error: + ASSERT(is_value(ok_result)); + if (error_message) + erts_send_local_spawn_reply(BIF_P, ERTS_PROC_LOCK_MAIN, NULL, + tag, ref, error, am_undefined); + ERTS_BIF_PREP_RET(ret_val, ok_result); + goto do_return; +} + + /**********************************************************************/ /* node(Object) -> Node */ @@ -4756,7 +5940,7 @@ BIF_RETTYPE monitor_node_2(BIF_ALIST_2) BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) { DistEntry *de; - Uint32 f; + Uint64 f; if (is_not_pid(BIF_ARG_1)) { BIF_ERROR(BIF_P,BADARG); } @@ -4766,7 +5950,7 @@ BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) BIF_RET(am_true); } erts_de_rlock(de); - f = de->flags; + f = de->dflags; erts_de_runlock(de); BIF_RET(((f & DFLAG_UNICODE_IO) ? am_true : am_false)); } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index f6cb79472f..65c29caeb3 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -25,46 +25,57 @@ #include "erl_node_tables.h" #include "zlib.h" -#define DFLAG_PUBLISHED 0x01 -#define DFLAG_ATOM_CACHE 0x02 -#define DFLAG_EXTENDED_REFERENCES 0x04 -#define DFLAG_DIST_MONITOR 0x08 -#define DFLAG_FUN_TAGS 0x10 -#define DFLAG_DIST_MONITOR_NAME 0x20 -#define DFLAG_HIDDEN_ATOM_CACHE 0x40 -#define DFLAG_NEW_FUN_TAGS 0x80 -#define DFLAG_EXTENDED_PIDS_PORTS 0x100 -#define DFLAG_EXPORT_PTR_TAG 0x200 -#define DFLAG_BIT_BINARIES 0x400 -#define DFLAG_NEW_FLOATS 0x800 -#define DFLAG_UNICODE_IO 0x1000 -#define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 -#define DFLAG_SMALL_ATOM_TAGS 0x4000 -#define DFLAG_INTERNAL_TAGS 0x8000 /* used by ETS 'compressed' option */ -#define DFLAG_UTF8_ATOMS 0x10000 -#define DFLAG_MAP_TAG 0x20000 -#define DFLAG_BIG_CREATION 0x40000 -#define DFLAG_SEND_SENDER 0x80000 -#define DFLAG_BIG_SEQTRACE_LABELS 0x100000 -#define DFLAG_NO_MAGIC 0x200000 /* internal for pending connection */ -#define DFLAG_EXIT_PAYLOAD 0x400000 -#define DFLAG_FRAGMENTS 0x800000 +#define DFLAG_PUBLISHED ((Uint64)0x01) +#define DFLAG_ATOM_CACHE ((Uint64)0x02) +#define DFLAG_EXTENDED_REFERENCES ((Uint64)0x04) +#define DFLAG_DIST_MONITOR ((Uint64)0x08) +#define DFLAG_FUN_TAGS ((Uint64)0x10) +#define DFLAG_DIST_MONITOR_NAME ((Uint64)0x20) +#define DFLAG_HIDDEN_ATOM_CACHE ((Uint64)0x40) +#define DFLAG_NEW_FUN_TAGS ((Uint64)0x80) +#define DFLAG_EXTENDED_PIDS_PORTS ((Uint64)0x100) +#define DFLAG_EXPORT_PTR_TAG ((Uint64)0x200) +#define DFLAG_BIT_BINARIES ((Uint64)0x400) +#define DFLAG_NEW_FLOATS ((Uint64)0x800) +#define DFLAG_UNICODE_IO ((Uint64)0x1000) +#define DFLAG_DIST_HDR_ATOM_CACHE ((Uint64)0x2000) +#define DFLAG_SMALL_ATOM_TAGS ((Uint64)0x4000) +#define DFLAG_ETS_COMPRESSED ((Uint64)0x8000) /* internal */ +#define DFLAG_UTF8_ATOMS ((Uint64)0x10000) +#define DFLAG_MAP_TAG ((Uint64)0x20000) +#define DFLAG_BIG_CREATION ((Uint64)0x40000) +#define DFLAG_SEND_SENDER ((Uint64)0x80000) +#define DFLAG_BIG_SEQTRACE_LABELS ((Uint64)0x100000) +#define DFLAG_PENDING_CONNECT ((Uint64)0x200000) /* internal */ +#define DFLAG_EXIT_PAYLOAD ((Uint64)0x400000) +#define DFLAG_FRAGMENTS ((Uint64)0x800000) +#define DFLAG_HANDSHAKE_23 ((Uint64)0x1000000) +#define DFLAG_RESERVED 0xfe000000 +/* + * As the old handshake only support 32 flag bits, we reserve the remainding + * bits in the lower 32 for changes in the handshake protocol or potentially + * new capabilities that we also want to backport to OTP-22 or older. + */ +#define DFLAG_SPAWN ((Uint64)0x100000000) + /* Mandatory flags for distribution */ #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ | DFLAG_EXTENDED_PIDS_PORTS \ | DFLAG_UTF8_ATOMS \ - | DFLAG_NEW_FUN_TAGS) + | DFLAG_NEW_FUN_TAGS \ + | DFLAG_BIG_CREATION) /* * Additional optimistic flags when encoding toward pending connection. - * If remote node (erl_interface) does not supporting these then we may need + * If remote node (erl_interface) does not support these then we may need * to transcode messages enqueued before connection setup was finished. */ #define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_DIST_MONITOR \ - | DFLAG_DIST_MONITOR_NAME) + | DFLAG_DIST_MONITOR_NAME \ + | DFLAG_SPAWN) /* Our preferred set of flags. Used for connection setup handshake */ #define DFLAG_DIST_DEFAULT (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY \ @@ -75,11 +86,12 @@ | DFLAG_SMALL_ATOM_TAGS \ | DFLAG_UTF8_ATOMS \ | DFLAG_MAP_TAG \ - | DFLAG_BIG_CREATION \ | DFLAG_SEND_SENDER \ | DFLAG_BIG_SEQTRACE_LABELS \ | DFLAG_EXIT_PAYLOAD \ - | DFLAG_FRAGMENTS) + | DFLAG_FRAGMENTS \ + | DFLAG_HANDSHAKE_23 \ + | DFLAG_SPAWN) /* Flags addable by local distr implementations */ #define DFLAG_DIST_ADDABLE DFLAG_DIST_DEFAULT @@ -87,6 +99,7 @@ /* Flags rejectable by local distr implementation */ #define DFLAG_DIST_REJECTABLE (DFLAG_DIST_HDR_ATOM_CACHE \ | DFLAG_HIDDEN_ATOM_CACHE \ + | DFLAG_FRAGMENTS \ | DFLAG_ATOM_CACHE) /* Flags for all features needing strict order delivery */ @@ -130,9 +143,17 @@ enum dop { DOP_PAYLOAD_EXIT_TT = 25, DOP_PAYLOAD_EXIT2 = 26, DOP_PAYLOAD_EXIT2_TT = 27, - DOP_PAYLOAD_MONITOR_P_EXIT = 28 + DOP_PAYLOAD_MONITOR_P_EXIT = 28, + + DOP_SPAWN_REQUEST = 29, + DOP_SPAWN_REQUEST_TT = 30, + DOP_SPAWN_REPLY = 31, + DOP_SPAWN_REPLY_TT = 32 }; +#define ERTS_DIST_SPAWN_FLAG_LINK (1 << 0) +#define ERTS_DIST_SPAWN_FLAG_MONITOR (1 << 1) + /* distribution trap functions */ extern Export* dmonitor_node_trap; @@ -144,7 +165,7 @@ typedef enum { /* Must be larger or equal to 16 */ #ifdef DEBUG -#define ERTS_DIST_FRAGMENT_SIZE 16 +#define ERTS_DIST_FRAGMENT_SIZE 1024 #else /* This should be made configurable */ #define ERTS_DIST_FRAGMENT_SIZE (64 * 1024) @@ -175,6 +196,9 @@ extern int erts_is_alive; /* dist_ctrl_{g,s}et_option/2 */ #define ERTS_DIST_CTRL_OPT_GET_SIZE ((Uint32) (1 << 0)) +/* for emulator internal testing... */ +extern int erts_dflags_test_remove_hopefull_flags; + #ifdef DEBUG #define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) \ erts_dbg_chk_no_dist_proc_link((D), (R), (L)) @@ -193,23 +217,75 @@ extern int erts_is_alive; typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; typedef struct TTBSizeContext_ { - Uint flags; + Uint64 dflags; int level; + Sint vlen; + int iovec; + Uint fragment_size; + Uint last_result; + Uint extra_size; Uint result; Eterm obj; ErtsWStack wstack; } TTBSizeContext; +#define ERTS_INIT_TTBSizeContext(Ctx, Flags) \ + do { \ + (Ctx)->wstack.wstart = NULL; \ + (Ctx)->dflags = (Flags); \ + (Ctx)->level = 0; \ + (Ctx)->vlen = -1; \ + (Ctx)->fragment_size = ~((Uint) 0); \ + (Ctx)->extra_size = 0; \ + (Ctx)->last_result = 0; \ + } while (0) + typedef struct TTBEncodeContext_ { - Uint flags; - Uint hopefull_flags; + Uint64 dflags; + Uint64 hopefull_flags; + byte *hopefull_flagsp; int level; byte* ep; Eterm obj; ErtsWStack wstack; Binary *result_bin; + byte *cptr; + Sint vlen; + Uint size; + byte *payload_ixp; + byte *hopefull_ixp; + SysIOVec* iov; + ErlDrvBinary** binv; + Eterm *termv; + int iovec; + Uint fragment_size; + Sint frag_ix; + ErlIOVec **fragment_eiovs; +#ifdef DEBUG + int debug_fragments; + int debug_vlen; +#endif } TTBEncodeContext; +#define ERTS_INIT_TTBEncodeContext(Ctx, Flags) \ + do { \ + (Ctx)->wstack.wstart = NULL; \ + (Ctx)->dflags = (Flags); \ + (Ctx)->level = 0; \ + (Ctx)->vlen = 0; \ + (Ctx)->size = 0; \ + (Ctx)->termv = NULL; \ + (Ctx)->iov = NULL; \ + (Ctx)->binv = NULL; \ + (Ctx)->fragment_size = ~((Uint) 0); \ + if ((Flags) & DFLAG_PENDING_CONNECT) { \ + (Ctx)->hopefull_flags = 0; \ + (Ctx)->hopefull_flagsp = NULL; \ + (Ctx)->hopefull_ixp = NULL; \ + (Ctx)->payload_ixp = NULL; \ + } \ + } while (0) + typedef struct { Uint real_size; Uint dest_len; @@ -246,7 +322,7 @@ typedef struct erts_dsig_send_context { Eterm ctl; Eterm msg; Eterm from; - Eterm ctl_heap[6]; + Eterm ctl_heap[8]; /* 7-tuple (SPAWN_REQUEST_TT) */ Eterm return_term; DistEntry *dep; @@ -258,12 +334,13 @@ typedef struct erts_dsig_send_context { enum erts_dsig_send_phase phase; Sint reds; - Uint32 max_finalize_prepend; Uint data_size, dhdr_ext_size; + byte *dhdrp, *extp; ErtsAtomCacheMap *acmp; ErtsDistOutputBuf *obuf; - Uint fragments; - Uint32 flags; + Uint alloced_fragments, fragments; + Sint vlen; + Uint64 dflags; Process *c_p; union { TTBSizeContext sc; @@ -305,6 +382,7 @@ extern int erts_dsig_send_exit2(ErtsDSigSendContext *, Eterm, Eterm, Eterm); extern int erts_dsig_send_demonitor(ErtsDSigSendContext *, Eterm, Eterm, Eterm); extern int erts_dsig_send_monitor(ErtsDSigSendContext *, Eterm, Eterm, Eterm); extern int erts_dsig_send_m_exit(ErtsDSigSendContext *, Eterm, Eterm, Eterm, Eterm); +extern int erts_dsig_send_spawn_reply(ErtsDSigSendContext *, Eterm, Eterm, Eterm, Eterm, Eterm); extern int erts_dsig_send(ErtsDSigSendContext *dsdp); extern int erts_dsend_context_dtor(Binary*); @@ -333,4 +411,8 @@ extern int erts_dsig_prepare(ErtsDSigSendContext *, void erts_dist_print_procs_suspended_on_de(fmtfn_t to, void *to_arg); int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks); + +Uint erts_ttb_iov_size(int use_termv, Sint vlen, Uint fragments); +ErlIOVec **erts_ttb_iov_init(TTBEncodeContext *ctx, int use_termv, char *ptr, + Sint vlen, Uint fragments, Uint fragments_size); #endif diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index b9f0334172..1bbc7d7f1e 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -653,8 +653,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] - = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] = sizeof(ErtsNSchedMagicRefTableEntry); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)] @@ -2392,10 +2390,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); - add_fix_values(&size.processes, - &size.processes_used, - fi, - ERTS_ALC_T_NIF_EXP_TRACE); } if (want.atom || want.atom_used) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index b40f5d0622..3d14e86445 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -195,6 +195,8 @@ type DB_DMC_ERR_INFO ETS ETS db_dmc_error_info type DB_TERM ETS ETS db_term type DB_PROC_CLEANUP SHORT_LIVED ETS db_proc_cleanup_state type ETS_ALL_REQ SHORT_LIVED ETS ets_all_request +type ETS_CTRS ETS ETS ets_decentralized_ctrs +type ETS_I_LST_TRAP SHORT_LIVED ETS ets_insert_list_bif_trap_state type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf @@ -279,6 +281,7 @@ type SETUP_CONN_ARG SHORT_LIVED PROCESSES setup_connection_argument type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state type CONT_EXIT_TRAP SHORT_LIVED PROCESSES continue_exit_trap_state type SEQ_YIELD_STATE SHORT_LIVED SYSTEM dist_seq_yield_state +type PHASH2_TRAP SHORT_LIVED PROCESSES phash2_trap_state type ENVIRONMENT SYSTEM SYSTEM environment @@ -286,6 +289,8 @@ type PERSISTENT_TERM LONG_LIVED CODE persisten_term type PERSISTENT_LOCK_Q SHORT_LIVED SYSTEM persistent_lock_q type PERSISTENT_TERM_TMP SHORT_LIVED SYSTEM persistent_term_tmp_table +type T2B_VEC SHORT_LIVED PROCESSES term_to_binary_vector + # # Types used for special emulators # @@ -331,8 +336,7 @@ type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data -type NIF_TRAP_EXPORT STANDARD PROCESSES nif_trap_export_entry -type NIF_EXP_TRACE FIXED_SIZE PROCESSES nif_export_trace +type NFUNC_TRAP_WRAPPER STANDARD PROCESSES nfunc_trap_wrapper type EXPORT LONG_LIVED CODE export_entry type MONITOR FIXED_SIZE PROCESSES monitor type MONITOR_SUSPEND STANDARD PROCESSES monitor_suspend diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index bfc2f5992c..8123f99375 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -481,244 +481,265 @@ static void check_blk_carrier(Allctr_t *, Block_t *); /* Statistics updating ... */ #ifdef DEBUG -#define DEBUG_CHECK_CARRIER_NO_SZ(AP) \ - ASSERT(((AP)->sbcs.curr.norm.mseg.no \ - && (AP)->sbcs.curr.norm.mseg.size) \ - || (!(AP)->sbcs.curr.norm.mseg.no \ - && !(AP)->sbcs.curr.norm.mseg.size)); \ - ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no \ - && (AP)->sbcs.curr.norm.sys_alloc.size) \ - || (!(AP)->sbcs.curr.norm.sys_alloc.no \ - && !(AP)->sbcs.curr.norm.sys_alloc.size)); \ - ASSERT(((AP)->mbcs.curr.norm.mseg.no \ - && (AP)->mbcs.curr.norm.mseg.size) \ - || (!(AP)->mbcs.curr.norm.mseg.no \ - && !(AP)->mbcs.curr.norm.mseg.size)); \ - ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ - && (AP)->mbcs.curr.norm.sys_alloc.size) \ - || (!(AP)->mbcs.curr.norm.sys_alloc.no \ - && !(AP)->mbcs.curr.norm.sys_alloc.size)); + +#define DEBUG_CHECK_CARRIER_NO_SZ_1(CSTATS) \ + do { \ + int ix__; \ + for (ix__ = ERTS_CRR_ALLOC_MIN; ix__ <= ERTS_CRR_ALLOC_MAX; ix__++) { \ + UWord no__ = (CSTATS)->carriers[ix__].no; \ + UWord size__= (CSTATS)->carriers[ix__].size; \ + ASSERT((no__ > 0 && size__ > 0) || (no__ == 0 && size__ == 0)); \ + } \ + } while (0) + +#define DEBUG_CHECK_CARRIER_NO_SZ(AP) \ + do { \ + DEBUG_CHECK_CARRIER_NO_SZ_1(&(AP)->sbcs); \ + DEBUG_CHECK_CARRIER_NO_SZ_1(&(AP)->mbcs); \ + } while (0) #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) #endif -#define STAT_SBC_ALLOC(AP, BSZ) \ - (AP)->sbcs.blocks.curr.size += (BSZ); \ - if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size) \ - (AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size; \ - if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no \ - + (AP)->sbcs.curr.norm.sys_alloc.no)) \ - (AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no \ - + (AP)->sbcs.curr.norm.sys_alloc.no); \ - if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size \ - + (AP)->sbcs.curr.norm.sys_alloc.size)) \ - (AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size \ - + (AP)->sbcs.curr.norm.sys_alloc.size) - -#define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \ -do { \ - (AP)->sbcs.curr.norm.mseg.no++; \ - (AP)->sbcs.curr.norm.mseg.size += (CSZ); \ - STAT_SBC_ALLOC((AP), (BSZ)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \ -do { \ - (AP)->sbcs.curr.norm.sys_alloc.no++; \ - (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ); \ - STAT_SBC_ALLOC((AP), (BSZ)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - - -#define STAT_SBC_FREE(AP, BSZ) \ - ASSERT((AP)->sbcs.blocks.curr.size >= (BSZ)); \ - (AP)->sbcs.blocks.curr.size -= (BSZ) - -#define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \ -do { \ - ASSERT((AP)->sbcs.curr.norm.mseg.no > 0); \ - (AP)->sbcs.curr.norm.mseg.no--; \ - ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ)); \ - (AP)->sbcs.curr.norm.mseg.size -= (CSZ); \ - STAT_SBC_FREE((AP), (BSZ)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \ -do { \ - ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0); \ - (AP)->sbcs.curr.norm.sys_alloc.no--; \ - ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ)); \ - (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ); \ - STAT_SBC_FREE((AP), (BSZ)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_MBC_ALLOC(AP) \ - if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no \ - + (AP)->mbcs.curr.norm.sys_alloc.no)) \ - (AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no \ - + (AP)->mbcs.curr.norm.sys_alloc.no); \ - if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size \ - + (AP)->mbcs.curr.norm.sys_alloc.size)) \ - (AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size \ - + (AP)->mbcs.curr.norm.sys_alloc.size) - - -#define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->mbcs.curr.norm.mseg.no++; \ - (AP)->mbcs.curr.norm.mseg.size += (CSZ); \ - STAT_MBC_ALLOC((AP)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->mbcs.curr.norm.sys_alloc.no++; \ - (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ); \ - STAT_MBC_ALLOC((AP)); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ -do { \ - UWord csz__ = CARRIER_SZ((CRR)); \ - if (IS_MSEG_CARRIER((CRR))) \ - STAT_MSEG_MBC_ALLOC((AP), csz__); \ - else \ - STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ - set_new_allctr_abandon_limit(AP); \ - (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks[(AP)->alloc_no]; \ - if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ - (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ - (AP)->mbcs.blocks.curr.size += \ - (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ - if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ - (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ -} while (0) - -#define STAT_MSEG_MBC_FREE(AP, CSZ) \ -do { \ - ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \ - (AP)->mbcs.curr.norm.mseg.no--; \ - ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ)); \ - (AP)->mbcs.curr.norm.mseg.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \ -do { \ - ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0); \ - (AP)->mbcs.curr.norm.sys_alloc.no--; \ - ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ)); \ - (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - -#define STAT_MBC_FREE(AP, CRR) \ -do { \ - UWord csz__ = CARRIER_SZ((CRR)); \ - if (IS_MSEG_CARRIER((CRR))) { \ - STAT_MSEG_MBC_FREE((AP), csz__); \ - } else { \ - STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ - } \ - set_new_allctr_abandon_limit(AP); \ -} while (0) - -#define STAT_MBC_ABANDON(AP, CRR) \ -do { \ - STAT_MBC_FREE(AP, CRR); \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ - >= (CRR)->cpool.blocks[(AP)->alloc_no]); \ - (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks[(AP)->alloc_no]; \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ - >= (CRR)->cpool.blocks_size[(AP)->alloc_no]); \ - (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ -} while (0) - -#define STAT_MBC_BLK_ALLOC_CRR(AP, CRR, BSZ) \ -do { \ - (CRR)->cpool.blocks[(AP)->alloc_no]++; \ - (CRR)->cpool.blocks_size[(AP)->alloc_no] += (BSZ); \ - (CRR)->cpool.total_blocks_size += (BSZ); \ -} while (0) - -#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ -do { \ - CarriersStats_t *cstats__ = &(AP)->mbcs; \ - cstats__->blocks.curr.no++; \ - if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ - cstats__->blocks.max.no = cstats__->blocks.curr.no; \ - cstats__->blocks.curr.size += (BSZ); \ - if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ - cstats__->blocks.max.size = cstats__->blocks.curr.size; \ - STAT_MBC_BLK_ALLOC_CRR((AP), (CRR), (BSZ)); \ -} while (0) +/* Carrier statistics */ + +#define STAT_CRR_UPDATED_1(CSTATS) \ + do { \ + UWord no_sum__, size_sum__; \ + int i__; \ + no_sum__ = size_sum__ = 0; \ + for (i__ = ERTS_CRR_ALLOC_MIN; i__ <= ERTS_CRR_ALLOC_MAX; i__++) { \ + ASSERT(ERTS_UWORD_MAX - no_sum__ > (CSTATS)->carriers[i__].no); \ + ASSERT(ERTS_UWORD_MAX - size_sum__ > (CSTATS)->carriers[i__].size); \ + no_sum__ += (CSTATS)->carriers[i__].no; \ + size_sum__ += (CSTATS)->carriers[i__].size; \ + } \ + if ((CSTATS)->max.no < no_sum__) { \ + (CSTATS)->max.no = no_sum__; \ + } \ + if ((CSTATS)->max.size < size_sum__) { \ + (CSTATS)->max.size = size_sum__; \ + } \ + } while (0) + +#define STAT_CRR_ALLOC_1(TYPE, CSTATS, SIZE) \ + do { \ + ASSERT(ERTS_UWORD_MAX - (CSTATS)->carriers[(TYPE)].no > 1); \ + ASSERT(ERTS_UWORD_MAX - (CSTATS)->carriers[(TYPE)].size > (SIZE)); \ + (CSTATS)->carriers[(TYPE)].no += 1; \ + (CSTATS)->carriers[(TYPE)].size += (SIZE); \ + STAT_CRR_UPDATED_1(CSTATS); \ + } while (0) + +#define STAT_CRR_FREE_1(TYPE, CSTATS, SIZE) \ + do { \ + ASSERT((CSTATS)->carriers[(TYPE)].no >= 1); \ + ASSERT((CSTATS)->carriers[(TYPE)].size >= (SIZE)); \ + (CSTATS)->carriers[(TYPE)].no -= 1; \ + (CSTATS)->carriers[(TYPE)].size -= (SIZE); \ + STAT_CRR_UPDATED_1(CSTATS); \ + } while (0) + +#define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->sbcs; \ + STAT_CRR_ALLOC_1(ERTS_CRR_ALLOC_MSEG, cstats__, CSZ); \ + STAT_BLK_ALLOC_1(cstats__, (AP)->alloc_no, 1, (BSZ)); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->sbcs; \ + STAT_CRR_ALLOC_1(ERTS_CRR_ALLOC_SYS, cstats__, CSZ); \ + STAT_BLK_ALLOC_1(cstats__, (AP)->alloc_no, 1, (BSZ)); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->sbcs; \ + STAT_CRR_FREE_1(ERTS_CRR_ALLOC_MSEG, cstats__, CSZ); \ + STAT_BLK_FREE_1(cstats__, (AP)->alloc_no, 1, (BSZ)); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->sbcs; \ + STAT_CRR_FREE_1(ERTS_CRR_ALLOC_SYS, cstats__, CSZ); \ + STAT_BLK_FREE_1(cstats__, (AP)->alloc_no, 1, (BSZ)); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + STAT_CRR_ALLOC_1(ERTS_CRR_ALLOC_MSEG, cstats__, CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + STAT_CRR_ALLOC_1(ERTS_CRR_ALLOC_SYS, cstats__, CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_MSEG_MBC_FREE(AP, CSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + STAT_CRR_FREE_1(ERTS_CRR_ALLOC_MSEG, cstats__, CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + STAT_CRR_FREE_1(ERTS_CRR_ALLOC_SYS, cstats__, CSZ); \ + DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + } while (0) + +#define STAT_MBC_FREE(AP, CRR) \ + do { \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) { \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + } else { \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + } \ + set_new_allctr_abandon_limit(AP); \ + } while (0) + +/* Block statistics */ + +#define STAT_BLK_UPDATED_1(BSTATS) \ + do { \ + if ((BSTATS)->max.no < (BSTATS)->curr.no) { \ + (BSTATS)->max.no = (BSTATS)->curr.no; \ + } \ + if ((BSTATS)->max.size < (BSTATS)->curr.size) { \ + (BSTATS)->max.size = (BSTATS)->curr.size; \ + } \ + } while (0) + +#define STAT_BLK_ALLOC_1(CSTATS, ALLOC_NO, COUNT, SIZE) \ + do { \ + BlockStats_t *bstats__ = \ + &(CSTATS)->blocks[(ALLOC_NO) - ERTS_ALC_A_MIN]; \ + ASSERT(ERTS_UWORD_MAX - bstats__->curr.no > (COUNT)); \ + ASSERT(ERTS_UWORD_MAX - bstats__->curr.size > (SIZE)); \ + bstats__->curr.no += (COUNT); \ + bstats__->curr.size += (SIZE); \ + STAT_BLK_UPDATED_1(bstats__); \ + } while (0) + +#define STAT_BLK_FREE_1(CSTATS, ALLOC_NO, COUNT, SIZE) \ + do { \ + BlockStats_t *bstats__ = \ + &(CSTATS)->blocks[(ALLOC_NO) - ERTS_ALC_A_MIN]; \ + ASSERT(bstats__->curr.no >= (COUNT)); \ + ASSERT(bstats__->curr.size >= (SIZE)); \ + bstats__->curr.no -= (COUNT); \ + bstats__->curr.size -= (SIZE); \ + STAT_BLK_UPDATED_1(bstats__); \ + } while (0) + +#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + UWord csz__ = CARRIER_SZ((CRR)); \ + int alloc_no__; \ + if (IS_MSEG_CARRIER((CRR))) { \ + STAT_MSEG_MBC_ALLOC((AP), csz__); \ + } else { \ + STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ + } \ + set_new_allctr_abandon_limit(AP); \ + for (alloc_no__ = ERTS_ALC_A_MIN; \ + alloc_no__ <= ERTS_ALC_A_MAX; \ + alloc_no__++) { \ + int ix__ = alloc_no__ - ERTS_ALC_A_MIN; \ + UWord count = (CRR)->cpool.blocks[ix__]; \ + UWord size = (CRR)->cpool.blocks_size[ix__]; \ + STAT_BLK_ALLOC_1(cstats__, alloc_no__, count, size); \ + } \ + } while (0) + +#define STAT_MBC_CPOOL_ABANDON(AP, CRR) \ + do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + int alloc_no__; \ + STAT_MBC_FREE(AP, CRR); \ + for (alloc_no__ = ERTS_ALC_A_MIN; \ + alloc_no__ <= ERTS_ALC_A_MAX; \ + alloc_no__++) { \ + int ix__ = alloc_no__ - ERTS_ALC_A_MIN; \ + UWord count = (CRR)->cpool.blocks[ix__]; \ + UWord size = (CRR)->cpool.blocks_size[ix__]; \ + STAT_BLK_FREE_1(cstats__, alloc_no__, count, size); \ + } \ + } while (0) -static ERTS_INLINE int +static ERTS_INLINE void stat_cpool_mbc_blk_free(Allctr_t *allctr, - ErtsAlcType_t type, - Carrier_t *crr, - Carrier_t **busy_pcrr_pp, - UWord blksz) + int alloc_no, + Carrier_t *crr, + Carrier_t **busy_pcrr_pp, + UWord blksz) { - Allctr_t *orig_allctr; - int alloc_no; - - alloc_no = ERTS_ALC_T2A(type); - - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] > 0); - crr->cpool.blocks[alloc_no]--; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] >= blksz); - crr->cpool.blocks_size[alloc_no] -= blksz; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.total_blocks_size >= blksz); - crr->cpool.total_blocks_size -= blksz; - - if (allctr->alloc_no == alloc_no && (!busy_pcrr_pp || !*busy_pcrr_pp)) { + if (!busy_pcrr_pp || !*busy_pcrr_pp) { /* This is a local block, so we should not update the pool * statistics. */ - return 0; - } - - /* This is either a foreign block that's been fetched from the pool, or any - * block that's in the pool. The carrier's owner keeps the statistics for - * both pooled and foreign blocks. */ - - orig_allctr = crr->cpool.orig_allctr; + CarriersStats_t *cstats__ = &allctr->mbcs; + STAT_BLK_FREE_1(cstats__, alloc_no, 1, blksz); + } else { + Allctr_t *orig_allctr = crr->cpool.orig_allctr; + int ix = alloc_no - ERTS_ALC_A_MIN; - ERTS_ALC_CPOOL_ASSERT(alloc_no != allctr->alloc_no || - (crr == *busy_pcrr_pp && allctr == orig_allctr)); + ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp && allctr == orig_allctr); #ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT( - erts_atomic_dec_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0); - ERTS_ALC_CPOOL_ASSERT( - erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], - -((erts_aint_t) blksz)) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_dec_read_nob( + &orig_allctr->cpool.stat.no_blocks[ix]) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[ix], + -((erts_aint_t) blksz)) >= 0); #else - erts_atomic_dec_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]); - erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], - -((erts_aint_t) blksz)); + erts_atomic_dec_nob(&orig_allctr->cpool.stat.no_blocks[ix]); + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[ix], + -((erts_aint_t) blksz)); #endif - - return 1; + } } -#define STAT_MBC_BLK_FREE(AP, TYPE, CRR, BPCRRPP, BSZ, FLGS) \ -do { \ - if (!stat_cpool_mbc_blk_free((AP), (TYPE), (CRR), (BPCRRPP), (BSZ))) { \ - CarriersStats_t *cstats__ = &(AP)->mbcs; \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ - } \ -} while (0) +#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ) \ + do { \ + int alloc_no__ = (AP)->alloc_no; \ + int ix__ = alloc_no__ - ERTS_ALC_A_MIN; \ + ASSERT(ERTS_UWORD_MAX - (CRR)->cpool.blocks[ix__] > 1); \ + ASSERT(ERTS_UWORD_MAX - (CRR)->cpool.blocks_size[ix__] > (BSZ)); \ + ASSERT(ERTS_UWORD_MAX - (CRR)->cpool.total_blocks_size > (BSZ)); \ + (CRR)->cpool.blocks[ix__] += 1; \ + (CRR)->cpool.blocks_size[ix__] += (BSZ); \ + (CRR)->cpool.total_blocks_size += (BSZ); \ + STAT_BLK_ALLOC_1(&(AP)->mbcs, alloc_no__, 1, (BSZ)); \ + } while (0) + +#define STAT_MBC_BLK_FREE(AP, TYPE, CRR, BPCRRPP, BSZ) \ + do { \ + int alloc_no__ = ERTS_ALC_T2A(TYPE); \ + int ix__ = (alloc_no__) - ERTS_ALC_A_MIN; \ + ASSERT((CRR)->cpool.blocks[ix__] >= 1); \ + ASSERT((CRR)->cpool.blocks_size[ix__] >= (BSZ)); \ + ASSERT((CRR)->cpool.total_blocks_size >= (BSZ)); \ + (CRR)->cpool.blocks[ix__] -= 1; \ + (CRR)->cpool.blocks_size[ix__] -= (BSZ); \ + (CRR)->cpool.total_blocks_size -= (BSZ); \ + stat_cpool_mbc_blk_free((AP), alloc_no__, (CRR), (BPCRRPP), (BSZ)); \ + } while (0) /* Debug stuff... */ #ifdef DEBUG @@ -1211,9 +1232,14 @@ static Uint get_next_mbc_size(Allctr_t *allctr) { Uint size; - int cs = (allctr->mbcs.curr.norm.mseg.no - + allctr->mbcs.curr.norm.sys_alloc.no - - (allctr->main_carrier ? 1 : 0)); + int cs; + int i; + + cs = 0; + for (i = ERTS_CRR_ALLOC_MIN; i <= ERTS_CRR_ALLOC_MAX; i++) { + cs += allctr->mbcs.carriers[i].no; + } + cs -= (allctr->main_carrier ? 1 : 0); ASSERT(cs >= 0); ASSERT(allctr->largest_mbc_size >= allctr->smallest_mbc_size); @@ -2285,8 +2311,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->cpool.disable_abandon) return; - if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) - return; + { + int ix = allctr->alloc_no - ERTS_ALC_A_MIN; + + /* We only consider the current allocation type; . */ + if (allctr->mbcs.blocks[ix].curr.size > allctr->cpool.abandon_limit) { + return; + } + } ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers); if (ncrr_in_pool >= allctr->cpool.in_pool_limit) @@ -2515,7 +2547,7 @@ mbc_alloc_finalize(Allctr_t *allctr, } ERTS_ALC_CPOOL_ALLOC_OP(allctr); - STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -2734,7 +2766,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, type, crr, busy_pcrr_pp, blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, busy_pcrr_pp, blk_sz); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -2934,8 +2966,8 @@ mbc_realloc(Allctr_t *allctr, ErtsAlcType_t type, void *p, Uint size, crr = ABLK_TO_MBC(blk); ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -3038,8 +3070,8 @@ mbc_realloc(Allctr_t *allctr, ErtsAlcType_t type, void *p, Uint size, } ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -3186,7 +3218,7 @@ mbc_realloc(Allctr_t *allctr, ErtsAlcType_t type, void *p, Uint size, 0); ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz); return new_p; } @@ -3357,27 +3389,17 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) erts_aint_t val; ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; Allctr_t *orig_allctr = crr->cpool.orig_allctr; + int ix; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); - { - int alloc_no = allctr->alloc_no; - - ERTS_ALC_CPOOL_ASSERT( - erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no]) >= 0 && - crr->cpool.blocks_size[alloc_no] >= 0); - - ERTS_ALC_CPOOL_ASSERT( - erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0 && - crr->cpool.blocks[alloc_no] >= 0); - - /* We only modify the counter for our current type since the others are - * conceptually still in the pool. */ - erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], - ((erts_aint_t) crr->cpool.blocks_size[alloc_no])); - erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], - ((erts_aint_t) crr->cpool.blocks[alloc_no])); + /* Add the carrier's block statistics to the pool. */ + for (ix = 0; ix < ERTS_ALC_A_COUNT; ix++) { + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[ix], + ((erts_aint_t) crr->cpool.blocks_size[ix])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[ix], + ((erts_aint_t) crr->cpool.blocks[ix])); } erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, @@ -3454,11 +3476,15 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) #ifdef ERTS_ALC_CPOOL_DEBUG ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; #endif + Allctr_t *orig_allctr = crr->cpool.orig_allctr; + int ix; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); + ERTS_ALC_CPOOL_ASSERT(orig_allctr == prev_allctr); + /* Set mod marker on next ptr of our predecessor */ val = (erts_aint_t) &crr->cpool; @@ -3531,30 +3557,31 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) crr->cpool.thr_prgr = erts_thr_progress_later(NULL); - { - Allctr_t *orig_allctr = crr->cpool.orig_allctr; - int alloc_no = allctr->alloc_no; - - ERTS_ALC_CPOOL_ASSERT(orig_allctr == prev_allctr); - - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] <= - erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no])); - - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] <= - erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no])); + /* Subtract the carrier's block statistics from the pool. */ + for (ix = 0; ix < ERTS_ALC_A_COUNT; ix++) { +#ifdef ERTS_ALC_CPOOL_DEBUG + SWord new_blk_sz, new_blk_no; - /* We only modify the counters for our current type since the others - * were, conceptually, never taken out of the pool. */ - erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], - -((erts_aint_t) crr->cpool.blocks_size[alloc_no])); - erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], - -((erts_aint_t) crr->cpool.blocks[alloc_no])); + new_blk_sz = + erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[ix], + -((erts_aint_t)crr->cpool.blocks_size[ix])); + new_blk_no = + erts_atomic_add_read_nob(&orig_allctr->cpool.stat.no_blocks[ix], + -((erts_aint_t)crr->cpool.blocks[ix])); - erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, - -((erts_aint_t) CARRIER_SZ(crr))); - erts_atomic_dec_wb(&orig_allctr->cpool.stat.no_carriers); + ERTS_ALC_CPOOL_ASSERT(new_blk_sz >= 0); + ERTS_ALC_CPOOL_ASSERT(new_blk_no >= 0); +#else + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[ix], + -((erts_aint_t) crr->cpool.blocks_size[ix])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[ix], + -((erts_aint_t) crr->cpool.blocks[ix])); +#endif } + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, + -((erts_aint_t) CARRIER_SZ(crr))); + erts_atomic_dec_wb(&orig_allctr->cpool.stat.no_carriers); } static Carrier_t * @@ -3937,9 +3964,12 @@ allctr_abandon_limit(Allctr_t *allctr) { UWord limit; UWord csz; + int i; - csz = allctr->mbcs.curr.norm.mseg.size; - csz += allctr->mbcs.curr.norm.sys_alloc.size; + csz = 0; + for (i = ERTS_CRR_ALLOC_MIN; i <= ERTS_CRR_ALLOC_MAX; i++) { + csz += allctr->mbcs.carriers[i].size; + } limit = csz*allctr->cpool.util_limit; if (limit > csz) @@ -3961,7 +3991,7 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr) { erts_aint_t iallctr; - STAT_MBC_ABANDON(allctr, crr); + STAT_MBC_CPOOL_ABANDON(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); allctr->remove_mbc(allctr, crr); @@ -4023,9 +4053,12 @@ static void cpool_read_stat(Allctr_t *allctr, int alloc_no, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) { + int block_ix; int i; UWord noc = 0, csz = 0, nob = 0, bsz = 0; + block_ix = alloc_no - ERTS_ALC_A_MIN; + /* * We try to get consistent values, but after * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed @@ -4041,10 +4074,10 @@ cpool_read_stat(Allctr_t *allctr, int alloc_no, ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) : 0); tnob = (UWord) (nobp - ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks[alloc_no]) + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks[block_ix]) : 0); tbsz = (UWord) (bszp - ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size[alloc_no]) + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size[block_ix]) : 0); if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) break; @@ -4192,12 +4225,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (erts_mseg_no(&allctr->mseg_opt) >= max_mseg_carriers) goto try_sys_alloc; if (flags & CFLG_SBC) { - if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) + if (allctr->sbcs.carriers[ERTS_CRR_ALLOC_MSEG].no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } #if !ERTS_SUPER_ALIGNED_MSEG_ONLY else { - if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) + if (allctr->mbcs.carriers[ERTS_CRR_ALLOC_MSEG].no >= allctr->max_mseg_mbcs) goto try_sys_alloc; } #endif @@ -4638,10 +4671,10 @@ static struct { Eterm mseg_alloc_carriers; #endif Eterm carriers; - Eterm blocks_size; - Eterm blocks; - Eterm foreign_blocks; + Eterm blocks; + Eterm count; + Eterm size; Eterm calls; Eterm sys_alloc; @@ -4658,6 +4691,7 @@ static struct { } am; static Eterm alloc_type_atoms[ERTS_ALC_N_MAX + 1]; +static Eterm alloc_num_atoms[ERTS_ALC_A_MAX + 1]; static ERTS_INLINE void atom_init(Eterm *atom, char *name) { @@ -4741,9 +4775,9 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_alloc_carriers); #endif AM_INIT(carriers); - AM_INIT(blocks_size); AM_INIT(blocks); - AM_INIT(foreign_blocks); + AM_INIT(count); + AM_INIT(size); AM_INIT(calls); AM_INIT(sys_alloc); @@ -4767,6 +4801,13 @@ init_atoms(Allctr_t *allctr) alloc_type_atoms[ix] = am_atom_put(name, len); } + + for (ix = ERTS_ALC_A_MIN; ix <= ERTS_ALC_A_MAX; ix++) { + const char *name = ERTS_ALC_A2AD(ix); + size_t len = sys_strlen(name); + + alloc_num_atoms[ix] = am_atom_put(name, len); + } } if (allctr && !allctr->atoms_initialized) { @@ -4939,39 +4980,81 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + UWord curr_size; + int i; + + curr_size = 0; + for (i = ERTS_CRR_ALLOC_MIN; i <= ERTS_CRR_ALLOC_MAX; i++) { + curr_size += cs->carriers[i].size; + } if (print_to_p) { - fmtfn_t to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "%sblocks size: %bpu %bpu %bpu\n", - prefix, - cs->blocks.curr.size, - cs->blocks.max.size, - cs->blocks.max_ever.size); - erts_print(to, - arg, - "%scarriers size: %bpu %bpu %bpu\n", - prefix, - curr_size, - cs->max.size, - cs->max_ever.size); + fmtfn_t to = *print_to_p; + void *arg = print_to_arg; + int alloc_no; + + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + int ix = alloc_no - ERTS_ALC_A_MIN; + + erts_print(to, + arg, + "%sblocks[%s] size: %bpu %bpu %bpu\n", + prefix, + erts_alc_a2ad[alloc_no], + cs->blocks[ix].curr.size, + cs->blocks[ix].max.size, + cs->blocks[ix].max_ever.size); + } + + erts_print(to, + arg, + "%scarriers size: %bpu %bpu %bpu\n", + prefix, + curr_size, + cs->max.size, + cs->max_ever.size); } if (hpp || szp) { - res = NIL; - add_4tup(hpp, szp, &res, - am.carriers_size, - bld_unstable_uint(hpp, szp, curr_size), - bld_unstable_uint(hpp, szp, cs->max.size), - bld_unstable_uint(hpp, szp, cs->max_ever.size)); - add_4tup(hpp, szp, &res, - am.blocks_size, - bld_unstable_uint(hpp, szp, cs->blocks.curr.size), - bld_unstable_uint(hpp, szp, cs->blocks.max.size), - bld_unstable_uint(hpp, szp, cs->blocks.max_ever.size)); + Eterm blocks; + int alloc_no; + + res = NIL; + + add_4tup(hpp, szp, &res, + am.carriers_size, + bld_unstable_uint(hpp, szp, curr_size), + bld_unstable_uint(hpp, szp, cs->max.size), + bld_unstable_uint(hpp, szp, cs->max_ever.size)); + + blocks = NIL; + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + int ix = alloc_no - ERTS_ALC_A_MIN; + UWord curr, max, max_ever; + Eterm info = NIL; + + curr = cs->blocks[ix].curr.size; + max = cs->blocks[ix].max.size; + max_ever = cs->blocks[ix].max_ever.size; + + if (curr == 0 && max == 0 && max_ever == 0) { + continue; + } + + add_4tup(hpp, szp, &info, + am.size, + bld_unstable_uint(hpp, szp, curr), + bld_unstable_uint(hpp, szp, max), + bld_unstable_uint(hpp, szp, max_ever)); + + add_2tup(hpp, szp, &blocks, alloc_num_atoms[alloc_no], info); + } + + add_2tup(hpp, szp, &res, am.blocks, blocks); } return res; @@ -4988,33 +5071,43 @@ info_cpool(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - UWord noc, csz, nob, bsz; + UWord noc, csz; - noc = csz = nob = bsz = ~0; + noc = csz = ~0; if (print_to_p || hpp) { - if (sz_only) - cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); - else - cpool_read_stat(allctr, allctr->alloc_no, &noc, &csz, &nob, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, &noc, &csz, NULL, NULL); } if (print_to_p) { - fmtfn_t to = *print_to_p; - void *arg = print_to_arg; - if (!sz_only) - erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); - erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz); - if (!sz_only) - erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); - erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); + fmtfn_t to = *print_to_p; + void *arg = print_to_arg; + int alloc_no; + + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + UWord nob, bsz; + + nob = bsz = ~0; + cpool_read_stat(allctr, alloc_no, NULL, NULL, &nob, &bsz); + + if (!sz_only) + erts_print(to, arg, "%sblocks[%s] count: %bpu\n", + prefix, erts_alc_a2ad[alloc_no], nob); + erts_print(to, arg, "%sblocks[%s] size: %bpu\n", + prefix, erts_alc_a2ad[alloc_no], bsz); + } + + if (!sz_only) + erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); + erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); } if (hpp || szp) { - Eterm foreign_blocks; - int i; + Eterm blocks; + int alloc_no; - foreign_blocks = NIL; - res = NIL; + res = NIL; if (!sz_only) { add_3tup(hpp, szp, &res, am.fail_pooled, @@ -5072,49 +5165,36 @@ info_cpool(Allctr_t *allctr, bld_unstable_uint(hpp, szp, noc)); } - add_2tup(hpp, szp, &res, - am.blocks_size, - bld_unstable_uint(hpp, szp, bsz)); - - if (!sz_only) { - add_2tup(hpp, szp, &res, - am.blocks, - bld_unstable_uint(hpp, szp, nob)); - } - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - const char *name_str; - Eterm name, info; - - if (i == allctr->alloc_no) { - continue; - } + blocks = NIL; + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + UWord nob, bsz; + Eterm info; - cpool_read_stat(allctr, i, NULL, NULL, &nob, &bsz); + nob = bsz = ~0; + cpool_read_stat(allctr, alloc_no, NULL, NULL, &nob, &bsz); if (bsz == 0 && (nob == 0 || sz_only)) { continue; } - name_str = ERTS_ALC_A2AD(i); info = NIL; add_2tup(hpp, szp, &info, - am.blocks_size, + am.size, bld_unstable_uint(hpp, szp, bsz)); if (!sz_only) { add_2tup(hpp, szp, &info, - am.blocks, + am.count, bld_unstable_uint(hpp, szp, nob)); } - name = am_atom_put(name_str, sys_strlen(name_str)); - - add_2tup(hpp, szp, &foreign_blocks, name, info); + add_2tup(hpp, szp, &blocks, alloc_num_atoms[alloc_no], info); } - add_2tup(hpp, szp, &res, am.foreign_blocks, foreign_blocks); + add_2tup(hpp, szp, &res, am.blocks, blocks); } return res; @@ -5132,27 +5212,41 @@ info_carriers(Allctr_t *allctr, { Eterm res = THE_NON_VALUE; UWord curr_no, curr_size; - - curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; + int i; + + curr_no = curr_size = 0; + for (i = ERTS_CRR_ALLOC_MIN; i <= ERTS_CRR_ALLOC_MAX; i++) { + curr_no += cs->carriers[i].no; + curr_size += cs->carriers[i].size; + } if (print_to_p) { - fmtfn_t to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "%sblocks: %bpu %bpu %bpu\n", - prefix, - cs->blocks.curr.no, - cs->blocks.max.no, - cs->blocks.max_ever.no); - erts_print(to, - arg, - "%sblocks size: %bpu %bpu %bpu\n", - prefix, - cs->blocks.curr.size, - cs->blocks.max.size, - cs->blocks.max_ever.size); + fmtfn_t to = *print_to_p; + void *arg = print_to_arg; + int alloc_no; + + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + int ix = alloc_no - ERTS_ALC_A_MIN; + erts_print(to, + arg, + "%sblocks[%s] count: %bpu %bpu %bpu\n", + prefix, + erts_alc_a2ad[alloc_no], + cs->blocks[ix].curr.no, + cs->blocks[ix].max.no, + cs->blocks[ix].max_ever.no); + erts_print(to, + arg, + "%sblocks[%s] size: %bpu %bpu %bpu\n", + prefix, + erts_alc_a2ad[alloc_no], + cs->blocks[ix].curr.size, + cs->blocks[ix].max.size, + cs->blocks[ix].max_ever.size); + } + erts_print(to, arg, "%scarriers: %bpu %bpu %bpu\n", @@ -5165,13 +5259,13 @@ info_carriers(Allctr_t *allctr, arg, "%smseg carriers: %bpu\n", prefix, - cs->curr.norm.mseg.no); + cs->carriers[ERTS_CRR_ALLOC_MSEG].no); #endif erts_print(to, arg, "%ssys_alloc carriers: %bpu\n", prefix, - cs->curr.norm.sys_alloc.no); + cs->carriers[ERTS_CRR_ALLOC_SYS].no); erts_print(to, arg, "%scarriers size: %bpu %bpu %bpu\n", @@ -5184,24 +5278,27 @@ info_carriers(Allctr_t *allctr, arg, "%smseg carriers size: %bpu\n", prefix, - cs->curr.norm.mseg.size); + cs->carriers[ERTS_CRR_ALLOC_MSEG].size); #endif erts_print(to, arg, "%ssys_alloc carriers size: %bpu\n", prefix, - cs->curr.norm.sys_alloc.size); + cs->carriers[ERTS_CRR_ALLOC_SYS].size); } if (hpp || szp) { + Eterm blocks; + int alloc_no; + res = NIL; add_2tup(hpp, szp, &res, am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); + bld_unstable_uint(hpp, szp, cs->carriers[ERTS_CRR_ALLOC_SYS].size)); #if HAVE_ERTS_MSEG add_2tup(hpp, szp, &res, am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); + bld_unstable_uint(hpp, szp, cs->carriers[ERTS_CRR_ALLOC_MSEG].size)); #endif add_4tup(hpp, szp, &res, am.carriers_size, @@ -5210,27 +5307,57 @@ info_carriers(Allctr_t *allctr, bld_unstable_uint(hpp, szp, cs->max_ever.size)); add_2tup(hpp, szp, &res, am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); + bld_unstable_uint(hpp, szp, cs->carriers[ERTS_CRR_ALLOC_SYS].no)); #if HAVE_ERTS_MSEG add_2tup(hpp, szp, &res, am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); + bld_unstable_uint(hpp, szp, cs->carriers[ERTS_CRR_ALLOC_MSEG].no)); #endif add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), bld_unstable_uint(hpp, szp, cs->max.no), bld_unstable_uint(hpp, szp, cs->max_ever.no)); - add_4tup(hpp, szp, &res, - am.blocks_size, - bld_unstable_uint(hpp, szp, cs->blocks.curr.size), - bld_unstable_uint(hpp, szp, cs->blocks.max.size), - bld_unstable_uint(hpp, szp, cs->blocks.max_ever.size)); - add_4tup(hpp, szp, &res, - am.blocks, - bld_unstable_uint(hpp, szp, cs->blocks.curr.no), - bld_unstable_uint(hpp, szp, cs->blocks.max.no), - bld_unstable_uint(hpp, szp, cs->blocks.max_ever.no)); + + blocks = NIL; + for (alloc_no = ERTS_ALC_A_MIN; + alloc_no <= ERTS_ALC_A_MAX; + alloc_no++) { + int ix = alloc_no - ERTS_ALC_A_MIN; + UWord curr_size, max_size, max_ever_size; + UWord curr_no, max_no, max_ever_no; + Eterm info; + + curr_size = cs->blocks[ix].curr.size; + max_size = cs->blocks[ix].max.size; + max_ever_size = cs->blocks[ix].max_ever.size; + + curr_no = cs->blocks[ix].curr.no; + max_no = cs->blocks[ix].max.no; + max_ever_no = cs->blocks[ix].max_ever.no; + + if (max_ever_no == 0) { + continue; + } + + info = NIL; + + add_4tup(hpp, szp, &info, + am.size, + bld_unstable_uint(hpp, szp, curr_size), + bld_unstable_uint(hpp, szp, max_size), + bld_unstable_uint(hpp, szp, max_ever_size)); + + add_4tup(hpp, szp, &info, + am.count, + bld_unstable_uint(hpp, szp, curr_no), + bld_unstable_uint(hpp, szp, max_no), + bld_unstable_uint(hpp, szp, max_ever_no)); + + add_2tup(hpp, szp, &blocks, alloc_num_atoms[alloc_no], info); + } + + add_2tup(hpp, szp, &res, am.blocks, blocks); } return res; @@ -5490,23 +5617,50 @@ info_options(Allctr_t *allctr, static ERTS_INLINE void update_max_ever_values(CarriersStats_t *cs) { - if (cs->max_ever.no < cs->max.no) - cs->max_ever.no = cs->max.no; - if (cs->max_ever.size < cs->max.size) - cs->max_ever.size = cs->max.size; - if (cs->blocks.max_ever.no < cs->blocks.max.no) - cs->blocks.max_ever.no = cs->blocks.max.no; - if (cs->blocks.max_ever.size < cs->blocks.max.size) - cs->blocks.max_ever.size = cs->blocks.max.size; + int ix; + + for (ix = 0; ix < ERTS_ALC_A_COUNT; ix++) { + BlockStats_t *bstats = &cs->blocks[ix]; + + if (bstats->max_ever.no < bstats->max.no) { + bstats->max_ever.no = bstats->max.no; + } + + if (bstats->max_ever.size < bstats->max.size) { + bstats->max_ever.size = bstats->max.size; + } + } + + if (cs->max_ever.no < cs->max.no) { + cs->max_ever.no = cs->max.no; + } + + if (cs->max_ever.size < cs->max.size) { + cs->max_ever.size = cs->max.size; + } } static ERTS_INLINE void reset_max_values(CarriersStats_t *cs) { - cs->max.no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - cs->max.size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; - cs->blocks.max.no = cs->blocks.curr.no; - cs->blocks.max.size = cs->blocks.curr.size; + UWord curr_no, curr_size; + int ix; + + for (ix = 0; ix < ERTS_ALC_A_COUNT; ix++) { + BlockStats_t *bstats = &cs->blocks[ix]; + + bstats->max.no = bstats->curr.no; + bstats->max.size = bstats->curr.size; + } + + curr_no = curr_size = 0; + for (ix = ERTS_CRR_ALLOC_MIN; ix <= ERTS_CRR_ALLOC_MAX; ix++) { + curr_no += cs->carriers[ix].no; + curr_size += cs->carriers[ix].size; + } + + cs->max.no = curr_no; + cs->max.size = curr_size; } @@ -5614,11 +5768,6 @@ erts_alcu_sz_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continuously updated */ - allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; - allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -5690,11 +5839,6 @@ erts_alcu_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continuously updated */ - allctr->sbcs.blocks.curr.no - = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; - allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -5753,61 +5897,66 @@ erts_alcu_info(Allctr_t *allctr, void erts_alcu_foreign_size(Allctr_t *allctr, ErtsAlcType_t alloc_no, AllctrSize_t *size) { + int ix; + + sys_memset(size, 0, sizeof(*size)); + + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); + + for (ix = ERTS_CRR_ALLOC_MIN; ix <= ERTS_CRR_ALLOC_MAX; ix++) { + size->carriers += allctr->mbcs.carriers[ix].size; + size->carriers += allctr->sbcs.carriers[ix].size; + } + + ix = alloc_no - ERTS_ALC_A_MIN; + size->blocks += allctr->mbcs.blocks[ix].curr.size; + size->blocks += allctr->sbcs.blocks[ix].curr.size; + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { UWord csz, bsz; + + csz = bsz = 0; cpool_read_stat(allctr, alloc_no, NULL, &csz, NULL, &bsz); - size->carriers = csz; - size->blocks = bsz; - } else { - size->carriers = 0; - size->blocks = 0; + + size->carriers += csz; + size->blocks += bsz; } + + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); } void erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) { - - if (allctr->thread_safe) - erts_mtx_lock(&allctr->mutex); - - size->carriers = allctr->mbcs.curr.norm.mseg.size; - size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; - size->carriers += allctr->sbcs.curr.norm.mseg.size; - size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; - - size->blocks = allctr->mbcs.blocks.curr.size; - size->blocks += allctr->sbcs.blocks.curr.size; - - if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - UWord csz, bsz; - cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); - size->blocks += bsz; - size->carriers += csz; - } + erts_alcu_foreign_size(allctr, allctr->alloc_no, size); if (fi) { - int ix; - for (ix = 0; ix < fisz; ix++) { - if (allctr->fix) { - if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].u.cpool.allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].u.cpool.used); - } - else { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].u.nocpool.allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].u.nocpool.used); - } - } - } - } + int ix; + + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); + + for (ix = 0; ix < fisz; ix++) { + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.used); + } else { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.used); + } + } + } - if (allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); + } } /* ----------------------------------------------------------------------- */ @@ -6660,10 +6809,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; allctr->cpool.disable_abandon = 0; - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + + for (i = 0; i < ERTS_ALC_A_COUNT; i++) { erts_atomic_init_nob(&allctr->cpool.stat.blocks_size[i], 0); erts_atomic_init_nob(&allctr->cpool.stat.no_blocks[i], 0); } + erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); if (!init->ts && init->acul && init->acnl) { @@ -7696,14 +7847,15 @@ typedef struct chist_node__ { UWord carrier_size; UWord unscanned_size; - UWord allocated_size; - /* BLOCKSCAN_BAILOUT_THRESHOLD guarantees we won't overflow this or the - * counters in the free block histogram. */ - int allocated_count; int flags; - int histogram[1]; + /* A mirror of the block counters in the carrier's ErtsAlcCPoolData_t. */ + UWord alloc_count[ERTS_ALC_A_COUNT]; + UWord alloc_size[ERTS_ALC_A_COUNT]; + + /* BLOCKSCAN_BAILOUT_THRESHOLD guarantees we won't overflow. */ + int free_histogram[1]; } chist_node_t; typedef struct { @@ -7712,7 +7864,7 @@ typedef struct { ErtsIRefStorage iref; Process *process; - Eterm allocator_desc; + int allocator_number; chist_node_t *info_list; UWord info_count; @@ -7737,7 +7889,7 @@ static int gather_cinfo_scan(Allctr_t *allocator, state = (gather_cinfo_t*)user_data; node = calloc(1, sizeof(chist_node_t) + (state->hist_slot_count - 1) * - sizeof(node->histogram[0])); + sizeof(node->free_histogram[0])); blocks_scanned = 1; /* ERTS_CRR_ALCTR_FLG_BUSY is ignored since we've set it ourselves and it @@ -7747,16 +7899,20 @@ static int gather_cinfo_scan(Allctr_t *allocator, node->carrier_size = CARRIER_SZ(carrier); if (IS_SB_CARRIER(carrier)) { - UWord block_size; - - block = SBC2BLK(allocator, carrier); - block_size = SBC_BLK_SZ(block); + int ix = allocator->alloc_no - ERTS_ALC_A_MIN; - node->allocated_size = block_size; - node->allocated_count = 1; + node->alloc_count[ix] = 1; + node->alloc_size[ix] = node->carrier_size; } else { UWord scanned_bytes = MBC_HEADER_SIZE(allocator); + sys_memcpy(&node->alloc_count[0], + &carrier->cpool.blocks[0], + sizeof(UWord) * ERTS_ALC_A_COUNT); + sys_memcpy(&node->alloc_size[0], + &carrier->cpool.blocks_size[0], + sizeof(UWord) * ERTS_ALC_A_COUNT); + block = MBC_TO_FIRST_BLK(allocator, carrier); while (1) { @@ -7764,10 +7920,7 @@ static int gather_cinfo_scan(Allctr_t *allocator, scanned_bytes += block_size; - if (IS_ALLOCED_BLK(block)) { - node->allocated_size += block_size; - node->allocated_count++; - } else { + if (IS_FREE_BLK(block)) { UWord size_interval; int hist_slot; @@ -7776,7 +7929,7 @@ static int gather_cinfo_scan(Allctr_t *allocator, hist_slot = MIN(size_interval, state->hist_slot_count - 1); - node->histogram[hist_slot]++; + node->free_histogram[hist_slot]++; } if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) { @@ -7801,46 +7954,89 @@ static int gather_cinfo_scan(Allctr_t *allocator, static void gather_cinfo_append_result(gather_cinfo_t *state, chist_node_t *info) { - Eterm carrier_size, unscanned_size, allocated_size; - Eterm histogram_tuple, carrier_tuple; + Eterm carrier_tuple, block_list, histogram_tuple; + Eterm carrier_size, unscanned_size; Uint term_size; Eterm *hp; int ix; ASSERT(state->building_result); + term_size = 0; + + /* Free block histogram. */ + term_size += 1 + state->hist_slot_count; + + /* Per-type block list. */ + for (ix = ERTS_ALC_A_MIN; ix <= ERTS_ALC_A_MAX; ix++) { + UWord count = info->alloc_count[ix - ERTS_ALC_A_MIN]; + UWord size = info->alloc_size[ix - ERTS_ALC_A_MIN]; + + if (count > 0) { + /* We have at least one block of this type, so we'll need a cons + * cell and a 3-tuple; {Type, Count, Size}. */ + term_size += 2 + 4; + term_size += IS_USMALL(0, count) ? 0 : BIG_UINT_HEAP_SIZE; + term_size += IS_USMALL(0, size) ? 0 : BIG_UINT_HEAP_SIZE; + } + } - term_size = 11 + state->hist_slot_count; + /* Carrier tuple and its fields. */ + term_size += 7; term_size += IS_USMALL(0, info->carrier_size) ? 0 : BIG_UINT_HEAP_SIZE; term_size += IS_USMALL(0, info->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE; - term_size += IS_USMALL(0, info->allocated_size) ? 0 : BIG_UINT_HEAP_SIZE; + /* ... and a finally a cons cell to keep the result in. */ + term_size += 2; + + /* * * */ hp = erts_produce_heap(&state->msg_factory, term_size, 0); - hp[0] = make_arityval(state->hist_slot_count); + block_list = NIL; + for (ix = ERTS_ALC_A_MIN; ix <= ERTS_ALC_A_MAX; ix++) { + UWord count = info->alloc_count[ix - ERTS_ALC_A_MIN]; + UWord size = info->alloc_size[ix - ERTS_ALC_A_MIN]; - for (ix = 0; ix < state->hist_slot_count; ix++) { - hp[1 + ix] = make_small(info->histogram[ix]); + if (count > 0) { + Eterm block_count, block_size; + Eterm alloc_tuple; + + block_count = bld_unstable_uint(&hp, NULL, count); + block_size = bld_unstable_uint(&hp, NULL, size); + + hp[0] = make_arityval(3); + hp[1] = alloc_num_atoms[ix]; + hp[2] = block_count; + hp[3] = block_size; + + alloc_tuple = make_tuple(hp); + hp += 4; + + block_list = CONS(hp, alloc_tuple, block_list); + hp += 2; + } } + hp[0] = make_arityval(state->hist_slot_count); + for (ix = 0; ix < state->hist_slot_count; ix++) { + hp[1 + ix] = make_small(info->free_histogram[ix]); + } histogram_tuple = make_tuple(hp); hp += 1 + state->hist_slot_count; carrier_size = bld_unstable_uint(&hp, NULL, info->carrier_size); unscanned_size = bld_unstable_uint(&hp, NULL, info->unscanned_size); - allocated_size = bld_unstable_uint(&hp, NULL, info->allocated_size); - hp[0] = make_arityval(7); - hp[1] = state->allocator_desc; - hp[2] = carrier_size; - hp[3] = unscanned_size; - hp[4] = allocated_size; - hp[5] = make_small(info->allocated_count); - hp[6] = (info->flags & ERTS_CRR_ALCTR_FLG_IN_POOL) ? am_true : am_false; - hp[7] = histogram_tuple; + hp[0] = make_arityval(6); + hp[1] = alloc_num_atoms[state->allocator_number]; + hp[2] = (info->flags & ERTS_CRR_ALCTR_FLG_IN_POOL) ? am_true : am_false; + hp[3] = carrier_size; + hp[4] = unscanned_size; + hp[5] = block_list; + hp[6] = histogram_tuple; carrier_tuple = make_tuple(hp); - hp += 8; + hp += 7; state->result_list = CONS(hp, carrier_tuple, state->result_list); @@ -7936,8 +8132,6 @@ int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, { gather_cinfo_t *gather_state; blockscan_t *scanner; - - const char *allocator_desc; Allctr_t *allocator; ASSERT(is_internal_ref(ref)); @@ -7948,7 +8142,7 @@ int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, return 0; } - allocator_desc = ERTS_ALC_A2AD(allocator_num); + ensure_atoms_initialized(allocator); /* Plain calloc is intentional. */ gather_state = (gather_cinfo_t*)calloc(1, sizeof(gather_cinfo_t)); @@ -7959,9 +8153,7 @@ int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, scanner->finish = gather_cinfo_finish; scanner->user_data = gather_state; - gather_state->allocator_desc = erts_atom_put((byte *)allocator_desc, - sys_strlen(allocator_desc), - ERTS_ATOM_ENC_LATIN1, 1); + gather_state->allocator_number = allocator_num; erts_iref_storage_save(&gather_state->iref, ref); gather_state->hist_slot_start = hist_start * 2; gather_state->hist_slot_count = hist_width; @@ -8069,14 +8261,22 @@ void erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; + int ix; - no = allctr->sbcs.curr.norm.mseg.no; - no += allctr->sbcs.curr.norm.sys_alloc.no; - no += allctr->mbcs.blocks.curr.no; + no = 0; + for (ix = ERTS_CRR_ALLOC_MIN; ix <= ERTS_CRR_ALLOC_MAX; ix++) { + no += allctr->sbcs.carriers[ix].no; + } + + ix = allctr->alloc_no - ERTS_ALC_A_MIN; + no += allctr->mbcs.blocks[ix].curr.no; if (no) { - UWord sz = allctr->sbcs.blocks.curr.size; - sz += allctr->mbcs.blocks.curr.size; + UWord sz = 0; + + sz += allctr->sbcs.blocks[ix].curr.size; + sz += allctr->mbcs.blocks[ix].curr.size; + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index b46b311c59..07cbd8470e 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -456,8 +456,8 @@ typedef struct { ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; - UWord blocks[ERTS_ALC_A_MAX + 1]; - UWord blocks_size[ERTS_ALC_A_MAX + 1]; + UWord blocks[ERTS_ALC_A_COUNT]; + UWord blocks_size[ERTS_ALC_A_COUNT]; UWord total_blocks_size; enum { ERTS_MBC_IS_HOME, @@ -492,19 +492,28 @@ typedef struct { } StatValues_t; typedef struct { - union { - struct { - StatValues_t mseg; - StatValues_t sys_alloc; - } norm; - } curr; - StatValues_t max; - StatValues_t max_ever; - struct { - StatValues_t curr; - StatValues_t max; - StatValues_t max_ever; - } blocks; + StatValues_t curr; + StatValues_t max; + StatValues_t max_ever; +} BlockStats_t; + +enum { + ERTS_CRR_ALLOC_MIN = 0, + + ERTS_CRR_ALLOC_MSEG = ERTS_CRR_ALLOC_MIN, + ERTS_CRR_ALLOC_SYS = 1, + + ERTS_CRR_ALLOC_MAX, + ERTS_CRR_ALLOC_COUNT = ERTS_CRR_ALLOC_MAX + 1 +}; + +typedef struct { + StatValues_t carriers[ERTS_CRR_ALLOC_COUNT]; + + StatValues_t max; + StatValues_t max_ever; + + BlockStats_t blocks[ERTS_ALC_A_COUNT]; } CarriersStats_t; #ifdef USE_LTTNG_VM_TRACEPOINTS @@ -654,8 +663,8 @@ struct Allctr_t_ { UWord in_pool_limit; /* acnl */ UWord fblk_min_limit; /* acmfl */ struct { - erts_atomic_t blocks_size[ERTS_ALC_A_MAX + 1]; - erts_atomic_t no_blocks[ERTS_ALC_A_MAX + 1]; + erts_atomic_t blocks_size[ERTS_ALC_A_COUNT]; + erts_atomic_t no_blocks[ERTS_ALC_A_COUNT]; erts_atomic_t carriers_size; erts_atomic_t no_carriers; CallCounter_t fail_pooled; diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 44655ad5df..d160cda4df 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -254,25 +254,6 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif erts_thr_q_enqueue(&q->thr_q, a); -#ifdef USE_LTTNG_VM_TRACEPOINTS - if (LTTNG_ENABLED(aio_pool_put)) { - lttng_decl_portbuf(port_str); - lttng_portid_to_str(a->port, port_str); - LTTNG2(aio_pool_put, port_str, -1); - } -#endif -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(aio_pool_add)) { - DTRACE_CHARBUF(port_str, 16); - - erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), - "%T", a->port); - /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */ - len = -1; - DTRACE2(aio_pool_add, port_str, len); - } - gcc_optimizer_hack++; -#endif } static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, @@ -293,25 +274,6 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq); if (saved_fin_deq) erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); -#ifdef USE_LTTNG_VM_TRACEPOINTS - if (LTTNG_ENABLED(aio_pool_get)) { - lttng_decl_portbuf(port_str); - int length = erts_thr_q_length_dirty(q); - lttng_portid_to_str(a->port, port_str); - LTTNG2(aio_pool_get, port_str, length); - } -#endif -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(aio_pool_get)) { - DTRACE_CHARBUF(port_str, 16); - - erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), - "%T", a->port); - /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */ - len = -1; - DTRACE2(aio_pool_get, port_str, len); - } -#endif return a; } diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index b8e56390c1..03b8e0e632 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -2413,7 +2413,7 @@ HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_binary_list_to_bin_1]); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, &bif_trap_export[BIF_binary_list_to_bin_1]); } typedef struct { diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index cce8472ccb..4cab9bec1d 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -327,7 +327,7 @@ crc32_1(BIF_ALIST_1) res_sum = erts_make_integer(chksum,BIF_P); if (rest != NIL) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_crc32_2], BIF_P, res_sum, rest); + BIF_TRAP2(&bif_trap_export[BIF_crc32_2], BIF_P, res_sum, rest); } BIF_RET(res_sum); } @@ -354,7 +354,7 @@ crc32_2(BIF_ALIST_2) res_sum = erts_make_integer(chksum,BIF_P); if (rest != NIL) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_crc32_2], BIF_P, res_sum, rest); + BIF_TRAP2(&bif_trap_export[BIF_crc32_2], BIF_P, res_sum, rest); } BIF_RET(res_sum); } @@ -407,7 +407,7 @@ adler32_1(BIF_ALIST_1) res_sum = erts_make_integer(chksum,BIF_P); if (rest != NIL) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_adler32_2], BIF_P, res_sum, rest); + BIF_TRAP2(&bif_trap_export[BIF_adler32_2], BIF_P, res_sum, rest); } BIF_RET(res_sum); } @@ -434,7 +434,7 @@ adler32_2(BIF_ALIST_2) res_sum = erts_make_integer(chksum,BIF_P); if (rest != NIL) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_adler32_2], BIF_P, res_sum, rest); + BIF_TRAP2(&bif_trap_export[BIF_adler32_2], BIF_P, res_sum, rest); } BIF_RET(res_sum); } @@ -575,7 +575,7 @@ md5_update_2(BIF_ALIST_2) bin = new_binary(BIF_P, (byte *) &context, sizeof(MD5_CTX)); if (rest != NIL) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_md5_update_2], BIF_P, bin, rest); + BIF_TRAP2(&bif_trap_export[BIF_md5_update_2], BIF_P, bin, rest); } BUMP_REDS(BIF_P,res); BIF_RET(bin); diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 09757e473b..053797bc89 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -239,7 +239,7 @@ static BIF_RETTYPE erlang_length_trap(BIF_ALIST_3) * Signal an error. The original argument was tucked away in BIF_ARG_3. */ ERTS_BIF_ERROR_TRAPPED1(BIF_P, BIF_P->freason, - bif_export[BIF_length_1], BIF_ARG_3); + &bif_trap_export[BIF_length_1], BIF_ARG_3); } } } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 1064c89d84..26020ef5b2 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -158,8 +158,9 @@ static Eterm os_version_tuple; static Eterm current_function(Process* p, ErtsHeapFactory *hfact, Process* rp, int full_info, Uint reserve_size, int flags); -static Eterm current_stacktrace(ErtsHeapFactory *hfact, Process* rp, - Uint reserve_size); +static Eterm +current_stacktrace(Process* p, ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size, int flags); Eterm erts_bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh, Eterm tail) @@ -548,6 +549,8 @@ static int collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp, Sint reds) case ERTS_MON_TYPE_PORT: case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_TIME_OFFSET: + if (mon->flags & ERTS_ML_FLG_SPAWN_PENDING) + break; /* Not an active monitor... */ if (!(mon->flags & ERTS_ML_FLG_NAME)) { micp->mi[micp->mi_i].named = 0; micp->mi[micp->mi_i].entity.term = mon->other.item; @@ -1249,9 +1252,9 @@ exited: yield: if (pi2) - ERTS_BIF_PREP_YIELD2(ret, bif_export[BIF_process_info_2], c_p, pid, opt); + ERTS_BIF_PREP_YIELD2(ret, &bif_trap_export[BIF_process_info_2], c_p, pid, opt); else - ERTS_BIF_PREP_YIELD1(ret, bif_export[BIF_process_info_1], c_p, pid); + ERTS_BIF_PREP_YIELD1(ret, &bif_trap_export[BIF_process_info_1], c_p, pid); goto done; send_signal: { @@ -1384,7 +1387,7 @@ process_info_aux(Process *c_p, break; case ERTS_PI_IX_CURRENT_STACKTRACE: - res = current_stacktrace(hfact, rp, reserve_size); + res = current_stacktrace(c_p, hfact, rp, reserve_size, flags); break; case ERTS_PI_IX_INITIAL_CALL: @@ -2022,19 +2025,23 @@ current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp, } if (c_p == rp && !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) { - FunctionInfo fi2; + BeamInstr* return_address; + FunctionInfo caller_fi; - /* - * The current function is erlang:process_info/{1,2}, - * which is not the answer that the application want. - * We will use the function pointed into by rp->cp - * instead if it can be looked up. - */ - erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.mfa) { - fi = fi2; - rp->current = fi2.mfa; - } + /* + * The current function is erlang:process_info/{1,2}, and we've + * historically returned the *calling* function in that case. We + * therefore use the continuation pointer stored at the top of the + * stack instead, which is safe since process_info is a "heavy" BIF + * that is only called through its export entry. + */ + return_address = erts_printable_return_address(rp, STACK_TOP(rp)); + + erts_lookup_function_info(&caller_fi, return_address, full_info); + if (caller_fi.mfa) { + fi = caller_fi; + rp->current = caller_fi.mfa; + } } /* @@ -2055,8 +2062,8 @@ current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp, } static Eterm -current_stacktrace(ErtsHeapFactory *hfact, Process* rp, - Uint reserve_size) +current_stacktrace(Process *p, ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size, int flags) { Uint sz; struct StackTrace* s; @@ -2073,13 +2080,14 @@ current_stacktrace(ErtsHeapFactory *hfact, Process* rp, sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); s->depth = 0; - if (depth > 0 && rp->i) { - s->trace[s->depth++] = rp->i; - depth--; - } - if (depth > 0 && rp->cp != 0) { - s->trace[s->depth++] = rp->cp - 1; - depth--; + s->pc = NULL; + + /* We skip current pc when requesting our own stack trace since it will + * inevitably point to process_info/1,2 */ + if ((p != rp || (flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) && + depth > 0 && rp->i) { + s->trace[s->depth++] = rp->i; + depth--; } erts_save_stacktrace(rp, s, depth); @@ -2814,7 +2822,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (BIF_ARG_1 == am_threads) { return am_true; } else if (BIF_ARG_1 == am_creation) { - return make_small(erts_this_node->creation); + Uint hsz = 0; + erts_bld_uint(NULL, &hsz, erts_this_node->creation); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + BIF_RET(erts_bld_uint(&hp, NULL, erts_this_node->creation)); } else if (BIF_ARG_1 == am_break_ignored) { extern int ignore_break; if (ignore_break) @@ -2825,7 +2836,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) /* Arguments that are unusual follow ... */ else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) { int no; - erts_get_logical_processors(&no, NULL, NULL); + erts_get_logical_processors(&no, NULL, NULL, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2835,7 +2846,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) { int no; - erts_get_logical_processors(NULL, &no, NULL); + erts_get_logical_processors(NULL, &no, NULL, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2845,7 +2856,17 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) { int no; - erts_get_logical_processors(NULL, NULL, &no); + erts_get_logical_processors(NULL, NULL, &no, NULL); + if (no > 0) + BIF_RET(make_small((Uint) no)); + else { + DECL_AM(unknown); + BIF_RET(AM_unknown); + } + } + else if (ERTS_IS_ATOM_STR("cpu_quota", BIF_ARG_1)) { + int no; + erts_get_logical_processors(NULL, NULL, NULL, &no); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -4043,7 +4064,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_notsup); #endif } - + else if (ERTS_IS_ATOM_STR("flxctr_memory_usage", BIF_ARG_1)) { + Sint mem = erts_flxctr_debug_memory_usage(); + if (mem == -1) { + BIF_RET(am_notsup); + } else { + Uint hsz = BIG_UWORD_HEAP_SIZE((UWord)mem); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(uword_to_big((UWord)mem, hp)); + } + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -4254,9 +4284,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("term_to_binary_tuple_fallbacks", tp[1])) { - Uint dflags = (TERM_TO_BINARY_DFLAGS - & ~DFLAG_EXPORT_PTR_TAG - & ~DFLAG_BIT_BINARIES); + Uint64 dflags = (TERM_TO_BINARY_DFLAGS + & ~DFLAG_EXPORT_PTR_TAG + & ~DFLAG_BIT_BINARIES); Eterm res = erts_term_to_binary(BIF_P, tp[2], 0, dflags); if (is_value(res)) BIF_RET(res); @@ -4374,6 +4404,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) break; BIF_RET(res); } + else if (ERTS_IS_ATOM_STR("term_to_binary", tp[1])) { + return erts_debug_term_to_binary(BIF_P, tp[2], tp[3]); + } break; } default: @@ -4593,7 +4626,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (!flag && BIF_ARG_2 != am_false) { erts_atomic_set_nob(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2], + ERTS_BIF_YIELD2(&bif_trap_export[BIF_erts_debug_set_internal_state_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } erts_atomic_set_nob(&hipe_test_reschedule_flag, !flag); @@ -4639,9 +4672,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; else if (ERTS_IS_ATOM_STR("aux_work", BIF_ARG_2)) flag = ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK; + else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_2)) + flag = ERTS_DEBUG_WAIT_COMPLETED_THREAD_PROGRESS; - if (flag && erts_debug_wait_completed(BIF_P, flag)) { - ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + if (flag) { + if (erts_debug_wait_completed(BIF_P, flag)) + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + else + BIF_ERROR(BIF_P, SYSTEM_LIMIT); } } else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) { @@ -4720,7 +4758,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) else if (ERTS_IS_ATOM_STR("ets_debug_random_split_join", BIF_ARG_1)) { if (is_tuple(BIF_ARG_2)) { Eterm* tpl = tuple_val(BIF_ARG_2); - if (erts_ets_debug_random_split_join(tpl[1], tpl[2] == am_true)) BIF_RET(am_ok); } @@ -4735,11 +4772,225 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_P->mbuf_sz += sz; BIF_RET(copy); } + else if (ERTS_IS_ATOM_STR("remove_hopefull_dflags", BIF_ARG_1)) { + int old_val, new_val; + + switch (BIF_ARG_2) { + case am_true: new_val = !0; break; + case am_false: new_val = 0; break; + default: BIF_ERROR(BIF_P, BADARG); break; + } + + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); + + old_val = erts_dflags_test_remove_hopefull_flags; + erts_dflags_test_remove_hopefull_flags = new_val; + + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BIF_RET(old_val ? am_true : am_false); + } } BIF_ERROR(BIF_P, BADARG); } +Eterm +erts_get_ethread_info(Process *c_p) +{ + Uint sz, *szp; + Eterm res, *hp, **hpp, *end_hp = NULL; + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + Eterm tup, list, name; +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + char buf[1024]; + int i; + char **str; +#endif + + res = NIL; + +#ifdef ETHR_X86_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "sse2"), +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_bld_string(hpp, szp, + (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")) +#else + erts_bld_string(hpp, szp, "yes") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, + "x86" +#ifdef ARCH_64 + "_64" +#endif + " OOO"), + erts_bld_string(hpp, szp, +#ifdef ETHR_X86_OUT_OF_ORDER + "yes" +#else + "no" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); +#endif + +#ifdef ETHR_SPARC_V9_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Sparc V9"), + erts_bld_string(hpp, szp, +#if defined(ETHR_SPARC_TSO) + "TSO" +#elif defined(ETHR_SPARC_PSO) + "PSO" +#elif defined(ETHR_SPARC_RMO) + "RMO" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + +#ifdef ETHR_PPC_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "lwsync"), + erts_bld_string(hpp, szp, +#if defined(ETHR_PPC_HAVE_LWSYNC) + "yes" +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + "no" +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native rw-spinlocks"), +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native spinlocks"), +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + + list = NIL; +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + } + else +#endif + name = erts_bld_string(hpp, szp, "no"); + + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "Double word native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "64-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "32-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + if (hpp) { + HRelease(c_p, end_hp, *hpp) + return res; + } + + hp = HAlloc(c_p, sz); + end_hp = hp + sz; + hpp = &hp; + szp = NULL; + } +} + static BIF_RETTYPE gather_histograms_helper(Process * c_p, Eterm arg_tuple, int gather(Process *, int, int, int, UWord, Eterm)) diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index fa2edfef1e..40512a117d 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -32,8 +32,7 @@ #include "bif.h" #include "erl_binary.h" - -static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); +static Eterm keyfind(Export* Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); /* erlang:'++'/2 * @@ -308,12 +307,12 @@ static Eterm append(Export *bif_entry, BIF_ALIST_2) { Eterm ebif_plusplus_2(BIF_ALIST_2) { - return append(bif_export[BIF_ebif_plusplus_2], BIF_CALL_ARGS); + return append(&bif_trap_export[BIF_ebif_plusplus_2], BIF_CALL_ARGS); } BIF_RETTYPE append_2(BIF_ALIST_2) { - return append(bif_export[BIF_append_2], BIF_CALL_ARGS); + return append(&bif_trap_export[BIF_append_2], BIF_CALL_ARGS); } /* erlang:'--'/2 @@ -1039,11 +1038,11 @@ static Eterm subtract(Export *bif_entry, BIF_ALIST_2) { } BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) { - return subtract(bif_export[BIF_ebif_minusminus_2], BIF_CALL_ARGS); + return subtract(&bif_trap_export[BIF_ebif_minusminus_2], BIF_CALL_ARGS); } BIF_RETTYPE subtract_2(BIF_ALIST_2) { - return subtract(bif_export[BIF_subtract_2], BIF_CALL_ARGS); + return subtract(&bif_trap_export[BIF_subtract_2], BIF_CALL_ARGS); } @@ -1068,7 +1067,7 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2) while (is_list(list)) { if (--max_iter < 0) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_lists_member_2], BIF_P, term, list); + BIF_TRAP2(&bif_trap_export[BIF_lists_member_2], BIF_P, term, list); } item = CAR(list_val(list)); if ((item == term) || (non_immed_key && eq(item, term))) { @@ -1130,7 +1129,7 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p, } ASSERT(is_list(tail) && cells_left == 0); - BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); + BIF_TRAP2(&bif_trap_export[BIF_lists_reverse_2], c_p, list, tail); } static BIF_RETTYPE lists_reverse_onheap(Process *c_p, @@ -1179,7 +1178,7 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p, } BUMP_ALL_REDS(c_p); - BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); + BIF_TRAP2(&bif_trap_export[BIF_lists_reverse_2], c_p, list, tail); } BIF_ERROR(c_p, BADARG); @@ -1209,7 +1208,7 @@ lists_keymember_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keymember_3, BIF_P, + res = keyfind(&bif_trap_export[BIF_lists_keymember_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_value(res) && is_tuple(res)) { return am_true; @@ -1223,7 +1222,7 @@ lists_keysearch_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keysearch_3, BIF_P, + res = keyfind(&bif_trap_export[BIF_lists_keysearch_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_non_value(res) || is_not_tuple(res)) { return res; @@ -1236,12 +1235,12 @@ lists_keysearch_3(BIF_ALIST_3) BIF_RETTYPE lists_keyfind_3(BIF_ALIST_3) { - return keyfind(BIF_lists_keyfind_3, BIF_P, + return keyfind(&bif_trap_export[BIF_lists_keyfind_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } static Eterm -keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) +keyfind(Export *Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) { int max_iter = 10 * CONTEXT_REDS; Sint pos; @@ -1257,7 +1256,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); @@ -1282,7 +1281,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); @@ -1300,7 +1299,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c index 9dc5c66a0a..2610833a33 100644 --- a/erts/emulator/beam/erl_bif_persistent.c +++ b/erts/emulator/beam/erl_bif_persistent.c @@ -45,23 +45,45 @@ #define MUST_SHRINK(t) (((Uint)200) * t->num_entries <= LOAD_FACTOR * t->allocated && \ t->allocated > INITIAL_SIZE) + +typedef struct delete_op { + enum { DELETE_OP_TUPLE, DELETE_OP_TABLE } type; + struct delete_op* next; + ErtsThrPrgrLaterOp thr_prog_op; + int is_scheduled; +} DeleteOp; + typedef struct hash_table { Uint allocated; Uint num_entries; Uint mask; Uint first_to_delete; Uint num_to_delete; - erts_atomic_t refc; - struct hash_table* delete_next; - ErtsThrPrgrLaterOp thr_prog_op; - Eterm term[1]; + DeleteOp delete_op; + erts_atomic_t term[1]; } HashTable; +static ERTS_INLINE Eterm get_bucket(HashTable* tab, Uint idx) +{ + return (Eterm) erts_atomic_read_nob(&tab->term[idx]); +} + +static ERTS_INLINE void set_bucket(HashTable* tab, Uint idx, Eterm term) +{ + erts_atomic_set_nob(&tab->term[idx], (erts_aint_t)term); +} + +static ERTS_INLINE Uint sizeof_HashTable(Uint sz) +{ + return offsetof(HashTable, term) + (sz * sizeof(erts_atomic_t)); +} + typedef struct trap_data { HashTable* table; Uint idx; Uint remaining; Uint memory; /* Used by info/0 to count used memory */ + int got_update_permission; } TrapData; typedef enum { @@ -89,8 +111,7 @@ typedef struct { } ErtsPersistentTermCpyTableCtx; typedef enum { - PUT2_TRAP_LOCATION_NEW_KEY, - PUT2_TRAP_LOCATION_REPLACE_VALUE + PUT2_TRAP_LOCATION_NEW_KEY } ErtsPersistentTermPut2TrapLocation; typedef struct { @@ -117,6 +138,7 @@ typedef struct { Uint entry_index; Eterm old_term; HashTable* tmp_table; + int must_shrink; ErtsPersistentTermCpyTableCtx cpy_ctx; } ErtsPersistentTermErase1Context; @@ -126,20 +148,21 @@ typedef struct { static HashTable* create_initial_table(void); static Uint lookup(HashTable* hash_table, Eterm key); +static int is_erasable(HashTable* hash_table, Uint idx); static HashTable* copy_table(ErtsPersistentTermCpyTableCtx* ctx); static int try_seize_update_permission(Process* c_p); static void release_update_permission(int release_updater); static void table_updater(void* table); -static void table_deleter(void* hash_table); -static void dec_table_refc(Process* c_p, HashTable* old_table); -static void delete_table(Process* c_p, HashTable* table); +static void scheduled_deleter(void* delete_op); +static void delete_table(HashTable* table); +static void delete_tuple(Eterm term); static void mark_for_deletion(HashTable* hash_table, Uint entry_index); static ErtsLiteralArea* term_to_area(Eterm tuple); static void suspend_updater(Process* c_p); static Eterm do_get_all(Process* c_p, TrapData* trap_data, Eterm res); static Eterm do_info(Process* c_p, TrapData* trap_data); -static void append_to_delete_queue(HashTable* table); -static HashTable* next_to_delete(void); +static void append_to_delete_queue(DeleteOp*); +static DeleteOp* list_to_delete(DeleteOp*); static Eterm alloc_trap_data(Process* c_p); static int cleanup_trap_data(Binary *bp); @@ -174,13 +197,16 @@ static Process* updater_process = NULL; /* Protected by update_table_permission_mtx */ static ErtsThrPrgrLaterOp thr_prog_op; +static Uint fast_update_index; +static Eterm fast_update_term = THE_NON_VALUE; + /* * Queue of hash tables to be deleted. */ static erts_mtx_t delete_queue_mtx; -static HashTable* delete_queue_head = NULL; -static HashTable** delete_queue_tail = &delete_queue_head; +static DeleteOp* delete_queue_head = NULL; +static DeleteOp** delete_queue_tail = &delete_queue_head; /* * The following variables are only used during crash dumping. They @@ -284,7 +310,7 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) long iterations_until_trap; long max_iterations; #define PUT_TRAP_CODE \ - BIF_TRAP2(bif_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2) + BIF_TRAP2(&bif_trap_export[BIF_persistent_term_put_2], BIF_P, state_mref, BIF_ARG_2) #define TRAPPING_COPY_TABLE_PUT(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, COPY_TYPE, LOC_NAME, PUT_TRAP_CODE) @@ -306,12 +332,8 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) ctx = ERTS_MAGIC_BIN_DATA(state_bin); ASSERT(BIF_P->flags & F_DISABLE_GC); erts_set_gc_state(BIF_P, 1); - switch (ctx->trap_location) { - case PUT2_TRAP_LOCATION_NEW_KEY: - goto L_PUT2_TRAP_LOCATION_NEW_KEY; - case PUT2_TRAP_LOCATION_REPLACE_VALUE: - goto L_PUT2_TRAP_LOCATION_REPLACE_VALUE; - } + ASSERT(ctx->trap_location == PUT2_TRAP_LOCATION_NEW_KEY); + goto L_PUT2_TRAP_LOCATION_NEW_KEY; } else { /* Save state in magic bin in case trapping is necessary */ Eterm* hp; @@ -329,7 +351,7 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) if (!try_seize_update_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2], + ERTS_BIF_YIELD2(&bif_trap_export[BIF_persistent_term_put_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } ctx->hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); @@ -344,20 +366,19 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) ctx->heap[2] = ctx->term; ctx->tuple = make_tuple(ctx->heap); - if (is_nil(ctx->hash_table->term[ctx->entry_index])) { - Uint new_size = ctx->hash_table->allocated; + if (is_nil(get_bucket(ctx->hash_table, ctx->entry_index))) { if (MUST_GROW(ctx->hash_table)) { - new_size *= 2; + Uint new_size = ctx->hash_table->allocated * 2; + TRAPPING_COPY_TABLE_PUT(ctx->hash_table, + ctx->hash_table, + new_size, + ERTS_PERSISTENT_TERM_CPY_NO_REHASH, + PUT2_TRAP_LOCATION_NEW_KEY); + ctx->entry_index = lookup(ctx->hash_table, ctx->key); } - TRAPPING_COPY_TABLE_PUT(ctx->hash_table, - ctx->hash_table, - new_size, - ERTS_PERSISTENT_TERM_CPY_NO_REHASH, - PUT2_TRAP_LOCATION_NEW_KEY); - ctx->entry_index = lookup(ctx->hash_table, ctx->key); ctx->hash_table->num_entries++; } else { - Eterm tuple = ctx->hash_table->term[ctx->entry_index]; + Eterm tuple = get_bucket(ctx->hash_table, ctx->entry_index); Eterm old_term; ASSERT(is_tuple_arity(tuple, 2)); @@ -366,14 +387,6 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) /* Same value. No need to update anything. */ release_update_permission(0); BIF_RET(am_ok); - } else { - /* Mark the old term for deletion. */ - mark_for_deletion(ctx->hash_table, ctx->entry_index); - TRAPPING_COPY_TABLE_PUT(ctx->hash_table, - ctx->hash_table, - ctx->hash_table->allocated, - ERTS_PERSISTENT_TERM_CPY_NO_REHASH, - PUT2_TRAP_LOCATION_REPLACE_VALUE); } } @@ -402,8 +415,21 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) literal_area->off_heap = code_off_heap.first; DESTROY_SHCOPY(info); erts_set_literal_tag(&ctx->tuple, literal_area->start, term_size); - ctx->hash_table->term[ctx->entry_index] = ctx->tuple; + if (ctx->hash_table == (HashTable *) erts_atomic_read_nob(&the_hash_table)) { + /* Schedule fast update in active hash table */ + fast_update_index = ctx->entry_index; + fast_update_term = ctx->tuple; + } + else { + /* Do update in copied table */ + set_bucket(ctx->hash_table, ctx->entry_index, ctx->tuple); + } + + /* + * Now wait thread progress before making update visible to guarantee + * consistent view of table&term without memory barrier in every get/1. + */ erts_schedule_thr_prgr_later_op(table_updater, ctx->hash_table, &thr_prog_op); suspend_updater(BIF_P); } @@ -419,23 +445,25 @@ BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) Eterm magic_ref; Binary* mbp; + /* Prevent concurrent updates to get a consistent view */ + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD0(&bif_trap_export[BIF_persistent_term_get_0], BIF_P); + } + magic_ref = alloc_trap_data(BIF_P); mbp = erts_magic_ref2bin(magic_ref); trap_data = ERTS_MAGIC_BIN_DATA(mbp); trap_data->table = hash_table; trap_data->idx = 0; trap_data->remaining = hash_table->num_entries; + trap_data->got_update_permission = 1; res = do_get_all(BIF_P, trap_data, res); if (trap_data->remaining == 0) { + release_update_permission(0); + trap_data->got_update_permission = 0; BUMP_REDS(BIF_P, hash_table->num_entries); - trap_data->table = NULL; /* Prevent refc decrement */ BIF_RET(res); } else { - /* - * Increment the ref counter to prevent an update operation (by put/2 - * or erase/1) to delete this hash table. - */ - erts_atomic_inc_nob(&hash_table->refc); BUMP_ALL_REDS(BIF_P); BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res); } @@ -449,7 +477,7 @@ BIF_RETTYPE persistent_term_get_1(BIF_ALIST_1) Eterm term; entry_index = lookup(hash_table, key); - term = hash_table->term[entry_index]; + term = get_bucket(hash_table, entry_index); if (is_boxed(term)) { ASSERT(is_tuple_arity(term, 2)); BIF_RET(tuple_val(term)[2]); @@ -466,7 +494,7 @@ BIF_RETTYPE persistent_term_get_2(BIF_ALIST_2) Eterm term; entry_index = lookup(hash_table, key); - term = hash_table->term[entry_index]; + term = get_bucket(hash_table, entry_index); if (is_boxed(term)) { ASSERT(is_tuple_arity(term, 2)); result = tuple_val(term)[2]; @@ -507,7 +535,7 @@ BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(BIF_P); #endif #define ERASE_TRAP_CODE \ - BIF_TRAP1(bif_export[BIF_persistent_term_erase_1], BIF_P, state_mref); + BIF_TRAP1(&bif_trap_export[BIF_persistent_term_erase_1], BIF_P, state_mref); #define TRAPPING_COPY_TABLE_ERASE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME) \ TRAPPING_COPY_TABLE(TABLE_DEST, OLD_TABLE, NEW_SIZE, REHASH, LOC_NAME, ERASE_TRAP_CODE) if (is_internal_magic_ref(BIF_ARG_1) && @@ -542,58 +570,75 @@ BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1) ctx->tmp_table = NULL; } if (!try_seize_update_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1], + ERTS_BIF_YIELD1(&bif_trap_export[BIF_persistent_term_erase_1], BIF_P, BIF_ARG_1); } ctx->key = BIF_ARG_1; ctx->old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); ctx->entry_index = lookup(ctx->old_table, ctx->key); - ctx->old_term = ctx->old_table->term[ctx->entry_index]; + ctx->old_term = get_bucket(ctx->old_table, ctx->entry_index); if (is_boxed(ctx->old_term)) { - Uint new_size; - /* - * Since we don't use any delete markers, we must rehash - * the table when deleting terms to ensure that all terms - * can still be reached if there are hash collisions. - * We can't rehash in place and it would not be safe to modify - * the old table yet, so we will first need a new - * temporary table copy of the same size as the old one. - */ + ctx->must_shrink = MUST_SHRINK(ctx->old_table); + if (!ctx->must_shrink && is_erasable(ctx->old_table, ctx->entry_index)) { + /* + * Fast erase in active hash table. + * We schedule with thread progress even here (see put/2). + * It's not needed for read consistenty of the NIL word, BUT it's + * needed to guarantee sequential read consistenty of multiple + * updates. As we do thread progress between all updates, there is + * no risk seeing them out of order. + */ + fast_update_index = ctx->entry_index; + fast_update_term = NIL; + ctx->old_table->num_entries--; + erts_schedule_thr_prgr_later_op(table_updater, ctx->old_table, &thr_prog_op); + } + else { + Uint new_size; + /* + * Since we don't use any delete markers, we must rehash the table + * to ensure that all terms can still be reached if there are + * hash collisions. + * We can't rehash in place and it would not be safe to modify + * the old table yet, so we will first need a new + * temporary table copy of the same size as the old one. + */ - ASSERT(is_tuple_arity(ctx->old_term, 2)); - TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table, - ctx->old_table, - ctx->old_table->allocated, - ERTS_PERSISTENT_TERM_CPY_TEMP, - ERASE1_TRAP_LOCATION_TMP_COPY); + ASSERT(is_tuple_arity(ctx->old_term, 2)); + TRAPPING_COPY_TABLE_ERASE(ctx->tmp_table, + ctx->old_table, + ctx->old_table->allocated, + ERTS_PERSISTENT_TERM_CPY_TEMP, + ERASE1_TRAP_LOCATION_TMP_COPY); - /* - * Delete the term from the temporary table. Then copy the - * temporary table to a new table, rehashing the entries - * while copying. - */ + /* + * Delete the term from the temporary table. Then copy the + * temporary table to a new table, rehashing the entries + * while copying. + */ - ctx->tmp_table->term[ctx->entry_index] = NIL; - ctx->tmp_table->num_entries--; - new_size = ctx->tmp_table->allocated; - if (MUST_SHRINK(ctx->tmp_table)) { - new_size /= 2; - } - TRAPPING_COPY_TABLE_ERASE(ctx->new_table, - ctx->tmp_table, - new_size, - ERTS_PERSISTENT_TERM_CPY_REHASH, - ERASE1_TRAP_LOCATION_FINAL_COPY); - erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table); - /* - * IMPORTANT: Memory management depends on that ctx->tmp_table - * is set to NULL on the line below - */ - ctx->tmp_table = NULL; + set_bucket(ctx->tmp_table, ctx->entry_index, NIL); + ctx->tmp_table->num_entries--; + new_size = ctx->tmp_table->allocated; + if (ctx->must_shrink) { + new_size /= 2; + } + TRAPPING_COPY_TABLE_ERASE(ctx->new_table, + ctx->tmp_table, + new_size, + ERTS_PERSISTENT_TERM_CPY_REHASH, + ERASE1_TRAP_LOCATION_FINAL_COPY); + erts_free(ERTS_ALC_T_PERSISTENT_TERM_TMP, ctx->tmp_table); + /* + * IMPORTANT: Memory management depends on that ctx->tmp_table + * is set to NULL on the line below + */ + ctx->tmp_table = NULL; - mark_for_deletion(ctx->old_table, ctx->entry_index); - erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op); + mark_for_deletion(ctx->old_table, ctx->entry_index); + erts_schedule_thr_prgr_later_op(table_updater, ctx->new_table, &thr_prog_op); + } suspend_updater(BIF_P); BUMP_REDS(BIF_P, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); @@ -614,7 +659,7 @@ BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0) HashTable* new_table; if (!try_seize_update_permission(BIF_P)) { - ERTS_BIF_YIELD0(bif_export[BIF_erts_internal_erase_persistent_terms_0], + ERTS_BIF_YIELD0(&bif_trap_export[BIF_erts_internal_erase_persistent_terms_0], BIF_P); } old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); @@ -634,6 +679,11 @@ BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) Eterm magic_ref; Binary* mbp; + /* Prevent concurrent updates to get a consistent view */ + if (!try_seize_update_permission(BIF_P)) { + ERTS_BIF_YIELD0(&bif_trap_export[BIF_persistent_term_info_0], BIF_P); + } + magic_ref = alloc_trap_data(BIF_P); mbp = erts_magic_ref2bin(magic_ref); trap_data = ERTS_MAGIC_BIN_DATA(mbp); @@ -641,29 +691,19 @@ BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) trap_data->idx = 0; trap_data->remaining = hash_table->num_entries; trap_data->memory = 0; + trap_data->got_update_permission = 0; res = do_info(BIF_P, trap_data); if (trap_data->remaining == 0) { + release_update_permission(0); + trap_data->got_update_permission = 0; BUMP_REDS(BIF_P, hash_table->num_entries); - trap_data->table = NULL; /* Prevent refc decrement */ BIF_RET(res); } else { - /* - * Increment the ref counter to prevent an update operation (by put/2 - * or erase/1) to delete this hash table. - */ - erts_atomic_inc_nob(&hash_table->refc); BUMP_ALL_REDS(BIF_P); BIF_TRAP2(&persistent_term_info_export, BIF_P, magic_ref, res); } } -Uint -erts_persistent_term_count(void) -{ - HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); - return hash_table->num_entries; -} - void erts_init_persistent_dumping(void) { @@ -677,15 +717,15 @@ erts_init_persistent_dumping(void) */ erts_persistent_areas = (ErtsLiteralArea **) hash_table->term; - erts_num_persistent_areas = hash_table->num_entries; area_p = erts_persistent_areas; for (i = 0; i < hash_table->allocated; i++) { - Eterm term = hash_table->term[i]; + Eterm term = get_bucket(hash_table, i); if (is_boxed(term)) { *area_p++ = term_to_area(term); } } + erts_num_persistent_areas = area_p - erts_persistent_areas; } /* @@ -699,16 +739,14 @@ create_initial_table(void) int i; hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM, - sizeof(HashTable)+sizeof(Eterm) * - (INITIAL_SIZE-1)); + sizeof_HashTable(INITIAL_SIZE)); hash_table->allocated = INITIAL_SIZE; hash_table->num_entries = 0; hash_table->mask = INITIAL_SIZE-1; hash_table->first_to_delete = 0; hash_table->num_to_delete = 0; - erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1); for (i = 0; i < INITIAL_SIZE; i++) { - hash_table->term[i] = NIL; + erts_atomic_init_nob(&hash_table->term[i], NIL); } return hash_table; } @@ -731,12 +769,8 @@ persistent_term_get_all_trap(BIF_ALIST_2) BUMP_ALL_REDS(BIF_P); BIF_TRAP2(&persistent_term_get_all_export, BIF_P, BIF_ARG_1, res); } else { - /* - * Decrement ref count (and possibly delete the hash table - * and associated literal area). - */ - dec_table_refc(BIF_P, trap_data->table); - trap_data->table = NULL; /* Prevent refc decrement */ + release_update_permission(0); + trap_data->got_update_permission = 0; BUMP_REDS(BIF_P, bump_reds); BIF_RET(res); } @@ -774,7 +808,9 @@ do_get_all(Process* c_p, TrapData* trap_data, Eterm res) i = 0; heap_size = (2 + 3) * remaining; while (remaining != 0) { - Eterm term = hash_table->term[idx]; + Eterm term; + ASSERT(idx < hash_table->allocated); + term = get_bucket(hash_table, idx); if (is_tuple(term)) { Uint key_size; Eterm* tup_val; @@ -829,12 +865,8 @@ persistent_term_info_trap(BIF_ALIST_1) BUMP_ALL_REDS(BIF_P); BIF_TRAP1(&persistent_term_info_export, BIF_P, BIF_ARG_1); } else { - /* - * Decrement ref count (and possibly delete the hash table - * and associated literal area). - */ - dec_table_refc(BIF_P, trap_data->table); - trap_data->table = NULL; /* Prevent refc decrement */ + release_update_permission(0); + trap_data->got_update_permission = 0; BUMP_REDS(BIF_P, bump_reds); ASSERT(is_map(res)); BIF_RET(res); @@ -861,9 +893,9 @@ do_info(Process* c_p, TrapData* trap_data) remaining = trap_data->remaining < max_iter ? trap_data->remaining : max_iter; trap_data->remaining -= remaining; while (remaining != 0) { - if (is_boxed(hash_table->term[idx])) { + if (is_boxed(get_bucket(hash_table, idx))) { ErtsLiteralArea* area; - area = term_to_area(hash_table->term[idx]); + area = term_to_area(get_bucket(hash_table, idx)); trap_data->memory += sizeof(ErtsLiteralArea) + sizeof(Eterm) * (area->end - area->start - 1); remaining--; @@ -911,13 +943,8 @@ cleanup_trap_data(Binary *bp) { TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp); - if (trap_data->table) { - /* - * The process has been killed and is now exiting. - * Decrement the reference counter for the table. - */ - dec_table_refc(NULL, trap_data->table); - } + if (trap_data->got_update_permission) + release_update_permission(0); return 1; } @@ -925,17 +952,26 @@ static Uint lookup(HashTable* hash_table, Eterm key) { Uint mask = hash_table->mask; - Eterm* table = hash_table->term; Uint32 idx = make_internal_hash(key, 0); Eterm term; - do { + while (1) { + term = get_bucket(hash_table, idx & mask); + if (is_nil(term) || EQ(key, (tuple_val(term))[1])) + break; idx++; - term = table[idx & mask]; - } while (is_boxed(term) && !EQ(key, (tuple_val(term))[1])); + } return idx & mask; } +static int +is_erasable(HashTable* hash_table, Uint idx) +{ + /* It's ok to erase [idx] if it's not a stepping stone to [idx+1] */ + return get_bucket(hash_table, (idx + 1) & hash_table->mask) == NIL; +} + + static HashTable* copy_table(ErtsPersistentTermCpyTableCtx* ctx) { @@ -956,8 +992,7 @@ copy_table(ErtsPersistentTermCpyTableCtx* ctx) alloc_type = ERTS_ALC_T_PERSISTENT_TERM; } ctx->new_table = (HashTable *) erts_alloc(alloc_type, - sizeof(HashTable) + - sizeof(Eterm) * (ctx->new_size-1)); + sizeof_HashTable(ctx->new_size)); if (ctx->old_table->allocated == ctx->new_size && (ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_NO_REHASH || ctx->copy_type == ERTS_PERSISTENT_TERM_CPY_TEMP)) { @@ -970,7 +1005,8 @@ copy_table(ErtsPersistentTermCpyTableCtx* ctx) i < MIN(ctx->iterations_done + ctx->max_iterations, ctx->new_size); i++) { - ctx->new_table->term[i] = ctx->old_table->term[i]; + erts_atomic_init_nob(&ctx->new_table->term[i], + erts_atomic_read_nob(&ctx->old_table->term[i])); } ctx->total_iterations_done = (i - ctx->iterations_done); if (i < ctx->new_size) { @@ -993,7 +1029,7 @@ copy_table(ErtsPersistentTermCpyTableCtx* ctx) i < MIN(ctx->iterations_done + ctx->max_iterations, ctx->new_size); i++) { - ctx->new_table->term[i] = NIL; + erts_atomic_init_nob(&ctx->new_table->term[i], (erts_aint_t)NIL); } ctx->total_iterations_done = (i - ctx->iterations_done); ctx->max_iterations -= ctx->total_iterations_done; @@ -1008,11 +1044,12 @@ copy_table(ErtsPersistentTermCpyTableCtx* ctx) i < MIN(ctx->iterations_done + ctx->max_iterations, old_size); i++) { - if (is_tuple(ctx->old_table->term[i])) { - Eterm key = tuple_val(ctx->old_table->term[i])[1]; + Eterm term = get_bucket(ctx->old_table, i); + if (is_tuple(term)) { + Eterm key = tuple_val(term)[1]; Uint entry_index = lookup(ctx->new_table, key); - ASSERT(is_nil(ctx->new_table->term[entry_index])); - ctx->new_table->term[entry_index] = ctx->old_table->term[i]; + ASSERT(is_nil(get_bucket(ctx->new_table, entry_index))); + set_bucket(ctx->new_table, entry_index, term); } } ctx->total_iterations_done += (i - ctx->iterations_done); @@ -1025,7 +1062,6 @@ copy_table(ErtsPersistentTermCpyTableCtx* ctx) } ctx->new_table->first_to_delete = 0; ctx->new_table->num_to_delete = 0; - erts_atomic_init_nob(&ctx->new_table->refc, (erts_aint_t)1); { HashTable* new_table = ctx->new_table; /* @@ -1052,47 +1088,118 @@ term_to_area(Eterm tuple) offsetof(ErtsLiteralArea, start)); } +typedef struct { + Eterm term; + ErtsLiteralArea* area; + DeleteOp delete_op; +} OldLiteral; + +static OldLiteral* alloc_old_literal(void) +{ + return erts_alloc(ERTS_ALC_T_RELEASE_LAREA, sizeof(OldLiteral)); +} + +static void free_old_literal(OldLiteral* olp) +{ + return erts_free(ERTS_ALC_T_RELEASE_LAREA, olp); +} + static void table_updater(void* data) { HashTable* old_table; HashTable* new_table; + UWord cleanup_bytes; old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); new_table = (HashTable *) data; - ASSERT(new_table->num_to_delete == 0); - erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table); - append_to_delete_queue(old_table); - erts_schedule_thr_prgr_later_op(table_deleter, - old_table, - &old_table->thr_prog_op); - release_update_permission(1); -} - -static void -table_deleter(void* data) -{ - HashTable* old_table = (HashTable *) data; + if (new_table == old_table) { + Eterm old_term = get_bucket(old_table, fast_update_index); + ASSERT(is_value(fast_update_term)); + ASSERT(fast_update_index < old_table->allocated); + set_bucket(old_table, fast_update_index, fast_update_term); +#ifdef DEBUG + fast_update_term = THE_NON_VALUE; +#endif - dec_table_refc(NULL, old_table); + if (is_not_nil(old_term)) { + OldLiteral *olp = alloc_old_literal(); + ASSERT(is_tuple_arity(old_term,2)); + olp->term = old_term; + olp->area = term_to_area(old_term); + olp->delete_op.type = DELETE_OP_TUPLE; + olp->delete_op.is_scheduled = 1; + append_to_delete_queue(&olp->delete_op); + cleanup_bytes = (ERTS_LITERAL_AREA_SIZE(olp->area) + + sizeof(OldLiteral)); + erts_schedule_thr_prgr_later_cleanup_op(scheduled_deleter, + &olp->delete_op, + &olp->delete_op.thr_prog_op, + cleanup_bytes); + } + } + else { + ASSERT(is_non_value(fast_update_term)); + ASSERT(new_table->num_to_delete == 0); + erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table); + old_table->delete_op.type = DELETE_OP_TABLE; + old_table->delete_op.is_scheduled = 1; + append_to_delete_queue(&old_table->delete_op); + cleanup_bytes = sizeof_HashTable(old_table->allocated); + if (old_table->num_to_delete <= 1) { + if (old_table->num_to_delete == 1) { + ErtsLiteralArea* area; + area = term_to_area(get_bucket(old_table, + old_table->first_to_delete)); + cleanup_bytes += ERTS_LITERAL_AREA_SIZE(area); + } + erts_schedule_thr_prgr_later_cleanup_op(scheduled_deleter, + &old_table->delete_op, + &old_table->delete_op.thr_prog_op, + cleanup_bytes); + } + else { + /* Only at init:restart(). Don't bother with total cleanup size. */ + ASSERT(old_table->num_to_delete == old_table->allocated); + erts_schedule_thr_prgr_later_op(scheduled_deleter, + &old_table->delete_op, + &old_table->delete_op.thr_prog_op); + } + } + release_update_permission(1); } static void -dec_table_refc(Process* c_p, HashTable* old_table) +scheduled_deleter(void* data) { - erts_aint_t refc = erts_atomic_dec_read_nob(&old_table->refc); - - if (refc == 0) { - HashTable* to_delete; - - while ((to_delete = next_to_delete()) != NULL) { - delete_table(c_p, to_delete); + DeleteOp* dop = (DeleteOp*)data; + + dop = list_to_delete(dop); + + while (dop) { + DeleteOp* next = dop->next; + ASSERT(!dop->is_scheduled); + switch (dop->type) { + case DELETE_OP_TUPLE: { + OldLiteral* olp = ErtsContainerStruct(dop, OldLiteral, delete_op); + delete_tuple(olp->term); + free_old_literal(olp); + break; + } + case DELETE_OP_TABLE: { + HashTable* table = ErtsContainerStruct(dop, HashTable, delete_op); + delete_table(table); + break; + } + default: + ASSERT(!!"Invalid DeleteOp"); } + dop = next; } } static void -delete_table(Process* c_p, HashTable* table) +delete_table(HashTable* table) { Uint idx = table->first_to_delete; Uint n = table->num_to_delete; @@ -1106,25 +1213,32 @@ delete_table(Process* c_p, HashTable* table) #ifdef DEBUG if (n == 1) { - ASSERT(is_tuple_arity(table->term[idx], 2)); + ASSERT(is_tuple_arity(get_bucket(table, idx), 2)); } #endif while (n > 0) { - Eterm term = table->term[idx]; - - if (is_tuple_arity(term, 2)) { - if (is_immed(tuple_val(term)[2])) { - erts_release_literal_area(term_to_area(term)); - } else { - erts_queue_release_literals(c_p, term_to_area(term)); - } - } + delete_tuple(get_bucket(table, idx)); idx++, n--; } erts_free(ERTS_ALC_T_PERSISTENT_TERM, table); } +static void +delete_tuple(Eterm term) +{ + if (is_tuple_arity(term, 2)) { + if (is_immed(tuple_val(term)[2])) { + erts_release_literal_area(term_to_area(term)); + } else { + erts_queue_release_literals(NULL, term_to_area(term)); + } + } + else { + ASSERT(is_nil(term)); + } +} + /* * Caller *must* yield if this function returns 0. */ @@ -1199,42 +1313,46 @@ suspend_updater(Process* c_p) } static void -append_to_delete_queue(HashTable* table) +append_to_delete_queue(DeleteOp* dop) { erts_mtx_lock(&delete_queue_mtx); - table->delete_next = NULL; - *delete_queue_tail = table; - delete_queue_tail = &table->delete_next; + dop->next = NULL; + *delete_queue_tail = dop; + delete_queue_tail = &dop->next; erts_mtx_unlock(&delete_queue_mtx); } -static HashTable* -next_to_delete(void) +static DeleteOp* +list_to_delete(DeleteOp* scheduled_dop) { - HashTable* table; + DeleteOp* dop; + DeleteOp* dop_list; erts_mtx_lock(&delete_queue_mtx); - table = delete_queue_head; - if (table) { - if (erts_atomic_read_nob(&table->refc)) { - /* - * This hash table is still referenced. Hash tables - * must be deleted in order, so we return a NULL - * pointer. - */ - table = NULL; - } else { - /* - * Remove the first hash table from the queue. - */ - delete_queue_head = table->delete_next; - if (delete_queue_head == NULL) { - delete_queue_tail = &delete_queue_head; - } - } + ASSERT(delete_queue_head && delete_queue_head->is_scheduled); + ASSERT(scheduled_dop->is_scheduled); + scheduled_dop->is_scheduled = 0; + + if (scheduled_dop == delete_queue_head) { + dop = delete_queue_head; + while (dop->next && !dop->next->is_scheduled) + dop = dop->next; + + /* + * Remove list of ripe delete ops. + */ + dop_list = delete_queue_head; + delete_queue_head = dop->next; + dop->next = NULL; + if (delete_queue_head == NULL) + delete_queue_tail = &delete_queue_head; + } + else { + dop_list = NULL; } erts_mtx_unlock(&delete_queue_mtx); - return table; + + return dop_list; } /* @@ -1269,25 +1387,53 @@ erts_debug_save_accessed_literal_area(ErtsLiteralArea *lap) accessed_literal_areas[accessed_no_literal_areas++] = lap; } -static void debug_foreach_off_heap(HashTable *tbl, void (*func)(ErlOffHeap *, void *), void *arg) +static void debug_area_off_heap(ErtsLiteralArea* lap, + void (*func)(ErlOffHeap *, void *), + void *arg) +{ + ErlOffHeap oh; + if (!erts_debug_have_accessed_literal_area(lap)) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*func)(&oh, arg); + erts_debug_save_accessed_literal_area(lap); + } +} + +static void debug_table_foreach_off_heap(HashTable *tbl, + void (*func)(ErlOffHeap *, void *), + void *arg) { int i; for (i = 0; i < tbl->allocated; i++) { - Eterm term = tbl->term[i]; + Eterm term = get_bucket(tbl, i); if (is_tuple_arity(term, 2)) { - ErtsLiteralArea *lap = term_to_area(term); - ErlOffHeap oh; - if (!erts_debug_have_accessed_literal_area(lap)) { - ERTS_INIT_OFF_HEAP(&oh); - oh.first = lap->off_heap; - (*func)(&oh, arg); - erts_debug_save_accessed_literal_area(lap); - } + debug_area_off_heap(term_to_area(term), func, arg); } } } +static void debug_delete_op_foreach_off_heap(DeleteOp *dop, + void (*func)(ErlOffHeap *, void *), + void *arg) +{ + switch (dop->type) { + case DELETE_OP_TABLE: { + HashTable* table = ErtsContainerStruct(dop, HashTable, delete_op); + debug_table_foreach_off_heap(table, func, arg); + break; + } + case DELETE_OP_TUPLE: { + OldLiteral* olp = ErtsContainerStruct(dop, OldLiteral, delete_op); + debug_area_off_heap(olp->area, func, arg); + break; + } + default: + ASSERT(!!"Invalid DeleteOp"); + } +} + struct debug_la_oh { void (*func)(ErlOffHeap *, void *); void *arg; @@ -1299,13 +1445,16 @@ static void debug_handle_table(void *vfap, { struct debug_la_oh *fap = vfap; HashTable *tbl = vtbl; - debug_foreach_off_heap(tbl, fap->func, fap->arg); + debug_table_foreach_off_heap(tbl, fap->func, fap->arg); } + void -erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), void *arg) +erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), + void *arg) { HashTable *tbl; + DeleteOp *dop; struct debug_la_oh fa; accessed_no_literal_areas = 0; accessed_literal_areas_size = 10; @@ -1314,19 +1463,16 @@ erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), * accessed_literal_areas_size)); tbl = (HashTable *) erts_atomic_read_nob(&the_hash_table); - debug_foreach_off_heap(tbl, func, arg); + debug_table_foreach_off_heap(tbl, func, arg); erts_mtx_lock(&delete_queue_mtx); - for (tbl = delete_queue_head; tbl; tbl = tbl->delete_next) - debug_foreach_off_heap(tbl, func, arg); + for (dop = delete_queue_head; dop; dop = dop->next) + debug_delete_op_foreach_off_heap(dop, func, arg); erts_mtx_unlock(&delete_queue_mtx); fa.func = func; fa.arg = arg; erts_debug_later_op_foreach(table_updater, debug_handle_table, (void *) &fa); - erts_debug_later_op_foreach(table_deleter, - debug_handle_table, - (void *) &fa); erts_debug_foreach_release_literal_area_off_heap(func, arg); erts_free(ERTS_ALC_T_TMP, accessed_literal_areas); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 63bfaf8572..a989ba97ec 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -44,6 +44,7 @@ #include "erl_bif_unique.h" #include "dtrace-wrapper.h" #include "erl_proc_sig_queue.h" +#include "erl_osenv.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs); @@ -210,7 +211,7 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3) ERTS_BIF_PREP_RET(res, am_false); else { erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, prt); - ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_erts_internal_port_command_3], + ERTS_BIF_PREP_YIELD3(res, &bif_trap_export[BIF_erts_internal_port_command_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } break; @@ -1288,22 +1289,25 @@ static int http_request_erl(void* arg, const http_atom_t* meth, } static int -http_header_erl(void* arg, const http_atom_t* name, const char* name_ptr, - int name_len, const char* value_ptr, int value_len) +http_header_erl(void* arg, const http_atom_t* name, + const char* name_ptr, int name_len, + const char* oname_ptr, int oname_len, + const char* value_ptr, int value_len) { struct packet_callback_args* pca = (struct packet_callback_args*) arg; - Eterm bit_term, name_term, val_term; + Eterm bit_term, name_term, oname_term, val_term; Uint sz = 6; Eterm* hp; #ifdef DEBUG Eterm* hend; #endif - /* {http_header,Bit,Name,IValue,Value} */ + /* {http_header,Bit,Name,Oname,Value} */ if (name == NULL) { http_bld_string(pca, NULL, &sz, name_ptr, name_len); } + http_bld_string(pca, NULL, &sz, oname_ptr, oname_len); http_bld_string(pca, NULL, &sz, value_ptr, value_len); hp = HAlloc(pca->p, sz); @@ -1320,8 +1324,9 @@ http_header_erl(void* arg, const http_atom_t* name, const char* name_ptr, name_term = http_bld_string(pca, &hp,NULL,name_ptr,name_len); } + oname_term = http_bld_string(pca, &hp, NULL, oname_ptr, oname_len); val_term = http_bld_string(pca, &hp, NULL, value_ptr, value_len); - pca->res = TUPLE5(hp, am_http_header, bit_term, name_term, am_undefined, val_term); + pca->res = TUPLE5(hp, am_http_header, bit_term, name_term, oname_term, val_term); ASSERT(hp+6==hend); return 1; } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 4162a6c591..e0777de298 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -80,9 +80,6 @@ static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); static Eterm trace_info_event(Process* p, Eterm event, Eterm key); - -static void reset_bif_trace(void); -static void setup_bif_trace(void); static void install_exp_breakpoints(BpFunctions* f); static void uninstall_exp_breakpoints(BpFunctions* f); static void clean_export_entries(BpFunctions* f); @@ -133,7 +130,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) ErtsTracer meta_tracer = erts_tracer_nil; if (!erts_try_seize_code_write_permission(p)) { - ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_pattern_3], p, MFA, Pattern, flaglist); + ERTS_BIF_YIELD3(&bif_trap_export[BIF_erts_internal_trace_pattern_3], p, MFA, Pattern, flaglist); } finish_bp.current = -1; @@ -543,7 +540,7 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_3], + ERTS_BIF_YIELD3(&bif_trap_export[BIF_erts_internal_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } @@ -793,7 +790,7 @@ Eterm trace_info_2(BIF_ALIST_2) Eterm res; if (!erts_try_seize_code_write_permission(p)) { - ERTS_BIF_YIELD2(bif_export[BIF_trace_info_2], p, What, Key); + ERTS_BIF_YIELD2(&bif_trap_export[BIF_trace_info_2], p, What, Key); } if (What == am_on_load) { @@ -1047,14 +1044,13 @@ static int function_is_traced(Process *p, e.info.mfa.function = mfa[1]; e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->beam; + pc = ep->trampoline.raw; if (ep->addressv[erts_active_code_ix()] == pc && ! BeamIsOpCode(*pc, op_call_error_handler)) { int r = 0; - ASSERT(BeamIsOpCode(*pc, op_apply_bif) || - BeamIsOpCode(*pc, op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; @@ -1426,18 +1422,21 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, int n; BpFunction* fp; - /* - * First work on normal functions (not real BIFs). - */ - erts_bp_match_export(&finish_bp.e, mfa, specified); fp = finish_bp.e.matching; n = finish_bp.e.matched; for (i = 0; i < n; i++) { ErtsCodeInfo *ci = fp[i].ci; - BeamInstr* pc = erts_codeinfo_to_code(ci); - Export* ep = ErtsContainerStruct(ci, Export, info); + BeamInstr* pc; + Export* ep; + + pc = erts_codeinfo_to_code(ci); + ep = ErtsContainerStruct(ci, Export, info); + + if (ep->bif_number != -1) { + ep->is_bif_traced = !!on; + } if (on && !flags.breakpoint) { /* Turn on global call tracing */ @@ -1446,12 +1445,12 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, #ifdef DEBUG ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); #endif - ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); - ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; + ep->trampoline.op = BeamOpCodeAddr(op_trace_jump_W); + ep->trampoline.trace.address = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(ci, match_prog_set, 0); + erts_set_export_trace(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint); + ep->trampoline.op = BeamOpCodeAddr(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1460,91 +1459,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(ci, 0); - if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { - ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); + erts_clear_export_trace(ci, 0); + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { + ep->trampoline.op = BeamOpCodeAddr(op_trace_jump_W); } } } /* - ** OK, now for the bif's - */ - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - - if (!ExportIsBuiltIn(ep)) { - continue; - } - - if (bif_table[i].f == bif_table[i].traced) { - /* Trace wrapper same as regular function - untraceable */ - continue; - } - - switch (specified) { - case 3: - if (mfa->arity != ep->info.mfa.arity) - continue; - case 2: - if (mfa->function != ep->info.mfa.function) - continue; - case 1: - if (mfa->module != ep->info.mfa.module) - continue; - case 0: - break; - default: - ASSERT(0); - } - - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(&ep->info, 1); - erts_clear_mtrace_bif(&ep->info); - erts_set_call_trace_bif(&ep->info, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(&ep->info, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(&ep->info, 0); - erts_set_call_trace_bif(&ep->info, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(&ep->info, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(&ep->info, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(&ep->info, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(&ep->info); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(&ep->info); - m = 1; - } - } - matches += m; - } - } - - /* ** So, now for breakpoint tracing */ erts_bp_match_functions(&finish_bp.f, mfa, specified); @@ -1670,7 +1592,6 @@ erts_finish_breakpointing(void) install_exp_breakpoints(&finish_bp.e); } } - setup_bif_trace(); return 1; case 1: /* @@ -1699,7 +1620,6 @@ erts_finish_breakpointing(void) uninstall_exp_breakpoints(&finish_bp.e); } } - reset_bif_trace(); return 1; case 3: /* @@ -1710,7 +1630,6 @@ erts_finish_breakpointing(void) * updated). If any breakpoints have been totally disabled, * deallocate the GenericBp structs for them. */ - erts_consolidate_bif_bp_data(); clean_export_entries(&finish_bp.e); erts_consolidate_bp_data(&finish_bp.e, 0); erts_consolidate_bp_data(&finish_bp.f, 1); @@ -1736,7 +1655,7 @@ install_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = ep->beam; + ep->addressv[code_ix] = ep->trampoline.raw; } } @@ -1751,11 +1670,12 @@ uninstall_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != ep->beam) { - continue; - } - ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W)); - ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + if (ep->addressv[code_ix] != ep->trampoline.raw) { + continue; + } + + ASSERT(BeamIsOpCode(ep->trampoline.op, op_trace_jump_W)); + ep->addressv[code_ix] = (BeamInstr *) ep->trampoline.trace.address; } } @@ -1770,48 +1690,14 @@ clean_export_entries(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == ep->beam) { - continue; - } - if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) { - ep->beam[0] = (BeamInstr) 0; - ep->beam[1] = (BeamInstr) 0; - } - } -} - -static void -setup_bif_trace(void) -{ - int i; - - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - GenericBp* g = ep->info.u.gen_bp; - if (g) { - if (ExportIsBuiltIn(ep)) { - ASSERT(ep->beam[1]); - ep->beam[1] = (BeamInstr) bif_table[i].traced; - } - } - } -} + if (ep->addressv[code_ix] == ep->trampoline.raw) { + continue; + } -static void -reset_bif_trace(void) -{ - int i; - ErtsBpIndex active = erts_active_bp_ix(); - - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - GenericBp* g = ep->info.u.gen_bp; - if (g && g->data[active].flags == 0) { - if (ExportIsBuiltIn(ep)) { - ASSERT(ep->beam[1]); - ep->beam[1] = (BeamInstr) bif_table[i].f; - } - } + if (BeamIsOpCode(ep->trampoline.op, op_trace_jump_W)) { + ep->trampoline.op = (BeamInstr) 0; + ep->trampoline.trace.address = (BeamInstr) 0; + } } } @@ -1976,8 +1862,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) } if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { - if ((item == am_send) || (item == am_receive) || - (item == am_print) || (item == am_timestamp) + if ((item == am_send) || (item == am_spawn) || + (item == am_receive) || (item == am_print) + || (item == am_timestamp) || (item == am_monotonic_timestamp) || (item == am_strict_monotonic_timestamp)) { hp = HAlloc(p,3); @@ -2041,7 +1928,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } - seq_trace_update_send(BIF_P); + seq_trace_update_serial(BIF_P); seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_1, SEQ_TRACE_PRINT, NIL, BIF_P); BIF_RET(am_true); @@ -2062,7 +1949,7 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) } if (!EQ(BIF_ARG_1, SEQ_TRACE_TOKEN_LABEL(BIF_P))) BIF_RET(am_false); - seq_trace_update_send(BIF_P); + seq_trace_update_serial(BIF_P); seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_2, SEQ_TRACE_PRINT, NIL, BIF_P); BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 6a4f43297e..67eebfe8f6 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -1632,7 +1632,7 @@ erts_get_cpu_topology_term(Process *c_p, Eterm which) } static void -get_logical_processors(int *conf, int *onln, int *avail) +get_logical_processors(int *conf, int *onln, int *avail, int *quota) { if (conf) *conf = erts_get_cpu_configured(cpuinfo); @@ -1640,13 +1640,15 @@ get_logical_processors(int *conf, int *onln, int *avail) *onln = erts_get_cpu_online(cpuinfo); if (avail) *avail = erts_get_cpu_available(cpuinfo); + if (quota) + *quota = erts_get_cpu_quota(cpuinfo); } void -erts_get_logical_processors(int *conf, int *onln, int *avail) +erts_get_logical_processors(int *conf, int *onln, int *avail, int *quota) { erts_rwmtx_rlock(&cpuinfo_rwmtx); - get_logical_processors(conf, onln, avail); + get_logical_processors(conf, onln, avail, quota); erts_rwmtx_runlock(&cpuinfo_rwmtx); } @@ -1655,14 +1657,15 @@ erts_pre_early_init_cpu_topology(int *max_dcg_p, int *max_rg_p, int *conf_p, int *onln_p, - int *avail_p) + int *avail_p, + int *quota_p) { cpu_groups_maps = NULL; no_cpu_groups_callbacks = 0; *max_rg_p = ERTS_MAX_READER_GROUPS; *max_dcg_p = ERTS_MAX_FLXCTR_GROUPS; cpuinfo = erts_cpu_info_create(); - get_logical_processors(conf_p, onln_p, avail_p); + get_logical_processors(conf_p, onln_p, avail_p, quota_p); } void diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index 4a428d7972..91e1322504 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -32,7 +32,8 @@ erts_pre_early_init_cpu_topology(int *max_dcg_p, int *max_rg_p, int *conf_p, int *onln_p, - int *avail_p); + int *avail_p, + int *quota_p); void erts_early_init_cpu_topology(int no_schedulers, int *max_main_threads_p, @@ -81,7 +82,7 @@ Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); int erts_update_cpu_info(void); -void erts_get_logical_processors(int *conf, int *onln, int *avail); +void erts_get_logical_processors(int *conf, int *onln, int *avail, int *quota); int erts_sched_bind_atthrcreate_prepare(void); int erts_sched_bind_atthrcreate_child(int unbind); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index b4a97b42c8..91625dd516 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -51,6 +51,15 @@ erts_atomic_t erts_ets_misc_mem_size; ** Utility macros */ +#if defined(DEBUG) +# define DBG_RANDOM_REDS(REDS, SEED) \ + ((REDS) * 0.1 * erts_sched_local_random_float(SEED)) +#else +# define DBG_RANDOM_REDS(REDS, SEED) (REDS) +#endif + + + #define DB_BIF_GET_TABLE(TB, WHAT, KIND, BIF_IX) \ DB_GET_TABLE(TB, BIF_ARG_1, WHAT, KIND, BIF_IX, NULL, BIF_P) @@ -75,7 +84,7 @@ static BIF_RETTYPE db_bif_fail(Process* p, Uint freason, { if (freason == TRAP) { if (!bif_exp) - bif_exp = bif_export[bif_ix]; + bif_exp = &bif_trap_export[bif_ix]; p->arity = bif_exp->info.mfa.arity; p->i = (BeamInstr*) bif_exp->addressv[erts_active_code_ix()]; } @@ -353,7 +362,9 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + NOLCK_ACCESS=4 /* Used to access the table structure + without acquiring the table lock */ } db_lock_kind_t; extern DbTableMethod db_hash; @@ -409,9 +420,11 @@ static void free_dbtable(void *vtb) { DbTable *tb = (DbTable *) vtb; + erts_flxctr_add(&tb->common.counters, + ERTS_DB_TABLE_MEM_COUNTER_ID, + -((Sint)erts_flxctr_nr_of_allocated_bytes(&tb->common.counters))); ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) || - sizeof(DbTable) == erts_flxctr_read_approx(&tb->common.counters, - ERTS_DB_TABLE_MEM_COUNTER_ID)); + sizeof(DbTable) == DB_GET_APPROX_MEM_CONSUMED(tb)); ASSERT(is_immed(tb->common.heir_data)); @@ -423,7 +436,7 @@ free_dbtable(void *vtb) if (tb->common.btid) erts_bin_release(tb->common.btid); - erts_flxctr_destroy(&tb->common.counters, ERTS_ALC_T_DB_TABLE); + erts_flxctr_destroy(&tb->common.counters, ERTS_ALC_T_ETS_CTRS); erts_free(ERTS_ALC_T_DB_TABLE, tb); } @@ -619,7 +632,7 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) if (kind == LCK_WRITE) { erts_rwmtx_rwlock(&tb->common.rwlock); tb->common.is_thread_safe = 1; - } else { + } else if (kind != NOLCK_ACCESS) { erts_rwmtx_rlock(&tb->common.rwlock); ASSERT(!tb->common.is_thread_safe); } @@ -631,6 +644,8 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) case LCK_WRITE_REC: erts_rwmtx_rwlock(&tb->common.rwlock); break; + case NOLCK_ACCESS: + return; default: erts_rwmtx_rlock(&tb->common.rwlock); } @@ -640,7 +655,7 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) { - if (DB_LOCK_FREE(tb)) + if (DB_LOCK_FREE(tb) || kind == NOLCK_ACCESS) return; if (tb->common.type & DB_FINE_LOCKED) { if (kind == LCK_WRITE) { @@ -671,7 +686,10 @@ static ERTS_INLINE int db_is_exclusive(DbTable* tb, db_lock_kind_t kind) if (DB_LOCK_FREE(tb)) return 1; - return kind != LCK_READ && tb->common.is_thread_safe; + return + kind != LCK_READ && + kind != NOLCK_ACCESS && + tb->common.is_thread_safe; } static DbTable* handle_lacking_permission(Process* p, DbTable* tb, @@ -679,11 +697,27 @@ static DbTable* handle_lacking_permission(Process* p, DbTable* tb, Uint* freason_p) { if (tb->common.status & DB_BUSY) { + void* continuation_state; if (!db_is_exclusive(tb, kind)) { db_unlock(tb, kind); db_lock(tb, LCK_WRITE); } - delete_all_objects_continue(p, tb); + continuation_state = (void*)erts_atomic_read_nob(&tb->common.continuation_state); + if (continuation_state != NULL) { + const long iterations_per_red = 10; + const long reds = iterations_per_red * ERTS_BIF_REDS_LEFT(p); + long nr_of_reductions = DBG_RANDOM_REDS(reds, (Uint)freason_p); + const long init_reds = nr_of_reductions; + tb->common.continuation(&nr_of_reductions, + &continuation_state, + NULL); + if (continuation_state == NULL) { + erts_atomic_set_relb(&tb->common.continuation_state, (Sint)NULL); + } + BUMP_REDS(p, (init_reds - nr_of_reductions) / iterations_per_red); + } else { + delete_all_objects_continue(p, tb); + } db_unlock(tb, LCK_WRITE); tb = NULL; *freason_p = TRAP; @@ -722,9 +756,9 @@ DbTable* db_get_table_aux(Process *p, if (!meta_already_locked) erts_rwmtx_rlock(mtl); else { - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) - || erts_lc_rwmtx_is_rwlocked(mtl) - || META_DB_LOCK_FREE()); + ERTS_LC_ASSERT(META_DB_LOCK_FREE() + || erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); } tb = NULL; if (bucket->pu.tb != NULL) { @@ -752,15 +786,28 @@ DbTable* db_get_table_aux(Process *p, if (tb) { db_lock(tb, kind); #ifdef ETS_DBG_FORCE_TRAP - if (erts_atomic_read_nob(&tb->common.dbg_force_trap) && - erts_atomic_add_read_nob(&tb->common.dbg_force_trap, 2) & 2) { - db_unlock(tb, kind); - tb = NULL; - *freason_p = TRAP; + if (erts_atomic_read_nob(&tb->common.dbg_force_trap)) { + Uint32 rand = erts_sched_local_random((Uint)&p); + if ( !(rand & 7) ) { + /* About 7 of 8 threads that are on the line above + will get here */ + if (erts_atomic_add_read_nob(&tb->common.dbg_force_trap, 2) & 2) { + db_unlock(tb, kind); + tb = NULL; + *freason_p = TRAP; + return tb; + } + } + + } - else #endif - if (ERTS_UNLIKELY(!(tb->common.status & what))) + if (ERTS_UNLIKELY(what != DB_READ_TBL_STRUCT + /* IMPORTANT: the above check is + necessary as the status field might + be in an intermediate state when + kind==NOLCK_ACCESS */ && + !(tb->common.status & what))) tb = handle_lacking_permission(p, tb, kind, freason_p); } else @@ -779,6 +826,17 @@ DbTable* db_get_table(Process *p, return db_get_table_aux(p, id, what, kind, 0, freason_p); } +static BIF_RETTYPE db_get_table_or_fail_return(DbTable **tb, /* out */ + Eterm table_id, + Uint32 what, + db_lock_kind_t kind, + Uint bif_ix, + Process* p) +{ + DB_GET_TABLE(*tb, table_id, what, kind, bif_ix, NULL, p); + return THE_NON_VALUE; +} + static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; @@ -861,7 +919,7 @@ static int remove_named_tab(DbTable *tb, int have_lock) db_lock(tb, LCK_WRITE); } - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock) || META_DB_LOCK_FREE()); + ERTS_LC_ASSERT(META_DB_LOCK_FREE() || erts_lc_rwmtx_is_rwlocked(rwlock)); if (bucket->pu.tb == NULL) { goto done; @@ -1382,6 +1440,506 @@ BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4) return do_update_counter(BIF_P, tb, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); } +typedef enum { + ETS_INSERT_2_LIST_PROCESS_LOCAL, + ETS_INSERT_2_LIST_FAILED_TO_GET_LOCK, + ETS_INSERT_2_LIST_FAILED_TO_GET_LOCK_DESTROY, + ETS_INSERT_2_LIST_GLOBAL +} ets_insert_2_list_status; + +typedef struct { + ets_insert_2_list_status status; + BIF_RETTYPE destroy_return_value; + DbTable* tb; + void* continuation_state; + Binary* continuation_res_bin; +} ets_insert_2_list_info; + + +static ERTS_INLINE BIF_RETTYPE +ets_cret_to_return_value(Process* p, int cret) +{ + switch (cret) { + case DB_ERROR_NONE_FALSE: + BIF_RET(am_false); + case DB_ERROR_NONE: + BIF_RET(am_true); + case DB_ERROR_SYSRES: + BIF_ERROR(p, SYSTEM_LIMIT); + default: + BIF_ERROR(p, BADARG); + } +} + +/* + * > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > + * + * Start of code section that Yielding C Fun (YCF) transforms + * + * The functions within #idef YCF_FUNCTIONS below are not called directly. + * YCF generates yieldable versions of these functions before "erl_db.c" is + * compiled. These generated functions are placed in the file + * "erl_db_insert_list.ycf.h" which is included below. The generation of + * "erl_db_insert_list.ycf.h" is defined in + * "$ERL_TOP/erts/emulator/Makefile.in". See + * "$ERL_TOP/erts/emulator/internal_doc/AutomaticYieldingOfCCode.md" + * for more information about YCF. + * + * > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > + */ + +/* + * The LOCAL_VARIABLE macro is a trick to create a local variable that does not + * get renamed by YCF. + * Such variables will not retain their values over yields. Beware! + * + * I use this as a workaround for a limitation/bug in YCF. It does not do + * proper variable name substitution in expressions passed as argument to + * YCF_CONSUME_REDS(Expr). + */ +#define LOCAL_VARIABLE(TYPE, NAME) TYPE NAME + +#ifdef YCF_FUNCTIONS +static long ets_insert_2_list_check(int keypos, Eterm list) +{ + Eterm lst = THE_NON_VALUE; + long i = 0; + for (lst = list; is_list(lst); lst = CDR(list_val(lst))) { + i++; + if (is_not_tuple(CAR(list_val(lst))) || + (arityval(*tuple_val(CAR(list_val(lst)))) < keypos)) { + return -1; + } + } + if (lst != NIL) { + return -1; + } + return i; +} + +static int ets_insert_new_2_list_has_member(DbTable* tb, Eterm list) +{ + Eterm lst; + Eterm lookup_ret; + DbTableMethod* meth = tb->common.meth; + for (lst = list; is_list(lst); lst = CDR(list_val(lst))) { + meth->db_member(tb, + TERM_GETKEY(tb,CAR(list_val(lst))), + &lookup_ret); + if (lookup_ret != am_false) { + return 1; + } + } + return 0; +} + +static int ets_insert_2_list_from_p_heap(DbTable* tb, Eterm list) +{ + Eterm lst; + DbTableMethod* meth = tb->common.meth; + int cret = DB_ERROR_NONE; + for (lst = list; is_list(lst); lst = CDR(list_val(lst))) { + LOCAL_VARIABLE(SWord, consumed_reds); + consumed_reds = 1; + cret = meth->db_put(tb, CAR(list_val(lst)), 0, &consumed_reds); + if (cret != DB_ERROR_NONE) + return cret; + YCF_CONSUME_REDS(consumed_reds); + } + return DB_ERROR_NONE; +} +#endif /* YCF_FUNCTIONS */ + +/* This function is called both as is, and as YCF transformed. */ +static void ets_insert_2_list_destroy_copied_dbterms(DbTableMethod* meth, + int compressed, + void* db_term_list) +{ + void* lst = db_term_list; + void* term = NULL; + while (lst != NULL) { + term = meth->db_dbterm_list_remove_first(&lst); + meth->db_free_dbterm(compressed, term); + } +} + +#ifdef YCF_FUNCTIONS +static void* ets_insert_2_list_copy_term_list(DbTableMethod* meth, + int compress, + int keypos, + Eterm list) +{ + void* db_term_list = NULL; + void *term; + Eterm lst; + for (lst = list; is_list(lst); lst = CDR(list_val(lst))) { + term = meth->db_eterm_to_dbterm(compress, + keypos, + CAR(list_val(lst))); + if (db_term_list != NULL) { + db_term_list = + meth->db_dbterm_list_prepend(db_term_list, + term); + } else { + db_term_list = term; + } + } + + return db_term_list; + + /* The following code will be executed if the calling process is + killed in the middle of the for loop above*/ + YCF_SPECIAL_CODE_START(ON_DESTROY_STATE); { + ets_insert_2_list_destroy_copied_dbterms(meth, + compress, + db_term_list); + } YCF_SPECIAL_CODE_END(); +} + +static int ets_insert_new_2_dbterm_list_has_member(DbTable* tb, void* db_term_list) +{ + Eterm lookup_ret; + DbTableMethod* meth = tb->common.meth; + void* lst = db_term_list; + void* term = NULL; + Eterm key; + while (lst != NULL) { + term = meth->db_dbterm_list_remove_first(&lst); + key = meth->db_get_dbterm_key(tb, term); + meth->db_member(tb, key, &lookup_ret); + if (lookup_ret != am_false) { + return 1; + } + } + return 0; +} + +static void ets_insert_2_list_insert_db_term_list(DbTable* tb, + void* list) +{ + void* lst = list; + void* term = NULL; + DbTableMethod* meth = tb->common.meth; + do { + LOCAL_VARIABLE(SWord, consumed_reds); + consumed_reds = 1; + term = meth->db_dbterm_list_remove_first(&lst); + meth->db_put_dbterm(tb, term, 0, &consumed_reds); + YCF_CONSUME_REDS(consumed_reds); + } while (lst != NULL); + return; +} + +static void ets_insert_2_list_lock_tbl(Eterm table_id, + Process* p, + Uint bif_ix, + ets_insert_2_list_status on_success_status) +{ + BIF_RETTYPE fail_ret; + DbTable* tb; + ets_insert_2_list_info *ctx; + do { + fail_ret = db_get_table_or_fail_return(&tb, + table_id, + DB_WRITE, + LCK_WRITE, + bif_ix, + p); + ctx = YCF_GET_EXTRA_CONTEXT(); + if (tb == NULL) { + if (p->freason == TRAP) { + ctx->status = ETS_INSERT_2_LIST_FAILED_TO_GET_LOCK; + } else { + ctx->status = ETS_INSERT_2_LIST_FAILED_TO_GET_LOCK_DESTROY; + ctx->destroy_return_value = fail_ret; + } + YCF_YIELD(); + } else { + ctx->status = on_success_status; + ASSERT(DB_LOCK_FREE(tb) || erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); + ASSERT(!(tb->common.status & DB_DELETE)); + } + } while (tb == NULL); +} +#endif /* YCF_FUNCTIONS */ + +static ERTS_INLINE int can_insert_without_yield(Uint32 tb_type, + long list_len, + long reds_left) +{ + if (tb_type & DB_BAG) { + /* Bag inserts can be really bad and we don't know how much searching + * for duplicates we will do */ + return 0; + } + else { + return list_len <= reds_left; + } +} + +#ifdef YCF_FUNCTIONS +static BIF_RETTYPE ets_insert_2_list(Process* p, + Eterm table_id, + DbTable *tb, + Eterm list, + int is_insert_new) +{ + int cret = DB_ERROR_NONE; + void* db_term_list = NULL; /* OBS: memory managements depends on that + db_term_list is initialized to NULL */ + DbTableMethod* meth = tb->common.meth; + int compressed = tb->common.compress; + int keypos = tb->common.keypos; + Uint32 tb_type = tb->common.type; + Uint bif_ix = (is_insert_new ? BIF_ets_insert_new_2 : BIF_ets_insert_2); + long list_len; + /* tb should not be accessed after this point unless the table + lock is held as the table can get deleted while the function is + yielding */ + list_len = ets_insert_2_list_check(keypos, list); + if (list_len < 0) { + return ets_cret_to_return_value(p, DB_ERROR_BADITEM); + } + if (can_insert_without_yield(tb_type, list_len, YCF_NR_OF_REDS_LEFT())) { + long reds_boost; + /* There is enough reductions left to do the inserts directly + from the heap without yielding */ + ets_insert_2_list_lock_tbl(table_id, p, bif_ix, ETS_INSERT_2_LIST_PROCESS_LOCAL); + /* Ensure that we will not yield while inserting from heap */ + reds_boost = YCF_MAX_NR_OF_REDS - YCF_NR_OF_REDS_LEFT(); + YCF_SET_NR_OF_REDS_LEFT(YCF_MAX_NR_OF_REDS); + if (is_insert_new) { + if (ets_insert_new_2_list_has_member(tb, list)) { + cret = DB_ERROR_NONE_FALSE; + } else { + cret = ets_insert_2_list_from_p_heap(tb, list); + } + } else { + cret = ets_insert_2_list_from_p_heap(tb, list); + } + db_unlock(tb, LCK_WRITE); + YCF_SET_NR_OF_REDS_LEFT(YCF_NR_OF_REDS_LEFT() - reds_boost); + return ets_cret_to_return_value(p, cret); + } + /* Copy term list from heap so that other processes can help */ + db_term_list = + ets_insert_2_list_copy_term_list(meth, compressed, keypos, list); + /* Lock table */ + ets_insert_2_list_lock_tbl(table_id, p, bif_ix, ETS_INSERT_2_LIST_GLOBAL); + /* The operation must complete after this point */ + if (is_insert_new) { + if (ets_insert_new_2_dbterm_list_has_member(tb, db_term_list)) { + ets_insert_2_list_destroy_copied_dbterms(meth, + compressed, + db_term_list); + cret = DB_ERROR_NONE_FALSE; + } else { + ets_insert_2_list_insert_db_term_list(tb, db_term_list); + } + } else { + ets_insert_2_list_insert_db_term_list(tb, db_term_list); + } + if (tb->common.continuation != NULL) { + /* Uninstall the continuation from the table struct */ + tb->common.continuation = NULL; + if (is_insert_new) { + int* result_ptr = + ERTS_MAGIC_BIN_DATA(tb->common.continuation_res_bin); + *result_ptr = cret; + erts_bin_release(tb->common.continuation_res_bin); + } + tb->common.status |= tb->common.type & (DB_PRIVATE|DB_PROTECTED|DB_PUBLIC); + tb->common.status &= ~DB_BUSY; + erts_atomic_set_relb(&tb->common.continuation_state, (Sint)NULL); + } + + return ets_cret_to_return_value(NULL, cret); + + /* The following code will be executed if the initiating process + is killed before an ets_insert_2_list_lock_tbl call has + succeeded */ + YCF_SPECIAL_CODE_START(ON_DESTROY_STATE); { + ets_insert_2_list_destroy_copied_dbterms(meth, + compressed, + db_term_list); + } YCF_SPECIAL_CODE_END(); +} +#endif /* YCF_FUNCTIONS */ + +/* + * < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < + * < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < + * + * End of code section that Yielding C Fun (YCF) transforms + * + * < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < + * < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < + */ +#include "erl_db_insert_list.ycf.h" + +static void* ets_insert_2_yield_alloc(size_t size, void* ctx) +{ + (void)ctx; + return erts_alloc(ERTS_ALC_T_ETS_I_LST_TRAP, size); +} + +static void ets_insert_2_yield_free(void* data, void* ctx) +{ + (void)ctx; + erts_free(ERTS_ALC_T_ETS_I_LST_TRAP, data); +} + +static int ets_insert_2_list_yield_dtor(Binary* bin) +{ + ets_insert_2_list_info* ctx = ERTS_MAGIC_BIN_DATA(bin); + if (ctx->status != ETS_INSERT_2_LIST_GLOBAL && + ctx->continuation_state != NULL) { + /* The operation has not been committed to the table and has + not completed*/ + ets_insert_2_list_ycf_gen_destroy(ctx->continuation_state); + } + return 1; +} + +static void ets_insert_2_list_continuation(long *reds_ptr, + void** state, + void* extra_context) +{ + ets_insert_2_list_ycf_gen_continue(reds_ptr, state, extra_context); +} + +static int db_insert_new_2_res_bin_dtor(Binary *context_bin) +{ + (void)context_bin; + return 1; +} + +#define ITERATIONS_PER_RED 8 + +static BIF_RETTYPE ets_insert_2_list_driver(Process* p, + Eterm tid, + Eterm list, + int is_insert_new) { + const long reds = ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); + long nr_of_reductions = DBG_RANDOM_REDS(reds, (Uint)&p); + const long init_reds = nr_of_reductions; + ets_insert_2_list_info* ctx = NULL; + ets_insert_2_list_info ictx; + BIF_RETTYPE ret = THE_NON_VALUE; + Eterm state_mref = list; + Uint bix = (is_insert_new ? BIF_ets_insert_new_2 : BIF_ets_insert_2); + if (is_internal_magic_ref(state_mref)) { + Binary* state_bin = erts_magic_ref2bin(state_mref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) != ets_insert_2_list_yield_dtor) { + BIF_ERROR(p, BADARG); + } + /* Continue a trapped call */ + erts_set_gc_state(p, 1); + ctx = ERTS_MAGIC_BIN_DATA(state_bin); + if (ctx->status == ETS_INSERT_2_LIST_GLOBAL) { + /* An operation that can be helped by other operations is + handled here */ + Uint freason__; + int cret = DB_ERROR_NONE; + DbTable* tb; + /* First check if another process has completed the + operation without acquiring the lock */ + if (NULL == (tb = db_get_table(p, tid, DB_READ_TBL_STRUCT, NOLCK_ACCESS, &freason__))) { + if (freason__ == TRAP){ + erts_set_gc_state(p, 0); + return db_bif_fail(p, freason__, bix, NULL); + } + } + if (tb != NULL && + (void*)erts_atomic_read_acqb(&tb->common.continuation_state) == + ctx->continuation_state) { + /* The lock has to be taken to complete the operation */ + if (NULL == (tb = db_get_table(p, tid, DB_WRITE, LCK_WRITE, &freason__))) { + if (freason__ == TRAP){ + erts_set_gc_state(p, 0); + return db_bif_fail(p, freason__, bix, NULL); + } + } + /* Must be done since the db_get_table call did not trap */ + if (tb != NULL) { + db_unlock(tb, LCK_WRITE); + } + } + if (is_insert_new) { + int* res = ERTS_MAGIC_BIN_DATA(ctx->continuation_res_bin); + cret = *res; + } + return ets_cret_to_return_value(NULL, cret); + } else { + ret = ets_insert_2_list_ycf_gen_continue(&nr_of_reductions, + &ctx->continuation_state, + ctx); + } + } else { + /* Start call */ + ictx.continuation_state = NULL; + ictx.status = ETS_INSERT_2_LIST_PROCESS_LOCAL; + ictx.tb = NULL; + ctx = &ictx; + DB_GET_TABLE(ctx->tb, tid, DB_READ_TBL_STRUCT, NOLCK_ACCESS, bix, NULL, p); + ret = ets_insert_2_list_ycf_gen_yielding(&nr_of_reductions, + &ctx->continuation_state, + ctx, + ets_insert_2_yield_alloc, + ets_insert_2_yield_free, + NULL, + 0, + NULL, + p, + tid, + ctx->tb, + list, + is_insert_new); + if (ctx->continuation_state != NULL) { + Binary* state_bin = erts_create_magic_binary(sizeof(ets_insert_2_list_info), + ets_insert_2_list_yield_dtor); + Eterm* hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin); + ctx = ERTS_MAGIC_BIN_DATA(state_bin); + *ctx = ictx; + } + } + BUMP_REDS(p, (init_reds - nr_of_reductions) / ITERATIONS_PER_RED); + if (ctx->status == ETS_INSERT_2_LIST_GLOBAL && + ctx->continuation_state != NULL && + ctx->tb->common.continuation == NULL) { + /* Install the continuation in the table structure so other + threads can help */ + if (is_insert_new) { + Binary* bin = + erts_create_magic_binary(sizeof(int), + db_insert_new_2_res_bin_dtor); + Eterm* hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + erts_mk_magic_ref(&hp, &MSO(p), bin); + erts_refc_inctest(&bin->intern.refc, 2); + ctx->tb->common.continuation_res_bin = bin; + ctx->continuation_res_bin = bin; + } + ctx->tb->common.continuation = ets_insert_2_list_continuation; + ctx->tb->common.status &= ~(DB_PRIVATE|DB_PROTECTED|DB_PUBLIC); + ctx->tb->common.status |= DB_BUSY; + erts_atomic_set_relb(&ctx->tb->common.continuation_state, + (Sint)ctx->continuation_state); + } + if (ctx->status == ETS_INSERT_2_LIST_FAILED_TO_GET_LOCK_DESTROY) { + return ctx->destroy_return_value; + } + if (ctx->status == ETS_INSERT_2_LIST_GLOBAL) { + db_unlock(ctx->tb, LCK_WRITE); + } + if (ctx->continuation_state != NULL) { + erts_set_gc_state(p, 0); + BIF_TRAP2(&bif_trap_export[bix], p, tid, state_mref); + } + return ret; +} /* ** The put BIF @@ -1390,59 +1948,42 @@ BIF_RETTYPE ets_insert_2(BIF_ALIST_2) { DbTable* tb; int cret = DB_ERROR_NONE; - Eterm lst; + Eterm insert_term; DbTableMethod* meth; - db_lock_kind_t kind; - + SWord consumed_reds = 0; CHECK_TABLES(); - - /* Write lock table if more than one object to keep atomicity */ - kind = ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL) - ? LCK_WRITE : LCK_WRITE_REC); - - DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_2); - if (BIF_ARG_2 == NIL) { - db_unlock(tb, kind); + /* Check that the table exists */ + DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_insert_2); + db_unlock(tb, LCK_WRITE_REC); BIF_RET(am_true); - } - meth = tb->common.meth; - if (is_list(BIF_ARG_2)) { - for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) { - if (is_not_tuple(CAR(list_val(lst))) || - (arityval(*tuple_val(CAR(list_val(lst)))) < tb->common.keypos)) { - goto badarg; - } - } - if (lst != NIL) { - goto badarg; - } - for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) { - cret = meth->db_put(tb, CAR(list_val(lst)), 0); - if (cret != DB_ERROR_NONE) - break; - } + } if ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL) || + is_internal_magic_ref(BIF_ARG_2)) { + /* Handle list case */ + return ets_insert_2_list_driver(BIF_P, + BIF_ARG_1, + BIF_ARG_2, + 0); + } else if (is_list(BIF_ARG_2)) { + insert_term = CAR(list_val(BIF_ARG_2)); } else { - if (is_not_tuple(BIF_ARG_2) || - (arityval(*tuple_val(BIF_ARG_2)) < tb->common.keypos)) { - goto badarg; - } - cret = meth->db_put(tb, BIF_ARG_2, 0); + insert_term = BIF_ARG_2; } - db_unlock(tb, kind); - - switch (cret) { - case DB_ERROR_NONE: - BIF_RET(am_true); - case DB_ERROR_SYSRES: - BIF_ERROR(BIF_P, SYSTEM_LIMIT); - default: - BIF_ERROR(BIF_P, BADARG); + DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_insert_2); + + meth = tb->common.meth; + if (is_not_tuple(insert_term) || + (arityval(*tuple_val(insert_term)) < tb->common.keypos)) { + db_unlock(tb, LCK_WRITE_REC); + BIF_ERROR(BIF_P, BADARG); } - badarg: - db_unlock(tb, kind); - BIF_ERROR(BIF_P, BADARG); + cret = meth->db_put(tb, insert_term, 0, &consumed_reds); + + db_unlock(tb, LCK_WRITE_REC); + + BUMP_REDS(BIF_P, consumed_reds / ITERATIONS_PER_RED); + return ets_cret_to_return_value(BIF_P, cret); } @@ -1456,69 +1997,40 @@ BIF_RETTYPE ets_insert_new_2(BIF_ALIST_2) Eterm ret = am_true; Eterm obj; db_lock_kind_t kind; - + SWord consumed_reds = 0; CHECK_TABLES(); - if (is_list(BIF_ARG_2)) { - if (CDR(list_val(BIF_ARG_2)) != NIL) { - Eterm lst; - Eterm lookup_ret; - DbTableMethod* meth; - - /* More than one object, use LCK_WRITE to keep atomicity */ - kind = LCK_WRITE; - DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_new_2); - - meth = tb->common.meth; - for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) { - if (is_not_tuple(CAR(list_val(lst))) - || (arityval(*tuple_val(CAR(list_val(lst)))) - < tb->common.keypos)) { - goto badarg; - } - } - if (lst != NIL) { - goto badarg; - } - for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) { - cret = meth->db_member(tb, TERM_GETKEY(tb,CAR(list_val(lst))), - &lookup_ret); - if ((cret != DB_ERROR_NONE) || (lookup_ret != am_false)) { - ret = am_false; - goto done; - } - } - - for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) { - cret = meth->db_put(tb,CAR(list_val(lst)), 0); - if (cret != DB_ERROR_NONE) - break; - } - goto done; - } - obj = CAR(list_val(BIF_ARG_2)); - } - else { - obj = BIF_ARG_2; + if (BIF_ARG_2 == NIL) { + /* Check that the table exists */ + DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_insert_2); + db_unlock(tb, LCK_WRITE_REC); + BIF_RET(am_true); + } if ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL) || + is_internal_magic_ref(BIF_ARG_2)) { + /* Handle list case */ + return ets_insert_2_list_driver(BIF_P, BIF_ARG_1, BIF_ARG_2, 1); + } else if (is_list(BIF_ARG_2)) { + obj = CAR(list_val(BIF_ARG_2)); + } else { + obj = BIF_ARG_2; } - /* Only one object (or NIL) - */ + + /* Only one object */ kind = LCK_WRITE_REC; DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_new_2); - if (BIF_ARG_2 == NIL) { - db_unlock(tb, kind); - BIF_RET(am_true); - } if (is_not_tuple(obj) || (arityval(*tuple_val(obj)) < tb->common.keypos)) { - goto badarg; + db_unlock(tb, kind); + BIF_ERROR(BIF_P, BADARG); } cret = tb->common.meth->db_put(tb, obj, - 1); /* key_clash_fail */ + 1, /* key_clash_fail */ + &consumed_reds); -done: db_unlock(tb, kind); + + BUMP_REDS(BIF_P, consumed_reds / ITERATIONS_PER_RED); switch (cret) { case DB_ERROR_NONE: BIF_RET(ret); @@ -1529,9 +2041,6 @@ done: default: BIF_ERROR(BIF_P, BADARG); } - badarg: - db_unlock(tb, kind); - BIF_ERROR(BIF_P, BADARG); } /* @@ -1643,6 +2152,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) Sint keypos; int is_named, is_compressed; int is_fine_locked, frequent_read; + int is_decentralized_counters; + int is_decentralized_counters_option; int cret; DbTableMethod* meth; @@ -1658,6 +2169,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_named = 0; is_fine_locked = 0; frequent_read = 0; + is_decentralized_counters = 0; + is_decentralized_counters_option = -1; heir = am_none; heir_data = (UWord) am_undefined; is_compressed = erts_ets_always_compress; @@ -1674,6 +2187,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status &= ~(DB_SET | DB_BAG | DB_ORDERED_SET | DB_CA_ORDERED_SET); } else if (val == am_ordered_set) { + is_decentralized_counters = 1; status |= DB_ORDERED_SET; status &= ~(DB_SET | DB_BAG | DB_DUPLICATE_BAG | DB_CA_ORDERED_SET); } @@ -1699,12 +2213,18 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } else if (tp[2] == am_false) { frequent_read = 0; } else break; - } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; } + else if (tp[1] == am_decentralized_counters) { + if (tp[2] == am_true) { + is_decentralized_counters_option = 1; + } else if (tp[2] == am_false) { + is_decentralized_counters_option = 0; + } else break; + } else break; } else if (arityval(tp[0]) == 3 && tp[1] == am_heir @@ -1738,6 +2258,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) if (is_not_nil(list)) { /* bad opt or not a well formed list */ BIF_ERROR(BIF_P, BADARG); } + if (-1 != is_decentralized_counters_option) { + is_decentralized_counters = is_decentralized_counters_option; + } if (IS_TREE_TABLE(status) && is_fine_locked && !(status & DB_PRIVATE)) { meth = &db_catree; status |= DB_CA_ORDERED_SET; @@ -1762,27 +2285,30 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) /* we create table outside any table lock * and take the unusal cost of destroy table if it - * fails to find a slot + * fails to find a slot */ { DbTable init_tb; - erts_flxctr_init(&init_tb.common.counters, 0, 2, ERTS_ALC_T_DB_TABLE); + erts_flxctr_init(&init_tb.common.counters, 0, 2, ERTS_ALC_T_ETS_CTRS); tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); erts_flxctr_init(&tb->common.counters, - status & DB_CA_ORDERED_SET, + (status & DB_FINE_LOCKED) && is_decentralized_counters, 2, - ERTS_ALC_T_DB_TABLE); + ERTS_ALC_T_ETS_CTRS); erts_flxctr_add(&tb->common.counters, ERTS_DB_TABLE_MEM_COUNTER_ID, - DB_GET_APPROX_MEM_CONSUMED(&init_tb)); + DB_GET_APPROX_MEM_CONSUMED(&init_tb) + + erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)); } tb->common.meth = meth; tb->common.the_name = BIF_ARG_1; - tb->common.status = status; + tb->common.status = status; tb->common.type = status; /* Note, 'type' is *read only* from now on... */ + tb->common.continuation = NULL; + erts_atomic_set_nob(&tb->common.continuation_state, (Sint)NULL); erts_refc_init(&tb->common.fix_count, 0); db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ)); tb->common.keypos = keypos; @@ -1819,7 +2345,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) table_dec_refc(tb, 0); BIF_ERROR(BIF_P, BADARG); } - + BIF_P->flags |= F_USING_DB; /* So we can remove tb if p dies */ #ifdef HARDDEBUG @@ -2194,7 +2720,7 @@ BIF_RETTYPE ets_internal_delete_all_2(BIF_ALIST_2) tb->common.status |= DB_BUSY; db_unlock(tb, LCK_WRITE); BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_ets_internal_delete_all_2], BIF_P, + BIF_TRAP2(&bif_trap_export[BIF_ets_internal_delete_all_2], BIF_P, BIF_ARG_1, nitems_holder); } else { @@ -2225,7 +2751,7 @@ static void delete_all_objects_continue(Process* p, DbTable* tb) SWord initial_reds = ERTS_BIF_REDS_LEFT(p); SWord reds = initial_reds; - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock) || DB_LOCK_FREE(tb)); + ERTS_LC_ASSERT(DB_LOCK_FREE(tb) || erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); if ((tb->common.status & (DB_DELETE|DB_BUSY)) != DB_BUSY) return; @@ -3310,6 +3836,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, am_read_concurrency, + am_decentralized_counters, am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; @@ -3358,7 +3885,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) BIF_RET(am_undefined); } if (rp == ERTS_PROC_LOCK_BUSY) { - ERTS_BIF_YIELD1(bif_export[BIF_ets_info_1], BIF_P, BIF_ARG_1); + ERTS_BIF_YIELD1(&bif_trap_export[BIF_ets_info_1], BIF_P, BIF_ARG_1); } if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL || tb->common.owner != owner) { @@ -3373,16 +3900,16 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) if (!is_ctrs_read_result_set) { ErtsFlxCtrSnapshotResult res = - erts_flxctr_snapshot(&tb->common.counters, ERTS_ALC_T_DB_TABLE, BIF_P); + erts_flxctr_snapshot(&tb->common.counters, ERTS_ALC_T_ETS_CTRS, BIF_P); if (ERTS_FLXCTR_GET_RESULT_AFTER_TRAP == res.type) { Eterm tuple; db_unlock(tb, LCK_READ); hp = HAlloc(BIF_P, 3); tuple = TUPLE2(hp, res.trap_resume_state, table); - BIF_TRAP1(bif_export[BIF_ets_info_1], BIF_P, tuple); + BIF_TRAP1(&bif_trap_export[BIF_ets_info_1], BIF_P, tuple); } else if (res.type == ERTS_FLXCTR_TRY_AGAIN_AFTER_TRAP) { db_unlock(tb, LCK_READ); - BIF_TRAP1(bif_export[BIF_ets_info_1], BIF_P, table); + BIF_TRAP1(&bif_trap_export[BIF_ets_info_1], BIF_P, table); } else { size = res.result[ERTS_DB_TABLE_NITEMS_COUNTER_ID]; memory = res.result[ERTS_DB_TABLE_MEM_COUNTER_ID]; @@ -3450,13 +3977,13 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2) } if (BIF_ARG_2 == am_size || BIF_ARG_2 == am_memory) { ErtsFlxCtrSnapshotResult res = - erts_flxctr_snapshot(&tb->common.counters, ERTS_ALC_T_DB_TABLE, BIF_P); + erts_flxctr_snapshot(&tb->common.counters, ERTS_ALC_T_ETS_CTRS, BIF_P); if (ERTS_FLXCTR_GET_RESULT_AFTER_TRAP == res.type) { db_unlock(tb, LCK_READ); - BIF_TRAP2(bif_export[BIF_ets_info_2], BIF_P, res.trap_resume_state, BIF_ARG_2); + BIF_TRAP2(&bif_trap_export[BIF_ets_info_2], BIF_P, res.trap_resume_state, BIF_ARG_2); } else if (res.type == ERTS_FLXCTR_TRY_AGAIN_AFTER_TRAP) { db_unlock(tb, LCK_READ); - BIF_TRAP2(bif_export[BIF_ets_info_2], BIF_P, BIF_ARG_1, BIF_ARG_2); + BIF_TRAP2(&bif_trap_export[BIF_ets_info_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } else if (BIF_ARG_2 == am_size) { ret = erts_make_integer(res.result[ERTS_DB_TABLE_NITEMS_COUNTER_ID], BIF_P); } else { /* BIF_ARG_2 == am_memory */ @@ -3522,7 +4049,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) for (lst = BIF_ARG_1; is_list(lst); lst = CDR(list_val(lst))) { if (++i > CONTEXT_REDS) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], + BIF_TRAP3(&bif_trap_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } res = db_prog_match(BIF_P, BIF_P, @@ -4189,7 +4716,7 @@ static SWord free_fixations_locked(Process* p, DbTable *tb) { struct free_fixations_ctx ctx; - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock) || DB_LOCK_FREE(tb)); + ERTS_LC_ASSERT(DB_LOCK_FREE(tb) || erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); ctx.p = p; ctx.tb = tb; @@ -4360,7 +4887,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) } else if (What == am_heir) { ret = tb->common.heir; } else if (What == am_protection) { - if (tb->common.status & DB_PRIVATE) + if (tb->common.status & DB_PRIVATE) ret = am_private; else if (tb->common.status & DB_PROTECTED) ret = am_protected; @@ -4382,6 +4909,8 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = tb->common.compress ? am_true : am_false; } else if (What == am_id) { ret = make_tid(p, tb); + } else if (What == am_decentralized_counters) { + ret = tb->common.counters.is_decentralized ? am_true : am_false; } /* diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index c604744687..6327c56625 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -159,13 +159,15 @@ extern erts_aint_t erts_ets_dbg_force_trap; */ #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ) \ -do { \ - erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ - - ((erts_aint_t) (FREE_SZ))); \ - ASSERT((TAB)); \ - erts_flxctr_add(&(TAB)->common.counters, \ - ERTS_DB_TABLE_MEM_COUNTER_ID, \ - sz__); \ +do { \ + if ((TAB) != NULL) { \ + erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ + - ((erts_aint_t) (FREE_SZ))); \ + ASSERT((TAB)); \ + erts_flxctr_add(&(TAB)->common.counters, \ + ERTS_DB_TABLE_MEM_COUNTER_ID, \ + sz__); \ + } \ } while (0) #define ERTS_ETS_MISC_MEM_ADD(SZ) \ @@ -310,7 +312,8 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size) ASSERT(ptr != 0); ASSERT(size == ERTS_ALC_DBG_BLK_SZ(ptr)); ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0); - ASSERT(((void *) tab) != ptr || + ASSERT(tab == NULL || + ((void *) tab) != ptr || tab->common.counters.is_decentralized || 0 == erts_flxctr_read_centralized(&tab->common.counters, ERTS_DB_TABLE_MEM_COUNTER_ID)); diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c index 4e08f89692..ccf570d3de 100644 --- a/erts/emulator/beam/erl_db_catree.c +++ b/erts/emulator/beam/erl_db_catree.c @@ -104,7 +104,8 @@ static int db_last_catree(Process *p, DbTable *tbl, static int db_prev_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret); -static int db_put_catree(DbTable *tbl, Eterm obj, int key_clash_fail); +static int db_put_catree(DbTable *tbl, Eterm obj, int key_clash_fail, + SWord *consumed_reds_p); static int db_get_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret); static int db_member_catree(DbTable *tbl, Eterm key, Eterm *ret); @@ -160,6 +161,10 @@ db_lookup_dbterm_catree(Process *, DbTable *, Eterm key, Eterm obj, DbUpdateHandle*); static void db_finalize_dbterm_catree(int cret, DbUpdateHandle *); static int db_get_binary_info_catree(Process*, DbTable*, Eterm key, Eterm *ret); +static int db_put_dbterm_catree(DbTable* tbl, + void* obj, + int key_clash_fail, + SWord *consumed_reds_p); static void split_catree(DbTableCATree *tb, DbTableCATreeNode* ERTS_RESTRICT base, @@ -213,6 +218,12 @@ DbTableMethod db_catree = db_foreach_offheap_catree, db_lookup_dbterm_catree, db_finalize_dbterm_catree, + db_eterm_to_dbterm_tree_common, + db_dbterm_list_prepend_tree_common, + db_dbterm_list_remove_first_tree_common, + db_put_dbterm_catree, + db_free_dbterm_tree_common, + db_get_dbterm_key_tree_common, db_get_binary_info_catree, db_first_catree, /* raw_first same as first */ db_next_catree /* raw_next same as next */ @@ -1367,7 +1378,7 @@ static void split_catree(DbTableCATree *tb, } } -/* @brief Free the entire catree and its sub-trees. +/** @brief Free the entire catree and its sub-trees. * * @param reds Reductions to spend. * @return Reductions left. Negative value if not done. @@ -1464,7 +1475,7 @@ static SWord db_free_table_continue_catree(DbTable *tbl, SWord reds) return reds; } -/* @brief Free all objects of a base node, but keep the base node. +/** @brief Free all objects of a base node, but keep the base node. * * @param reds Reductions to spend. * @return Reductions left. Negative value if not done. @@ -1632,7 +1643,27 @@ static int db_prev_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return result; } -static int db_put_catree(DbTable *tbl, Eterm obj, int key_clash_fail) +static int db_put_dbterm_catree(DbTable* tbl, + void* obj, + int key_clash_fail, + SWord *consumed_reds_p) +{ + TreeDbTerm *value_to_insert = obj; + DbTableCATree *tb = &tbl->catree; + Eterm key = GETKEY(tb, value_to_insert->dbterm.tpl); + FindBaseNode fbn; + DbTableCATreeNode* node = find_wlock_valid_base_node(tb, key, &fbn); + int result = db_put_dbterm_tree_common(&tb->common, + &node->u.base.root, + value_to_insert, + key_clash_fail, + NULL); + wunlock_adapt_base_node(tb, node, fbn.parent, fbn.current_level); + return result; +} + +static int db_put_catree(DbTable *tbl, Eterm obj, int key_clash_fail, + SWord *consumed_reds_p) { DbTableCATree *tb = &tbl->catree; Eterm key = GETKEY(&tb->common, tuple_val(obj)); @@ -1776,7 +1807,7 @@ TreeDbTerm** catree_find_prev_root(CATreeRootIterator *iter, Eterm* keyp) return catree_find_nextprev_root(iter, 0, keyp); } -/* @brief Find root of tree where object with smallest key of all larger than +/** @brief Find root of tree where object with smallest key of all larger than * partially bound key may reside. Can be used as a starting point for * a reverse iteration with pb_key. * @@ -1829,7 +1860,7 @@ TreeDbTerm** catree_find_next_from_pb_key_root(Eterm pb_key, } } -/* @brief Find root of tree where object with largest key of all smaller than +/** @brief Find root of tree where object with largest key of all smaller than * partially bound key may reside. Can be used as a starting point for * a forward iteration with pb_key. * diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5508f5c34e..8076cf33ac 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -85,19 +85,58 @@ #include "erl_db_hash.h" -#define ADD_NITEMS(DB, TO_ADD) \ - erts_flxctr_add(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID, TO_ADD) -#define INC_NITEMS(DB) \ - erts_flxctr_inc_read_centralized(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID) -#define DEC_NITEMS(DB) \ - erts_flxctr_dec_read_centralized(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID) +#define IS_DECENTRALIZED_CTRS(DB) ((DB)->common.counters.is_decentralized) + +#define NITEMS_ESTIMATE_FROM_LCK_CTR(LCK_CTR_P) \ + (LCK_CTR_P->nitems <= 0 ? 1: LCK_CTR_P->nitems) + +#define NITEMS_ESTIMATE(DB, LCK_CTR, HASH) \ + (IS_DECENTRALIZED_CTRS(DB) ? \ + (DB_HASH_LOCK_CNT * \ + (LCK_CTR != NULL ? \ + NITEMS_ESTIMATE_FROM_LCK_CTR(LCK_CTR) : \ + NITEMS_ESTIMATE_FROM_LCK_CTR(GET_LOCK_AND_CTR(DB, HASH)))) : \ + erts_flxctr_read_centralized(&(DB)->common.counters, \ + ERTS_DB_TABLE_NITEMS_COUNTER_ID)) + +#define ADD_NITEMS(DB, LCK_CTR, HASH, TO_ADD) \ + do { \ + if (IS_DECENTRALIZED_CTRS(DB)) { \ + if (LCK_CTR != NULL) { \ + LCK_CTR->nitems += TO_ADD; \ + } else { \ + GET_LOCK_AND_CTR(DB,HASH)->nitems += TO_ADD; \ + } \ + } \ + erts_flxctr_add(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID, TO_ADD); \ + } while(0) +#define INC_NITEMS(DB, LCK_CTR, HASH) \ + do { \ + if (IS_DECENTRALIZED_CTRS(DB)) { \ + if (LCK_CTR != NULL) { \ + LCK_CTR->nitems++; \ + } else { \ + GET_LOCK_AND_CTR(DB,HASH)->nitems++; \ + } \ + } \ + erts_flxctr_inc(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID); \ + } while(0) +#define DEC_NITEMS(DB, LCK_CTR, HASH) \ + do { \ + if (IS_DECENTRALIZED_CTRS(DB)) { \ + if (LCK_CTR != NULL) { \ + LCK_CTR->nitems--; \ + } else { \ + GET_LOCK_AND_CTR(DB,HASH)->nitems--; \ + } \ + } \ + erts_flxctr_dec(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID); \ + } while(0) #define RESET_NITEMS(DB) \ erts_flxctr_reset(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID) -/* - * The following symbols can be manipulated to "tune" the linear hash array - */ + #define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) -#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) +#define SHRINK_LIMIT(TB) erts_atomic_read_nob(&(TB)->shrink_limit) /* ** We want the first mandatory segment to be small (to reduce minimal footprint) @@ -129,14 +168,16 @@ : ((struct segment**) erts_atomic_read_nob(&(tb)->segtab))) #endif #define NACTIVE(tb) ((int)erts_atomic_read_nob(&(tb)->nactive)) -#define NITEMS(tb) \ - ((Sint)erts_flxctr_read_centralized(&(tb)->common.counters, \ - ERTS_DB_TABLE_NITEMS_COUNTER_ID)) #define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) #define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] +#ifdef DEBUG +# define DBG_BUCKET_INACTIVE ((HashDbTerm*)0xdead5107) +#endif + + /* * When deleting a table, the number of records to delete. * Approximate number, because we must delete entire buckets. @@ -224,7 +265,8 @@ static ERTS_INLINE int is_pseudo_deleted(HashDbTerm* p) make_internal_hash(term, 0)) & MAX_HASH_MASK) # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) -# define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) +# define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck_ctr.lck) +# define GET_LOCK_AND_CTR(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck_ctr) # define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ @@ -252,6 +294,20 @@ static ERTS_INLINE erts_rwmtx_t* WLOCK_HASH(DbTableHash* tb, HashValue hval) } } +/* Fine grained write lock */ +static ERTS_INLINE +DbTableHashLockAndCounter* WLOCK_HASH_GET_LCK_AND_CTR(DbTableHash* tb, HashValue hval) +{ + if (tb->common.is_thread_safe) { + return NULL; + } else { + DbTableHashLockAndCounter* lck_ctr = GET_LOCK_AND_CTR(tb,hval); + ASSERT(tb->common.type & DB_FINE_LOCKED); + erts_rwmtx_rwlock(&lck_ctr->lck); + return lck_ctr; + } +} + static ERTS_INLINE void RUNLOCK_HASH(erts_rwmtx_t* lck) { if (lck != NULL) { @@ -266,6 +322,13 @@ static ERTS_INLINE void WUNLOCK_HASH(erts_rwmtx_t* lck) } } +static ERTS_INLINE void WUNLOCK_HASH_LCK_CTR(DbTableHashLockAndCounter* lck_ctr) +{ + if (lck_ctr != NULL) { + erts_rwmtx_rwunlock(&lck_ctr->lck); + } +} + #ifdef ERTS_ENABLE_LOCK_CHECK # define IFN_EXCL(tb,cmd) (((tb)->common.is_thread_safe) || (cmd)) @@ -377,7 +440,7 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete */ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); static void alloc_seg(DbTableHash *tb); -static int free_seg(DbTableHash *tb, int free_records); +static int free_seg(DbTableHash *tb); static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -468,18 +531,24 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, DbUpdateHandle* handle); static void db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); +static void* db_eterm_to_dbterm_hash(int compress, int keypos, Eterm obj); +static void* db_dbterm_list_prepend_hash(void* list, void* db_term); +static void* db_dbterm_list_remove_first_hash(void** list); +static int db_put_dbterm_hash(DbTable* tb, + void* obj, + int key_clash_fail, + SWord *consumed_reds_p); +static void db_free_dbterm_hash(int compressed, void* obj); +static Eterm db_get_dbterm_key_hash(DbTable* tb, void* db_term); static int db_get_binary_info_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret); static int db_raw_first_hash(Process* p, DbTable *tbl, Eterm *ret); static int db_raw_next_hash(Process* p, DbTable *tbl, Eterm key, Eterm *ret); -static ERTS_INLINE void try_shrink(DbTableHash* tb) +static ERTS_INLINE void try_shrink(DbTableHash* tb, Sint nitems) { - int nactive = NACTIVE(tb); - int nitems = NITEMS(tb); - if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) - && !IS_FIXED(tb)) { + if (nitems < SHRINK_LIMIT(tb) && !IS_FIXED(tb)) { shrink(tb, nitems); } } @@ -512,28 +581,55 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, } } -static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj) +static ERTS_INLINE HashDbTerm* new_dbterm_hash(DbTableCommon* tb, Eterm obj) { HashDbTerm* p; - if (tb->common.compress) { - p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + if (tb->compress) { + p = db_store_term_comp(tb, tb->keypos, NULL, offsetof(HashDbTerm,dbterm), obj); } else { - p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + p = db_store_term(tb, NULL, offsetof(HashDbTerm,dbterm), obj); + } + return p; +} + +/* + * This function only differ from new_dbterm_hash in that it does not + * adjust the memory size of a given table. + */ +static ERTS_INLINE HashDbTerm* new_dbterm_hash_no_tab(int compress, int keypos, Eterm obj) +{ + HashDbTerm* p; + if (compress) { + p = db_store_term_comp(NULL, keypos, NULL, offsetof(HashDbTerm,dbterm), obj); + } else { + p = db_store_term(NULL, NULL, offsetof(HashDbTerm,dbterm), obj); } return p; } +static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj) +{ + return new_dbterm_hash(&tb->common, obj); +} + static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old, Eterm obj) { HashDbTerm* ret; ASSERT(old != NULL); if (tb->common.compress) { - ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + ret = db_store_term_comp(&tb->common, + tb->common.keypos, + &(old->dbterm), + offsetof(HashDbTerm,dbterm), + obj); } else { - ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + ret = db_store_term(&tb->common, + &(old->dbterm), + offsetof(HashDbTerm,dbterm), + obj); } return ret; } @@ -575,6 +671,12 @@ DbTableMethod db_hash = db_foreach_offheap_hash, db_lookup_dbterm_hash, db_finalize_dbterm_hash, + db_eterm_to_dbterm_hash, + db_dbterm_list_prepend_hash, + db_dbterm_list_remove_first_hash, + db_put_dbterm_hash, + db_free_dbterm_hash, + db_get_dbterm_key_hash, db_get_binary_info_hash, db_raw_first_hash, db_raw_next_hash @@ -693,6 +795,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); erts_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); + erts_atomic_init_nob(&tb->shrink_limit, 0); erts_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); SET_SEGTAB(tb, tb->first_segtab); @@ -715,8 +818,9 @@ int db_create_hash(Process *p, DbTable *tbl) (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck_ctr.lck, &rwmtx_opt, "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); + tb->locks->lck_vec[i].lck_ctr.nitems = 0; } /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: @@ -779,7 +883,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) b = next_live(tb, &ix, &lck, b->next); if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { while (b != 0) { - if (!has_key(tb, b, key, hval) && !is_pseudo_deleted(b)) { + if (!has_key(tb, b, key, hval)) { break; } b = next_live(tb, &ix, &lck, b->next); @@ -789,13 +893,56 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = am_EOT; } else { + ASSERT(!is_pseudo_deleted(b)); *ret = db_copy_key(p, tbl, &b->dbterm); RUNLOCK_HASH(lck); } return DB_ERROR_NONE; } -int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) +static int db_eq_terms_comp(DbTableCommon* tb, DbTerm* a, DbTerm* b) +{ + ErlOffHeap tmp_offheap_a; + Eterm* allocp_a; + Eterm* hp_a; + Eterm tmp_a; + ErlOffHeap tmp_offheap_b; + Eterm* allocp_b; + Eterm* hp_b; + Eterm tmp_b; + int is_eq; + + ASSERT(tb->compress); + hp_a = allocp_a = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm)); + tmp_offheap_a.first = NULL; + tmp_a = db_copy_from_comp(tb, a, &hp_a, &tmp_offheap_a); + + hp_b = allocp_b = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm)); + tmp_offheap_b.first = NULL; + tmp_b = db_copy_from_comp(tb, b, &hp_b, &tmp_offheap_b); + + is_eq = eq(tmp_a,tmp_b); + erts_cleanup_offheap(&tmp_offheap_a); + erts_free(ERTS_ALC_T_TMP, allocp_a); + erts_cleanup_offheap(&tmp_offheap_b); + erts_free(ERTS_ALC_T_TMP, allocp_b); + return is_eq; +} + +static ERTS_INLINE int db_terms_eq(DbTableCommon* tb, DbTerm* a, DbTerm* b) +{ + if (!tb->compress) { + return EQ(make_tuple(a->tpl), make_tuple(b->tpl)); + } + else { + return db_eq_terms_comp(tb, a, b); + } +} + +static int db_put_dbterm_hash(DbTable* tbl, + void* ob, + int key_clash_fail, + SWord *consumed_reds_p) { DbTableHash *tb = &tbl->hash; HashValue hval; @@ -804,13 +951,126 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) HashDbTerm** bp; HashDbTerm* b; HashDbTerm* q; - erts_rwmtx_t* lck; + DbTableHashLockAndCounter* lck_ctr; int nitems; int ret = DB_ERROR_NONE; + HashDbTerm *value_to_insert = ob; + Uint size_to_insert = db_term_size(tbl, value_to_insert, offsetof(HashDbTerm, dbterm)); + ERTS_DB_ALC_MEM_UPDATE_(tbl, 0, size_to_insert); + key = GETKEY(tb, value_to_insert->dbterm.tpl); + hval = MAKE_HASH(key); + value_to_insert->hvalue = hval; + lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb, hval); + ix = hash_to_ix(tb, hval); + bp = &BUCKET(tb, ix); + b = *bp; + + for (;;) { + if (b == NULL) { + goto Lnew; + } + if (has_key(tb,b,key,hval)) { + break; + } + bp = &b->next; + b = b->next; + } + /* Key found + */ + if (tb->common.status & DB_SET) { + HashDbTerm* bnext = b->next; + if (is_pseudo_deleted(b)) { + INC_NITEMS(tb, lck_ctr, hval); + b->pseudo_deleted = 0; + } + else if (key_clash_fail) { + ret = DB_ERROR_BADKEY; + goto Ldone; + } + value_to_insert->pseudo_deleted = b->pseudo_deleted; + free_term(tb, b); + q = value_to_insert; + q->next = bnext; + ASSERT(q->hvalue == hval); + *bp = q; + goto Ldone; + } + else if (key_clash_fail) { /* && (DB_BAG || DB_DUPLICATE_BAG) */ + q = b; + do { + if (!is_pseudo_deleted(q)) { + ret = DB_ERROR_BADKEY; + goto Ldone; + } + q = q->next; + }while (q != NULL && has_key(tb,q,key,hval)); + } + else if (tb->common.status & DB_BAG) { + HashDbTerm** qp = bp; + q = b; + do { + if (db_terms_eq(&tb->common, + &value_to_insert->dbterm, + &q->dbterm)) { + if (is_pseudo_deleted(q)) { + INC_NITEMS(tb, lck_ctr, hval); + q->pseudo_deleted = 0; + ASSERT(q->hvalue == hval); + if (q != b) { /* must move to preserve key insertion order */ + *qp = q->next; + q->next = b; + *bp = q; + } + } + free_term(tb, value_to_insert); + goto Ldone; + } + qp = &q->next; + q = *qp; + (*consumed_reds_p)++; + }while (q != NULL && has_key(tb,q,key,hval)); + } + /*else DB_DUPLICATE_BAG */ + +Lnew: + q = value_to_insert; + q->hvalue = hval; + q->pseudo_deleted = 0; + q->next = b; + *bp = q; + INC_NITEMS(tb, lck_ctr, hval); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + WUNLOCK_HASH_LCK_CTR(lck_ctr); + { + int nactive = NACTIVE(tb); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); + } + } + return DB_ERROR_NONE; + +Ldone: + WUNLOCK_HASH_LCK_CTR(lck_ctr); + return ret; +} + +int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail, + SWord *consumed_reds_p) +{ + DbTableHash *tb = &tbl->hash; + HashValue hval; + int ix; + Eterm key; + HashDbTerm** bp; + HashDbTerm* b; + HashDbTerm* q; + DbTableHashLockAndCounter* lck_ctr; + Sint nitems; + int ret = DB_ERROR_NONE; key = GETKEY(tb, tuple_val(obj)); hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb, hval); + lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb, hval); ix = hash_to_ix(tb, hval); bp = &BUCKET(tb, ix); b = *bp; @@ -830,7 +1090,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) if (tb->common.status & DB_SET) { HashDbTerm* bnext = b->next; if (is_pseudo_deleted(b)) { - INC_NITEMS(tb); + INC_NITEMS(tb, lck_ctr, hval); b->pseudo_deleted = 0; } else if (key_clash_fail) { @@ -859,7 +1119,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) do { if (db_eq(&tb->common,obj,&q->dbterm)) { if (is_pseudo_deleted(q)) { - INC_NITEMS(tb); + INC_NITEMS(tb, lck_ctr, hval); q->pseudo_deleted = 0; ASSERT(q->hvalue == hval); if (q != b) { /* must move to preserve key insertion order */ @@ -871,8 +1131,10 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) goto Ldone; } qp = &q->next; - q = *qp; - }while (q != NULL && has_key(tb,q,key,hval)); + q = *qp; + (*consumed_reds_p)++; + }while (q != NULL && has_key(tb,q,key,hval)); + } /*else DB_DUPLICATE_BAG */ @@ -882,10 +1144,11 @@ Lnew: q->pseudo_deleted = 0; q->next = b; *bp = q; - nitems = INC_NITEMS(tb); - WUNLOCK_HASH(lck); + INC_NITEMS(tb, lck_ctr, hval); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + WUNLOCK_HASH_LCK_CTR(lck_ctr); { - int nactive = NACTIVE(tb); + int nactive = NACTIVE(tb); if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nitems); } @@ -893,7 +1156,7 @@ Lnew: return DB_ERROR_NONE; Ldone: - WUNLOCK_HASH(lck); + WUNLOCK_HASH_LCK_CTR(lck_ctr); return ret; } @@ -1047,11 +1310,11 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) HashDbTerm** bp; HashDbTerm* b; HashDbTerm* free_us = NULL; - erts_rwmtx_t* lck; + DbTableHashLockAndCounter* lck_ctr; int nitems_diff = 0; - + Sint nitems; hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb,hval); + lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb,hval); ix = hash_to_ix(tb, hval); bp = &BUCKET(tb, ix); b = *bp; @@ -1078,10 +1341,13 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) bp = &b->next; b = b->next; } - WUNLOCK_HASH(lck); if (nitems_diff) { - ADD_NITEMS(tb, nitems_diff); - try_shrink(tb); + ADD_NITEMS(tb, lck_ctr, hval, nitems_diff); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + } + WUNLOCK_HASH_LCK_CTR(lck_ctr); + if (nitems_diff) { + try_shrink(tb, nitems); } free_term_list(tb, free_us); *ret = am_true; @@ -1099,14 +1365,15 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) HashDbTerm** bp; HashDbTerm* b; HashDbTerm* free_us = NULL; - erts_rwmtx_t* lck; + DbTableHashLockAndCounter* lck_ctr; int nitems_diff = 0; + Sint nitems; int nkeys = 0; Eterm key; key = GETKEY(tb, tuple_val(object)); hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb,hval); + lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb,hval); ix = hash_to_ix(tb, hval); bp = &BUCKET(tb, ix); b = *bp; @@ -1139,10 +1406,13 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) bp = &b->next; b = b->next; } - WUNLOCK_HASH(lck); if (nitems_diff) { - ADD_NITEMS(tb, nitems_diff); - try_shrink(tb); + ADD_NITEMS(tb, lck_ctr, hval, nitems_diff); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + } + WUNLOCK_HASH_LCK_CTR(lck_ctr); + if (nitems_diff) { + try_shrink(tb, nitems); } free_term_list(tb, free_us); *ret = am_true; @@ -2029,9 +2299,11 @@ static int select_delete_on_match_res(traverse_context_t* ctx_base, Sint slot_ix HashDbTerm** current_ptr = *current_ptr_ptr; select_delete_context_t* ctx = (select_delete_context_t*) ctx_base; HashDbTerm* del; + DbTableHashLockAndCounter* lck_ctr; + Uint32 hval; if (match_res != am_true) return 0; - + hval = (*current_ptr)->hvalue; if (NFIXED(ctx->base.tb) > ctx->fixated_by_me) { /* fixated by others? */ if (slot_ix != ctx->last_pseudo_delete) { if (!add_fixed_deletion(ctx->base.tb, slot_ix, ctx->fixated_by_me)) @@ -2047,23 +2319,58 @@ static int select_delete_on_match_res(traverse_context_t* ctx_base, Sint slot_ix del->next = ctx->free_us; ctx->free_us = del; } - DEC_NITEMS(ctx->base.tb); + lck_ctr = GET_LOCK_AND_CTR(ctx->base.tb,slot_ix); + DEC_NITEMS(ctx->base.tb, lck_ctr, hval); return 1; } +/* This function is only safe to call while the table lock is held in + write mode */ +static Sint get_nitems_from_locks_or_counter(DbTableHash* tb) +{ + if (IS_DECENTRALIZED_CTRS(tb)) { + int i; + Sint total = 0; + for (i=0; i < DB_HASH_LOCK_CNT; ++i) { + total += tb->locks->lck_vec[i].lck_ctr.nitems; + } + return total; + } else { + return erts_flxctr_read_centralized(&tb->common.counters, + ERTS_DB_TABLE_NITEMS_COUNTER_ID); + } +} + static int select_delete_on_loop_ended(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) { select_delete_context_t* ctx = (select_delete_context_t*) ctx_base; - free_term_list(ctx->base.tb, ctx->free_us); + DbTableHash* tb = ctx->base.tb; + free_term_list(tb, ctx->free_us); ctx->free_us = NULL; ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS); BUMP_REDS(ctx->base.p, MAX_SELECT_DELETE_ITERATIONS - iterations_left); if (got) { - try_shrink(ctx->base.tb); + Sint nitems; + if (IS_DECENTRALIZED_CTRS(tb)) { + /* Get a random hash value so we can get an nitems + estimate from a random lock */ + HashValue hval = + (HashValue)&ctx + + (HashValue)iterations_left + + (HashValue)erts_get_scheduler_data()->reductions; + erts_rwmtx_t* lck = RLOCK_HASH(tb, hval); + DbTableHashLockAndCounter* lck_ctr = GET_LOCK_AND_CTR(tb, hval); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + RUNLOCK_HASH(lck); + } else { + nitems = erts_flxctr_read_centralized(&tb->common.counters, + ERTS_DB_TABLE_NITEMS_COUNTER_ID); + } + try_shrink(tb, nitems); } *ret = erts_make_integer(got, ctx->base.p); return DB_ERROR_NONE; @@ -2294,9 +2601,10 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) HashDbTerm **bp, *b; HashDbTerm *free_us = NULL; HashValue hval = MAKE_HASH(key); - erts_rwmtx_t *lck = WLOCK_HASH(tb, hval); + DbTableHashLockAndCounter *lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb, hval); int ix = hash_to_ix(tb, hval); int nitems_diff = 0; + Sint nitems; *ret = NIL; for (bp = &BUCKET(tb, ix), b = *bp; b; bp = &b->next, b = b->next) { @@ -2322,10 +2630,13 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) break; } } - WUNLOCK_HASH(lck); if (nitems_diff) { - ADD_NITEMS(tb, nitems_diff); - try_shrink(tb); + ADD_NITEMS(tb, lck_ctr, hval, nitems_diff); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + } + WUNLOCK_HASH_LCK_CTR(lck_ctr); + if (nitems_diff) { + try_shrink(tb, nitems); } free_term_list(tb, free_us); return DB_ERROR_NONE; @@ -2449,7 +2760,7 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) static int db_free_empty_table_hash(DbTable *tbl) { - ASSERT(NITEMS(tbl) == 0); + ASSERT(get_nitems_from_locks_or_counter(&tbl->hash) == 0); while (db_free_table_continue_hash(tbl, ERTS_SWORD_MAX) < 0) ; return 0; @@ -2474,7 +2785,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); while(tb->nslots != 0) { - reds -= EXT_SEGSZ/64 + free_seg(tb, 1); + reds -= EXT_SEGSZ/64 + free_seg(tb); /* * If we have done enough work, get out here. @@ -2492,8 +2803,11 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) (void*)tb->locks, sizeof(DbTableHashFineLocks)); tb->locks = NULL; } - ASSERT(sizeof(DbTable) == erts_flxctr_read_approx(&tb->common.counters, - ERTS_DB_TABLE_MEM_COUNTER_ID)); + ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) || + ((sizeof(DbTable) + + erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)) == + erts_flxctr_read_approx(&tb->common.counters, + ERTS_DB_TABLE_MEM_COUNTER_ID))); return reds; /* Done */ } @@ -2672,6 +2986,63 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) return est; } +static void calc_shrink_limit(DbTableHash* tb) +{ + erts_aint_t shrink_limit; + int sample_size_is_enough = 1; + + if (IS_DECENTRALIZED_CTRS(tb)) { + /* + Cochran’s Sample Size Formula indicates that we will get + good estimates if we have 100 buckets or more per lock (see + calculations below) + */ + /* square of z-score 95% confidence */ + /* const double z2 = 1.96*1.96; */ + /* Estimated propotion used buckets */ + /* const double p = 0.5; */ + /* margin of error */ + /* const double moe = 0.1; */ + /* const double moe2 = moe*moe; */ + /* Cochran’s Sample Size Formula x=96.040 */ + /* const double x = (z2 * p * (1-p)) / moe2; */ + /* Modification for smaller populations */ + /* for(int n = 10; n < 1000; n = n + 100){ */ + /* const double d = n*x / (x + n - 1) + 1; */ + /* printf("Cochran_formula=%f size=%d mod_with_size=%f\n", x, n, d); */ + /* } */ + const int needed_slots = 100 * DB_HASH_LOCK_CNT; + if (tb->nslots < needed_slots) { + sample_size_is_enough = 0; + } + } + + if (sample_size_is_enough && tb->nslots >= (FIRST_SEGSZ + 2*EXT_SEGSZ)) { + /* + * Start shrink when the sample size is big enough for + * decentralized counters if decentralized counters are used + * and when we can remove one extra segment and still remain + * below 50% load. + */ + shrink_limit = (tb->nslots - EXT_SEGSZ) / 2; + } + else { + /* + * But don't shrink below two segments. + * Why? In order to have chance of getting rid of the last extra segment, + * and rehash it into the first small segment, we either have to start + * early and do speculative joining of buckets or we have to join a lot + * of buckets during each delete-op. + * + * Instead keep segment #2 once allocated. I also think it's a good bet + * a shrinking large table will grow large again. + */ + shrink_limit = 0; + } + erts_atomic_set_nob(&tb->shrink_limit, shrink_limit); +} + + /* Extend table with one new segment */ static void alloc_seg(DbTableHash *tb) @@ -2690,8 +3061,17 @@ static void alloc_seg(DbTableHash *tb) segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, (DbTable *) tb, SIZEOF_SEGMENT(EXT_SEGSZ)); - sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); +#ifdef DEBUG + { + int i; + for (i = 0; i < EXT_SEGSZ; i++) { + segtab[seg_ix]->buckets[i] = DBG_BUCKET_INACTIVE; + } + } +#endif tb->nslots += EXT_SEGSZ; + + calc_shrink_limit(tb); } static void dealloc_ext_segtab(void* lop_data) @@ -2701,10 +3081,19 @@ static void dealloc_ext_segtab(void* lop_data) erts_free(ERTS_ALC_T_DB_SEG, est); } -/* Shrink table by freeing the top segment +struct dealloc_seg_ops { + struct segment* segp; + Uint seg_sz; + + struct ext_segtab* est; +}; + +/* Shrink table by removing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty +** ds_ops: (out) Instructions for dealloc_seg(). */ -static int free_seg(DbTableHash *tb, int free_records) +static int remove_seg(DbTableHash *tb, int free_records, + struct dealloc_seg_ops *ds_ops) { const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); @@ -2712,24 +3101,47 @@ static int free_seg(DbTableHash *tb, int free_records) Uint seg_sz; int nrecords = 0; + ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb) || tb->common.status & DB_DELETE + || erts_atomic_read_nob(&tb->is_resizing)); + ASSERT(segp != NULL); -#ifndef DEBUG - if (free_records) -#endif - { - int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; - while (i--) { - HashDbTerm* p = segp->buckets[i]; + if (free_records) { + int ix, n; + if (seg_ix == 0) { + /* First segment (always fully active) */ + n = FIRST_SEGSZ; + ix = FIRST_SEGSZ-1; + } + else if (NACTIVE(tb) < tb->nslots) { + /* Last extended segment partially active */ + n = (NACTIVE(tb) - FIRST_SEGSZ) & EXT_SEGSZ_MASK; + ix = (NACTIVE(tb)-1) & EXT_SEGSZ_MASK; + } + else { + /* Full extended segment */ + n = EXT_SEGSZ; + ix = EXT_SEGSZ - 1; + } + for ( ; n > 0; n--, ix--) { + HashDbTerm* p = segp->buckets[ix & EXT_SEGSZ_MASK]; while(p != 0) { HashDbTerm* nxt = p->next; - ASSERT(free_records); /* segment not empty as assumed? */ free_term(tb, p); p = nxt; ++nrecords; } } } - +#ifdef DEBUG + else { + int ix = (seg_ix == 0) ? FIRST_SEGSZ-1 : EXT_SEGSZ-1; + for ( ; ix >= 0; ix--) { + ASSERT(segp->buckets[ix] == DBG_BUCKET_INACTIVE); + } + } +#endif + + ds_ops->est = NULL; if (seg_ix >= NSEG_1) { struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); @@ -2738,35 +3150,64 @@ static int free_seg(DbTableHash *tb, int free_records) SET_SEGTAB(tb, est->prev_segtab); tb->nsegs = est->prev_nsegs; - if (!tb->common.is_thread_safe) { - /* - * Table is doing a graceful shrink operation and we must avoid - * deallocating this segtab while it may still be read by other - * threads. Schedule deallocation with thread progress to make - * sure no lingering threads are still hanging in BUCKET macro - * with an old segtab pointer. - */ - erts_schedule_db_free(&tb->common, dealloc_ext_segtab, - est, &est->lop, - SIZEOF_EXT_SEGTAB(est->nsegs)); - } - else - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, - SIZEOF_EXT_SEGTAB(est->nsegs)); + ds_ops->est = est; } } + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); + tb->nslots -= seg_sz; + ASSERT(tb->nslots >= 0); + + ds_ops->segp = segp; + ds_ops->seg_sz = seg_sz; #ifdef DEBUG if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= seg_sz; - ASSERT(tb->nslots >= 0); + calc_shrink_limit(tb); return nrecords; } +/* + * Deallocate segment removed by remove_seg() + */ +static void dealloc_seg(DbTableHash *tb, struct dealloc_seg_ops* ds_ops) +{ + struct ext_segtab* est = ds_ops->est; + + if (est) { + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + erts_schedule_db_free(&tb->common, dealloc_ext_segtab, + est, &est->lop, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } + else + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } + + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, + ds_ops->segp, SIZEOF_SEGMENT(ds_ops->seg_sz)); +} + +/* Remove and deallocate top segment and all its contained objects */ +static int free_seg(DbTableHash *tb) +{ + struct dealloc_seg_ops ds_ops; + int reds; + + reds = remove_seg(tb, 1, &ds_ops); + dealloc_seg(tb, &ds_ops); + return reds; +} /* ** Copy terms from ptr1 until ptr2 @@ -2811,9 +3252,11 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { - if (DB_USING_FINE_LOCKING(tb)) - return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); - else + if (DB_USING_FINE_LOCKING(tb)) { + return + !erts_atomic_read_acqb(&tb->is_resizing) && + !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + } else ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb)); return 1; } @@ -2888,6 +3331,7 @@ static void grow(DbTableHash* tb, int nitems) pnext = &BUCKET(tb, from_ix); p = *pnext; to_pnext = &BUCKET(tb, to_ix); + ASSERT(*to_pnext == DBG_BUCKET_INACTIVE); while (p != NULL) { if (is_pseudo_deleted(p)) { /* rare but possible with fine locking */ *pnext = p->next; @@ -2924,19 +3368,21 @@ abort: */ static void shrink(DbTableHash* tb, int nitems) { - HashDbTerm** src_bp; - HashDbTerm** dst_bp; + struct dealloc_seg_ops ds_ops; + HashDbTerm* src; + HashDbTerm* tail; HashDbTerm** bp; erts_rwmtx_t* lck; int src_ix, dst_ix, low_szm; int nactive; int loop_limit = 5; + ds_ops.segp = NULL; do { if (!begin_resizing(tb)) return; /* already in progress */ nactive = NACTIVE(tb); - if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + if (!(nitems < SHRINK_LIMIT(tb))) { goto abort; /* already done (race) */ } src_ix = nactive - 1; @@ -2953,41 +3399,49 @@ static void shrink(DbTableHash* tb, int nitems) goto abort; } - src_bp = &BUCKET(tb, src_ix); - dst_bp = &BUCKET(tb, dst_ix); - bp = src_bp; - - /* - * We join lists by appending "dst" at the end of "src" - * as we must step through "src" anyway to purge pseudo deleted. - */ - while(*bp != NULL) { - if (is_pseudo_deleted(*bp)) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - + src = BUCKET(tb, src_ix); +#ifdef DEBUG + BUCKET(tb, src_ix) = DBG_BUCKET_INACTIVE; +#endif nactive = src_ix; erts_atomic_set_nob(&tb->nactive, nactive); if (dst_ix == 0) { erts_atomic_set_relb(&tb->szm, low_szm); } - WUNLOCK_HASH(lck); - if (tb->nslots - src_ix >= EXT_SEGSZ) { - free_seg(tb, 0); + remove_seg(tb, 0, &ds_ops); } done_resizing(tb); - } while (--loop_limit - && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + if (src) { + /* + * We join buckets by appending "dst" list at the end of "src" list + * as we must step through "src" anyway to purge pseudo deleted. + */ + bp = &BUCKET(tb, dst_ix); + tail = *bp; + *bp = src; + + while(*bp != NULL) { + if (is_pseudo_deleted(*bp)) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = tail; + } + + WUNLOCK_HASH(lck); + + if (ds_ops.segp) { + dealloc_seg(tb, &ds_ops); + ds_ops.segp = NULL; + } + + } while (--loop_limit && nitems < SHRINK_LIMIT(tb)); return; abort: @@ -3047,13 +3501,13 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, DbTableHash *tb = &tbl->hash; HashValue hval; HashDbTerm **bp, *b; - erts_rwmtx_t* lck; + DbTableHashLockAndCounter* lck_ctr; int flags = 0; ASSERT(tb->common.status & DB_SET); hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb, hval); + lck_ctr = WLOCK_HASH_GET_LCK_AND_CTR(tb, hval); bp = &BUCKET(tb, hash_to_ix(tb, hval)); b = *bp; @@ -3072,7 +3526,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, } if (obj == THE_NON_VALUE) { - WUNLOCK_HASH(lck); + WUNLOCK_HASH_LCK_CTR(lck_ctr); return 0; } @@ -3105,7 +3559,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, ASSERT(q->hvalue == hval); q->pseudo_deleted = 0; *bp = b = q; - INC_NITEMS(tb); + INC_NITEMS(tb, lck_ctr, hval); } HRelease(p, hend, htop); @@ -3118,7 +3572,7 @@ Ldone: handle->dbterm = &b->dbterm; handle->flags = flags; handle->new_size = b->dbterm.size; - handle->u.hash.lck = lck; + handle->u.hash.lck_ctr = lck_ctr; return 1; } @@ -3131,10 +3585,12 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) DbTableHash *tb = &tbl->hash; HashDbTerm **bp = (HashDbTerm **) handle->bp; HashDbTerm *b = *bp; - erts_rwmtx_t* lck = handle->u.hash.lck; + Uint32 hval = b->hvalue; + DbTableHashLockAndCounter* lck_ctr = handle->u.hash.lck_ctr; HashDbTerm* free_me = NULL; + Sint nitems; - ERTS_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ + ERTS_LC_ASSERT(IS_HASH_WLOCKED(tb, &lck_ctr->lck)); /* locked by db_lookup_dbterm_hash */ ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE)); @@ -3146,11 +3602,11 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) *bp = b->next; free_me = b; } - - WUNLOCK_HASH(lck); if (!(handle->flags & DB_INC_TRY_GROW)) - DEC_NITEMS(tb); - try_shrink(tb); + DEC_NITEMS(tb, lck_ctr, hval); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + WUNLOCK_HASH_LCK_CTR(lck_ctr); + try_shrink(tb, nitems); } else { if (handle->flags & DB_MUST_RESIZE) { ASSERT(cret == DB_ERROR_NONE); @@ -3159,16 +3615,18 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) } if (handle->flags & DB_INC_TRY_GROW) { int nactive; - int nitems = INC_NITEMS(tb); + int nitems; ASSERT(cret == DB_ERROR_NONE); - WUNLOCK_HASH(lck); + INC_NITEMS(tb, lck_ctr, hval); + nitems = NITEMS_ESTIMATE(tb, lck_ctr, hval); + WUNLOCK_HASH_LCK_CTR(lck_ctr); nactive = NACTIVE(tb); if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nitems); } } else { - WUNLOCK_HASH(lck); + WUNLOCK_HASH_LCK_CTR(lck_ctr); } } @@ -3187,9 +3645,7 @@ static SWord db_delete_all_objects_hash(Process* p, Eterm* nitems_holder_wb) { if (nitems_holder_wb != NULL) { - Uint nr_of_items = - erts_flxctr_read_centralized(&tbl->common.counters, - ERTS_DB_TABLE_NITEMS_COUNTER_ID); + Uint nr_of_items = get_nitems_from_locks_or_counter(&tbl->hash); *nitems_holder_wb = erts_make_integer(nr_of_items, p); } if (IS_FIXED(tbl)) { @@ -3397,6 +3853,49 @@ static int db_raw_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } +static void* db_eterm_to_dbterm_hash(int compress, int keypos, Eterm obj) +{ + HashDbTerm* term = new_dbterm_hash_no_tab(compress, keypos, obj); + term->next = NULL; + return term; +} + +static void* db_dbterm_list_prepend_hash(void* list, void* db_term) +{ + HashDbTerm* l = list; + HashDbTerm* t = db_term; + t->next = l; + return t; +} + +static void* db_dbterm_list_remove_first_hash(void** list) +{ + if (*list == NULL) { + return NULL; + } else { + HashDbTerm* t = (*list); + HashDbTerm* l = t->next; + *list = l; + return t; + } +} + +/* + * Frees a HashDbTerm without updating the memory footprint of the + * table. + */ +static void db_free_dbterm_hash(int compressed, void* obj) +{ + HashDbTerm* p = obj; + db_free_term_no_tab(compressed, p, offsetof(HashDbTerm, dbterm)); +} + +static Eterm db_get_dbterm_key_hash(DbTable* tb, void* db_term) +{ + HashDbTerm *value_to_insert = db_term; + return GETKEY(tb, value_to_insert->dbterm.tpl); +} + /* For testing only */ Eterm erts_ets_hash_sizeof_ext_segtab(void) { @@ -3418,7 +3917,7 @@ void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable) { } for(i = 0; i < DB_HASH_LOCK_CNT; i++) { - erts_lcnt_ref_t *ref = &tb->locks->lck_vec[i].lck.lcnt; + erts_lcnt_ref_t *ref = &tb->locks->lck_vec[i].lck_ctr.lck.lcnt; if(enable) { erts_lcnt_install_new_lock_info(ref, "db_hash_slot", tb->common.the_name, diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 9759d8b466..b119129099 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -53,9 +53,14 @@ typedef struct hash_db_term { #define DB_HASH_LOCK_CNT 64 #endif +typedef struct DbTableHashLockAndCounter { + Sint nitems; + erts_rwmtx_t lck; +} DbTableHashLockAndCounter; + typedef struct db_table_hash_fine_locks { union { - erts_rwmtx_t lck; + DbTableHashLockAndCounter lck_ctr; byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_rwmtx_t))]; }lck_vec[DB_HASH_LOCK_CNT]; } DbTableHashFineLocks; @@ -63,9 +68,10 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; - /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ + /* szm, nactive, shrink_limit are write-protected by is_resizing or table write lock */ erts_atomic_t szm; /* current size mask. */ erts_atomic_t nactive; /* Number of "active" slots */ + erts_atomic_t shrink_limit; /* Shrink table when fewer objects than this */ erts_atomic_t segtab; /* The segment table (struct segment**) */ struct segment* first_segtab[1]; @@ -93,7 +99,7 @@ Uint db_kept_items_hash(DbTableHash *tb); int db_create_hash(Process *p, DbTable *tbl /* [in out] */); -int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail); +int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail, SWord* consumed_reds_p); int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret); diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 49158108a2..f22a7ae1e9 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -142,20 +142,33 @@ static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableCommon *tb, Eterm obj) { TreeDbTerm* p; if (tb->compress) { - p = db_store_term_comp(tb, NULL, offsetof(TreeDbTerm,dbterm), obj); + p = db_store_term_comp(tb, tb->keypos, NULL, offsetof(TreeDbTerm,dbterm), obj); } else { p = db_store_term(tb, NULL, offsetof(TreeDbTerm,dbterm), obj); } return p; } + +static ERTS_INLINE TreeDbTerm* new_dbterm_no_tab(int compress, int keypos, Eterm obj) +{ + TreeDbTerm* p; + if (compress) { + p = db_store_term_comp(NULL, keypos, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(NULL, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} + static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableCommon *tb, TreeDbTerm* old, Eterm obj) { TreeDbTerm* p; ASSERT(old != NULL); if (tb->compress) { - p = db_store_term_comp(tb, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + p = db_store_term_comp(tb, tb->keypos, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); } else { p = db_store_term(tb, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); @@ -396,7 +409,7 @@ static int db_last_tree(Process *p, DbTable *tbl, static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret); -static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail); +static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail, SWord *consumed_reds_p); static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret); static int db_member_tree(DbTable *tbl, Eterm key, Eterm *ret); @@ -458,9 +471,11 @@ db_lookup_dbterm_tree(Process *, DbTable *, Eterm key, Eterm obj, DbUpdateHandle*); static void db_finalize_dbterm_tree(int cret, DbUpdateHandle *); - static int db_get_binary_info_tree(Process*, DbTable*, Eterm key, Eterm *ret); - +static int db_put_dbterm_tree(DbTable* tbl, /* [in out] */ + void* obj, + int key_clash_fail, + SWord *consumed_reds_p); /* ** Static variables @@ -503,6 +518,12 @@ DbTableMethod db_tree = db_foreach_offheap_tree, db_lookup_dbterm_tree, db_finalize_dbterm_tree, + db_eterm_to_dbterm_tree_common, + db_dbterm_list_prepend_tree_common, + db_dbterm_list_remove_first_tree_common, + db_put_dbterm_tree, + db_free_dbterm_tree_common, + db_get_dbterm_key_tree_common, db_get_binary_info_tree, db_first_tree, /* raw_first same as first */ db_next_tree /* raw_next same as next */ @@ -660,6 +681,139 @@ static ERTS_INLINE int cmp_key_eq(DbTableCommon* tb, Eterm key, TreeDbTerm* obj) return is_same(key, obj_key) || CMP(key, obj_key) == 0; } +/* + * This function differ to db_put_tree_common in that it inserts a TreeDbTerm + * instead of an Eterm. + */ +int db_put_dbterm_tree_common(DbTableCommon *tb, + TreeDbTerm **root, + TreeDbTerm *value_to_insert, + int key_clash_fail, + DbTableTree *stack_container) +{ + /* Non recursive insertion in AVL tree, building our own stack */ + TreeDbTerm **tstack[STACK_NEED]; + int tpos = 0; + int dstack[STACK_NEED+1]; + int dpos = 0; + int state = 0; + TreeDbTerm **this = root; + Sint c; + Eterm key; + int dir; + TreeDbTerm *p1, *p2, *p; + Uint size_to_insert = db_term_size((DbTable*)tb, value_to_insert, offsetof(TreeDbTerm, dbterm)); + ERTS_DB_ALC_MEM_UPDATE_((DbTable*)tb, 0, size_to_insert); + key = GETKEY(tb, value_to_insert->dbterm.tpl); + + reset_static_stack(stack_container); + + dstack[dpos++] = DIR_END; + for (;;) + if (!*this) { /* Found our place */ + state = 1; + INC_NITEMS(((DbTable*)tb)); + *this = value_to_insert; + (*this)->balance = 0; + (*this)->left = (*this)->right = NULL; + break; + } else if ((c = cmp_key(tb, key, *this)) < 0) { + /* go lefts */ + dstack[dpos++] = DIR_LEFT; + tstack[tpos++] = this; + this = &((*this)->left); + } else if (c > 0) { /* go right */ + dstack[dpos++] = DIR_RIGHT; + tstack[tpos++] = this; + this = &((*this)->right); + } else if (!key_clash_fail) { /* Equal key and this is a set, replace. */ + value_to_insert->balance = (*this)->balance; + value_to_insert->left = (*this)->left; + value_to_insert->right = (*this)->right; + free_term((DbTable*)tb, *this); + *this = value_to_insert; + break; + } else { + return DB_ERROR_BADKEY; /* key already exists */ + } + + while (state && ( dir = dstack[--dpos] ) != DIR_END) { + this = tstack[--tpos]; + p = *this; + if (dir == DIR_LEFT) { + switch (p->balance) { + case 1: + p->balance = 0; + state = 0; + break; + case 0: + p->balance = -1; + break; + case -1: /* The icky case */ + p1 = p->left; + if (p1->balance == -1) { /* Single LL rotation */ + p->left = p1->right; + p1->right = p; + p->balance = 0; + (*this) = p1; + } else { /* Double RR rotation */ + p2 = p1->right; + p1->right = p2->left; + p2->left = p1; + p->left = p2->right; + p2->right = p; + p->balance = (p2->balance == -1) ? +1 : 0; + p1->balance = (p2->balance == 1) ? -1 : 0; + (*this) = p2; + } + (*this)->balance = 0; + state = 0; + break; + } + } else { /* dir == DIR_RIGHT */ + switch (p->balance) { + case -1: + p->balance = 0; + state = 0; + break; + case 0: + p->balance = 1; + break; + case 1: + p1 = p->right; + if (p1->balance == 1) { /* Single RR rotation */ + p->right = p1->left; + p1->left = p; + p->balance = 0; + (*this) = p1; + } else { /* Double RL rotation */ + p2 = p1->left; + p1->left = p2->right; + p2->right = p1; + p->right = p2->left; + p2->left = p; + p->balance = (p2->balance == 1) ? -1 : 0; + p1->balance = (p2->balance == -1) ? 1 : 0; + (*this) = p2; + } + (*this)->balance = 0; + state = 0; + break; + } + } + } + return DB_ERROR_NONE; +} + +static int db_put_dbterm_tree(DbTable* tbl, /* [in out] */ + void* obj, + int key_clash_fail, /* DB_ERROR_BADKEY if key exists */ + SWord *consumed_reds_p) +{ + DbTableTree *tb = &tbl->tree; + return db_put_dbterm_tree_common(&tb->common, &tb->root, obj, key_clash_fail, tb); +} + int db_put_tree_common(DbTableCommon *tb, TreeDbTerm **root, Eterm obj, int key_clash_fail, DbTableTree *stack_container) { @@ -772,7 +926,8 @@ int db_put_tree_common(DbTableCommon *tb, TreeDbTerm **root, Eterm obj, return DB_ERROR_NONE; } -static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) +static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail, + SWord *consumed_reds_p) { DbTableTree *tb = &tbl->tree; return db_put_tree_common(&tb->common, &tb->root, obj, key_clash_fail, tb); @@ -1175,7 +1330,7 @@ int db_select_continue_tree_common(Process *p, sc.accum, tptr[7], make_small(sc.got)); - RET_TO_BIF(bif_trap1(bif_export[BIF_ets_select_1], p, continuation), + RET_TO_BIF(bif_trap1(&bif_trap_export[BIF_ets_select_1], p, continuation), DB_ERROR_NONE); #undef RET_TO_BIF @@ -1320,7 +1475,7 @@ int db_select_tree_common(Process *p, DbTable *tb, make_small(sc.got)); /* Don't free mpi.mp, so don't use macro */ - *ret = bif_trap1(bif_export[BIF_ets_select_1], p, continuation); + *ret = bif_trap1(&bif_trap_export[BIF_ets_select_1], p, continuation); return DB_ERROR_NONE; #undef RET_TO_BIF @@ -1728,7 +1883,7 @@ int db_select_chunk_tree_common(Process *p, DbTable *tb, make_small(reverse), make_small(sc.got)); /* Don't let RET_TO_BIF macro free mpi.mp*/ - *ret = bif_trap1(bif_export[BIF_ets_select_1], p, continuation); + *ret = bif_trap1(&bif_trap_export[BIF_ets_select_1], p, continuation); return DB_ERROR_NONE; #undef RET_TO_BIF @@ -2307,9 +2462,12 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds) sizeof(TreeDbTerm *) * STACK_NEED); ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) || ((APPROX_MEM_CONSUMED(tb) - == sizeof(DbTable)) || + == (sizeof(DbTable) + + erts_flxctr_nr_of_allocated_bytes(&tb->common.counters))) || (APPROX_MEM_CONSUMED(tb) - == (sizeof(DbTable) + sizeof(DbFixation))))); + == (sizeof(DbTable) + + sizeof(DbFixation) + + erts_flxctr_nr_of_allocated_bytes(&tb->common.counters))))); } return reds; } @@ -3028,8 +3186,8 @@ found_prev: } -/* @brief Find object with smallest key of all larger than partially bound key. - * Can be used as a starting point for a reverse iteration with pb_key. +/** @brief Find object with smallest key of all larger than partially bound + * key. Can be used as a starting point for a reverse iteration with pb_key. * * @param pb_key The partially bound key. Example {42, '$1'} * @param *rootpp Will return pointer to root pointer of tree with found object. @@ -3078,8 +3236,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTable *tbl, TreeDbTerm*** rootpp, } } -/* @brief Find object with largest key of all smaller than partially bound key. - * Can be used as a starting point for a forward iteration with pb_key. +/** @brief Find object with largest key of all smaller than partially bound + * key. Can be used as a starting point for a forward iteration with pb_key. * * @param pb_key The partially bound key. Example {42, '$1'} * @param *rootpp Will return pointer to root pointer of found object. @@ -3355,6 +3513,50 @@ Eterm db_binary_info_tree_common(Process* p, TreeDbTerm* this) } +void* db_eterm_to_dbterm_tree_common(int compress, int keypos, Eterm obj) +{ + TreeDbTerm* term = new_dbterm_no_tab(compress, keypos, obj); + term->left = NULL; + term->right = NULL; + return term; +} + +void* db_dbterm_list_prepend_tree_common(void *list, void *db_term) +{ + TreeDbTerm* l = list; + TreeDbTerm* t = db_term; + t->left = l; + return t; +} + +void* db_dbterm_list_remove_first_tree_common(void **list) +{ + if (*list == NULL) { + return NULL; + } else { + TreeDbTerm* t = (*list); + TreeDbTerm* l = t->left; + *list = l; + return t; + } +} + +/* + * Frees a TreeDbTerm without updating the memory footprint of the + * table. + */ +void db_free_dbterm_tree_common(int compressed, void* obj) +{ + TreeDbTerm* p = obj; + db_free_term_no_tab(compressed, p, offsetof(TreeDbTerm, dbterm)); +} + +Eterm db_get_dbterm_key_tree_common(DbTable* tb, void* db_term) +{ + TreeDbTerm *term = db_term; + return GETKEY(tb, term->dbterm.tpl); +} + /* * Traverse the tree with a callback function, used by db_match_xxx */ diff --git a/erts/emulator/beam/erl_db_tree_util.h b/erts/emulator/beam/erl_db_tree_util.h index 90ac8c7ba7..4285111810 100644 --- a/erts/emulator/beam/erl_db_tree_util.h +++ b/erts/emulator/beam/erl_db_tree_util.h @@ -171,6 +171,13 @@ void db_finalize_dbterm_tree_common(int cret, DbUpdateHandle *handle, TreeDbTerm **root, DbTableTree *stack_container); +void* db_eterm_to_dbterm_tree_common(int compress, int keypos, Eterm obj); +void* db_dbterm_list_prepend_tree_common(void* list, void* db_term); +void* db_dbterm_list_remove_first_tree_common(void **list); +int db_put_dbterm_tree_common(DbTableCommon *tb, TreeDbTerm **root, TreeDbTerm *value_to_insert, + int key_clash_fail, DbTableTree *stack_container); +void db_free_dbterm_tree_common(int compressed, void* obj); +Eterm db_get_dbterm_key_tree_common(DbTable* tb, void* db_term); Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); TreeDbTerm *db_find_tree_node_common(DbTableCommon*, TreeDbTerm *root, diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 2a0b28f232..5435868c8c 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2612,7 +2612,10 @@ restart: break; case matchCaller: ASSERT(c_p == self); - if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { + t = c_p->stop[0]; + if (is_not_CP(t)) { + *esp++ = am_undefined; + } else if (!(cp = find_function_from_pc(cp_val(t)))) { *esp++ = am_undefined; } else { ehp = HAllocX(build_proc, 4, HEAP_XTRA); @@ -2999,6 +3002,34 @@ void db_free_term(DbTable *tb, void* basep, Uint offset) erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size); } +Uint db_term_size(DbTable *tb, void* basep, Uint offset) +{ + DbTerm* db = (DbTerm*) ((byte*)basep + offset); + if (tb->common.compress) { + return db_alloced_size_comp(db); + } + else { + return offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm); + } +} + +void db_free_term_no_tab(int compress, void* basep, Uint offset) +{ + DbTerm* db = (DbTerm*) ((byte*)basep + offset); + Uint size; + if (compress) { + db_cleanup_offheap_comp(db); + size = db_alloced_size_comp(db); + } + else { + ErlOffHeap tmp_oh; + tmp_oh.first = db->first_oh; + erts_cleanup_offheap(&tmp_oh); + size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm); + } + erts_db_free(ERTS_ALC_T_DB_TERM, NULL, basep, size); +} + static ERTS_INLINE Uint align_up(Uint value, Uint pow2) { ASSERT((pow2 & (pow2-1)) == 0); @@ -3007,7 +3038,7 @@ static ERTS_INLINE Uint align_up(Uint value, Uint pow2) /* Compressed size of an uncompressed term */ -static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj) +static Uint db_size_dbterm_comp(int keypos, Eterm obj) { Eterm* tpl = tuple_val(obj); int i; @@ -3016,11 +3047,11 @@ static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj) + sizeof(Uint); /* "alloc_size" */ for (i = arityval(*tpl); i>0; i--) { - if (i != tb->keypos && is_not_immed(tpl[i])) { + if (i != keypos && is_not_immed(tpl[i])) { size += erts_encode_ext_size_ets(tpl[i]); } } - size += size_object(tpl[tb->keypos]) * sizeof(Eterm); + size += size_object(tpl[keypos]) * sizeof(Eterm); return align_up(size, sizeof(Uint)); } @@ -3036,13 +3067,13 @@ static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix) return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE); } -static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, +static void* copy_to_comp(int keypos, Eterm obj, DbTerm* dest, Uint alloc_size) { ErlOffHeap tmp_offheap; Eterm* src = tuple_val(obj); Eterm* tpl = dest->tpl; - Eterm key = src[tb->keypos]; + Eterm key = src[keypos]; int arity = arityval(src[0]); union { Eterm* ep; @@ -3056,10 +3087,10 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); + tpl[keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { - if (i != tb->keypos) { + if (i != keypos) { if (is_immed(src[i])) { tpl[i] = src[i]; } @@ -3131,14 +3162,17 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) } -void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +void* db_store_term_comp(DbTableCommon *tb, /* May be NULL */ + int keypos, + DbTerm* old, + Uint offset,Eterm obj) { - Uint new_sz = offset + db_size_dbterm_comp(tb, obj); + Uint new_sz = offset + db_size_dbterm_comp(keypos, obj); byte* basep; DbTerm* newp; byte* top; - ASSERT(tb->compress); + ASSERT(tb == NULL || tb->compress); if (old != 0) { Uint old_sz = db_alloced_size_comp(old); db_cleanup_offheap_comp(old); @@ -3158,7 +3192,7 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) } newp->size = size_object(obj); - top = copy_to_comp(tb, obj, newp, new_sz); + top = copy_to_comp(keypos, obj, newp, new_sz); ASSERT(top <= basep + new_sz); (void)top; /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ @@ -3173,7 +3207,7 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) DbTerm* newDbTerm; Uint alloc_sz = offset + (tbl->common.compress ? - db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) : + db_size_dbterm_comp(tbl->common.keypos, make_tuple(handle->dbterm->tpl)) : sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1)); byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz); byte* oldp = *(handle->bp); @@ -3189,7 +3223,7 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) /* make a flat copy */ if (tbl->common.compress) { - copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl), + copy_to_comp(tbl->common.keypos, make_tuple(handle->dbterm->tpl), newDbTerm, alloc_sz); db_free_tmp_uncompressed(handle->dbterm); } @@ -5215,7 +5249,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) Eterm l; Uint32 ret_flags; Uint sz; - BeamInstr *save_cp; + Eterm save_cp; if (trace && !(is_list(against) || against == NIL)) { return THE_NON_VALUE; @@ -5259,13 +5293,13 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) ++n; l = CDR(list_val(l)); } - save_cp = p->cp; - p->cp = NULL; + save_cp = p->stop[0]; + p->stop[0] = NIL; res = erts_match_set_run_trace(p, p, mps, arr, n, ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, &ret_flags); - p->cp = save_cp; + p->stop[0] = save_cp; } else { n = 0; arr = NULL; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 7846a5c98a..4e384606d1 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -43,18 +43,19 @@ * checks for negative returns and issues BIF_ERRORS based * upon these values. */ -#define DB_ERROR_NONE 0 /* No error */ -#define DB_ERROR_BADITEM -1 /* The item was malformed ie no - tuple or to small*/ -#define DB_ERROR_BADTABLE -2 /* The Table is inconsisitent */ -#define DB_ERROR_SYSRES -3 /* Out of system resources */ -#define DB_ERROR_BADKEY -4 /* Returned if a key that should - exist does not. */ +#define DB_ERROR_NONE_FALSE 1 /* No error am_false reult */ +#define DB_ERROR_NONE 0 /* No error */ +#define DB_ERROR_BADITEM -1 /* The item was malformed ie no + tuple or to small*/ +#define DB_ERROR_BADTABLE -2 /* The Table is inconsisitent */ +#define DB_ERROR_SYSRES -3 /* Out of system resources */ +#define DB_ERROR_BADKEY -4 /* Returned if a key that should + exist does not. */ #define DB_ERROR_BADPARAM -5 /* Returned if a specified slot does not exist (hash table only) or the state parameter in db_match_object is broken.*/ -#define DB_ERROR_UNSPEC -10 /* Unspecified error */ +#define DB_ERROR_UNSPEC -10 /* Unspecified error */ /*#define DEBUG_CLONE*/ @@ -92,7 +93,7 @@ typedef struct { int flags; union { struct { - erts_rwmtx_t* lck; + struct DbTableHashLockAndCounter* lck_ctr; } hash; struct { struct DbTableCATreeNode* base_node; @@ -130,7 +131,8 @@ typedef struct db_table_method Eterm* ret); int (*db_put)(DbTable* tb, /* [in out] */ Eterm obj, - int key_clash_fail); /* DB_ERROR_BADKEY if key exists */ + int key_clash_fail, /* DB_ERROR_BADKEY if key exists */ + SWord *consumed_reds_p); int (*db_get)(Process* p, DbTable* tb, /* [in out] */ Eterm key, @@ -234,14 +236,20 @@ typedef struct db_table_method ** dbterm was not updated. If the handle was of a new object and cret is ** not DB_ERROR_NONE, the object is removed from the table. */ void (*db_finalize_dbterm)(int cret, DbUpdateHandle* handle); - + void* (*db_eterm_to_dbterm)(int compress, int keypos, Eterm obj); + void* (*db_dbterm_list_prepend)(void* list, void* db_term); + void* (*db_dbterm_list_remove_first)(void** list); + int (*db_put_dbterm)(DbTable* tb, /* [in out] */ + void* obj, + int key_clash_fail, /* DB_ERROR_BADKEY if key exists */ + SWord *consumed_reds_p); + void (*db_free_dbterm)(int compressed, void* obj); + Eterm (*db_get_dbterm_key)(DbTable* tb, void* db_term); int (*db_get_binary_info)(Process*, DbTable* tb, Eterm key, Eterm* ret); - /* Raw first/next same as first/next but also return pseudo deleted keys. Only internal use by ets:info(_,binary) */ int (*db_raw_first)(Process*, DbTable*, Eterm* ret); int (*db_raw_next)(Process*, DbTable*, Eterm key, Eterm* ret); - } DbTableMethod; typedef struct db_fixation { @@ -312,6 +320,12 @@ typedef struct db_table_common { int keypos; /* defaults to 1 */ int compress; + /* For unfinished operations that needs to be helped */ + void (*continuation)(long *reds_ptr, + void** state, + void* extra_context); /* To help yielded process */ + erts_atomic_t continuation_state; + Binary* continuation_res_bin; #ifdef ETS_DBG_FORCE_TRAP erts_atomic_t dbg_force_trap; /* &1 force enabled, &2 trap this call */ #endif @@ -407,6 +421,7 @@ ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) #define DB_READ (DB_PROTECTED|DB_PUBLIC) #define DB_WRITE DB_PUBLIC #define DB_INFO (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE) +#define DB_READ_TBL_STRUCT (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE|DB_BUSY) #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \ && (T)->common.owner == (P)->common.id) @@ -424,8 +439,13 @@ void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); void db_cleanup_offheap_comp(DbTerm* p); void db_free_term(DbTable *tb, void* basep, Uint offset); +void db_free_term_no_tab(int compress, void* basep, Uint offset); +Uint db_term_size(DbTable *tb, void* basep, Uint offset); void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); -void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void* db_store_term_comp(DbTableCommon *tb, /*May be NULL*/ + int keypos, + DbTerm* old, + Uint offset,Eterm obj); Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, Uint pos, Eterm** hpp, Uint extra); int db_has_map(Eterm obj); @@ -546,9 +566,9 @@ ERTS_GLB_INLINE Eterm erts_db_make_match_prog_ref(Process *p, Binary *mp, Eterm ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary(Eterm term); ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary_unchecked(Eterm term); -/* @brief Ensure off-heap header is word aligned, make a temporary copy if not. - * Needed when inspecting ETS off-heap lists that may contain unaligned - * ProcBins if table is 'compressed'. +/** @brief Ensure off-heap header is word aligned, make a temporary copy if + * not. Needed when inspecting ETS off-heap lists that may contain unaligned + * ProcBins if table is 'compressed'. */ struct erts_tmp_aligned_offheap { diff --git a/erts/emulator/beam/erl_flxctr.c b/erts/emulator/beam/erl_flxctr.c index 35f4a21508..35c4de1a27 100644 --- a/erts/emulator/beam/erl_flxctr.c +++ b/erts/emulator/beam/erl_flxctr.c @@ -46,6 +46,29 @@ typedef enum { ERTS_FLXCTR_SNAPSHOT_ONGOING_TP_THREAD_DO_FREE = 2 } erts_flxctr_snapshot_status; +#define ERTS_FLXCTR_DECENTRALIZED_COUNTER_ARRAY_SIZE \ + (sizeof(ErtsFlxCtrDecentralizedCtrArray) + \ + (sizeof(ErtsFlxCtrDecentralizedCtrArrayElem) * \ + ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS) + \ + ERTS_CACHE_LINE_SIZE) + +#ifdef DEBUG +#define FLXCTR_MEM_DEBUG 1 +#endif + +#ifdef FLXCTR_MEM_DEBUG +static erts_atomic_t debug_mem_usage; +#endif + +#ifdef FLXCTR_MEM_DEBUG +#define FLXCTR_FREE(ALLOC_TYPE, ADDRESS) do { \ + erts_free(ALLOC_TYPE, ADDRESS); \ + erts_atomic_add_mb(&debug_mem_usage, -ERTS_FLXCTR_DECENTRALIZED_COUNTER_ARRAY_SIZE); \ + } while(0) +#else +#define FLXCTR_FREE(ALLOC_TYPE, ADDRESS) erts_free(ALLOC_TYPE, ADDRESS) +#endif + static void thr_prg_wake_up_and_count(void* bin_p) { @@ -72,13 +95,13 @@ thr_prg_wake_up_and_count(void* bin_p) } /* Announce that the snapshot is done */ { - Sint expected = ERTS_FLXCTR_SNAPSHOT_ONGOING; - if (expected != erts_atomic_cmpxchg_mb(&next->snapshot_status, - ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING, - expected)) { - /* The CAS failed which means that this thread need to free the next array. */ - erts_free(info->alloc_type, next->block_start); - } + Sint expected = ERTS_FLXCTR_SNAPSHOT_ONGOING; + if (expected != erts_atomic_cmpxchg_mb(&next->snapshot_status, + ERTS_FLXCTR_SNAPSHOT_NOT_ONGOING, + expected)) { + /* The CAS failed which means that this thread need to free the next array. */ + FLXCTR_FREE(info->alloc_type, next->block_start); + } } /* Resume the process that requested the snapshot */ erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -86,7 +109,7 @@ thr_prg_wake_up_and_count(void* bin_p) erts_resume(p, ERTS_PROC_LOCK_STATUS); } /* Free the memory that is no longer needed */ - erts_free(info->alloc_type, array->block_start); + FLXCTR_FREE(info->alloc_type, array->block_start); erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); erts_proc_dec_refc(p); erts_bin_release(bin); @@ -141,6 +164,14 @@ static void suspend_until_thr_prg(Process* p) erts_schedule_thr_prgr_later_op(thr_prg_wake_up_later, state_bin, &info->later_op); } +size_t erts_flxctr_nr_of_allocated_bytes(ErtsFlxCtr* c) +{ + if (c->is_decentralized) { + return ERTS_FLXCTR_DECENTRALIZED_COUNTER_ARRAY_SIZE; + } else { + return 0; + } +} static ErtsFlxCtrDecentralizedCtrArray* create_decentralized_ctr_array(ErtsAlcType_t alloc_type, Uint nr_of_counters) { @@ -148,14 +179,14 @@ create_decentralized_ctr_array(ErtsAlcType_t alloc_type, Uint nr_of_counters) { the array field is located at the start of a cache line */ char* bytes = erts_alloc(alloc_type, - sizeof(ErtsFlxCtrDecentralizedCtrArray) + - (sizeof(ErtsFlxCtrDecentralizedCtrArrayElem) * - ERTS_FLXCTR_DECENTRALIZED_NO_SLOTS) + - ERTS_CACHE_LINE_SIZE); + ERTS_FLXCTR_DECENTRALIZED_COUNTER_ARRAY_SIZE); void* block_start = bytes; int bytes_to_next_cacheline_border; ErtsFlxCtrDecentralizedCtrArray* array; int i, sched; +#ifdef FLXCTR_MEM_DEBUG + erts_atomic_add_mb(&debug_mem_usage, ERTS_FLXCTR_DECENTRALIZED_COUNTER_ARRAY_SIZE); +#endif bytes = &bytes[offsetof(ErtsFlxCtrDecentralizedCtrArray, array)]; bytes_to_next_cacheline_border = ERTS_CACHE_LINE_SIZE - (((Uint)bytes) % ERTS_CACHE_LINE_SIZE); @@ -178,6 +209,9 @@ create_decentralized_ctr_array(ErtsAlcType_t alloc_type, Uint nr_of_counters) { void erts_flxctr_setup(int decentralized_counter_groups) { reader_groups_array_size = decentralized_counter_groups+1; +#ifdef FLXCTR_MEM_DEBUG + erts_atomic_init_mb(&debug_mem_usage, 0); +#endif } void erts_flxctr_init(ErtsFlxCtr* c, @@ -203,7 +237,7 @@ void erts_flxctr_init(ErtsFlxCtr* c, } } -void erts_flxctr_destroy(ErtsFlxCtr* c, ErtsAlcType_t type) +void erts_flxctr_destroy(ErtsFlxCtr* c, ErtsAlcType_t alloc_type) { if (c->is_decentralized) { if (erts_flxctr_is_snapshot_ongoing(c)) { @@ -220,10 +254,10 @@ void erts_flxctr_destroy(ErtsFlxCtr* c, ErtsAlcType_t type) snapshot is ongoing anymore and the freeing needs to be done here */ ERTS_ASSERT(!erts_flxctr_is_snapshot_ongoing(c)); - erts_free(type, array->block_start); + FLXCTR_FREE(alloc_type, array->block_start); } } else { - erts_free(type, ERTS_FLXCTR_GET_CTR_ARRAY_PTR(c)->block_start); + FLXCTR_FREE(alloc_type, ERTS_FLXCTR_GET_CTR_ARRAY_PTR(c)->block_start); } } } @@ -257,7 +291,7 @@ erts_flxctr_snapshot(ErtsFlxCtr* c, ErtsFlxCtrSnapshotResult res = {.type = ERTS_FLXCTR_TRY_AGAIN_AFTER_TRAP}; suspend_until_thr_prg(p); - erts_free(alloc_type, new_array->block_start); + FLXCTR_FREE(alloc_type, new_array->block_start); return res; } /* Create binary with info about the operation that can be @@ -364,7 +398,19 @@ void erts_flxctr_reset(ErtsFlxCtr* c, } -void erts_flxctr_set_slot(int group) { +void erts_flxctr_set_slot(int group) +{ ErtsSchedulerData *esdp = erts_get_scheduler_data(); esdp->flxctr_slot_no = group; } + +Sint erts_flxctr_debug_memory_usage(void) +{ +#ifdef FLXCTR_MEM_DEBUG + return erts_atomic_read_mb(&debug_mem_usage); +#else + return -1; +#endif +} + + diff --git a/erts/emulator/beam/erl_flxctr.h b/erts/emulator/beam/erl_flxctr.h index 5cab02b9eb..df60f3651e 100644 --- a/erts/emulator/beam/erl_flxctr.h +++ b/erts/emulator/beam/erl_flxctr.h @@ -288,6 +288,24 @@ int erts_flxctr_is_snapshot_ongoing(ErtsFlxCtr* c); */ int erts_flxctr_suspend_until_thr_prg_if_snapshot_ongoing(ErtsFlxCtr* c, Process* p); +/** + * @brief This function returns the number of bytes that are allocated + * for for the given FlxCtr. + * + * @return nr of bytes allocated for the FlxCtr + */ +size_t erts_flxctr_nr_of_allocated_bytes(ErtsFlxCtr* c); + +/** + * @brief This debug function returns the amount of memory allocated + * for decentralized counter arrays when compiled with the DEBUG + * macro. The function returns -1 if the DEBUG macro is undefined. + * + * @return number of bytes allocated for decentralized counter arrays + * if in debug mode and otherwise -1 + */ +Sint erts_flxctr_debug_memory_usage(void); + /* End: Public Interface */ /* Internal Declarations */ diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 9c866250bb..79a1fdb8b9 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -100,27 +100,6 @@ int erts_fun_table_sz(void) } ErlFunEntry* -erts_put_fun_entry(Eterm mod, int uniq, int index) -{ - ErlFunEntry template; - ErlFunEntry* fe; - erts_aint_t refc; - ASSERT(is_atom(mod)); - template.old_uniq = uniq; - template.old_index = index; - template.module = mod; - erts_fun_write_lock(); - fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); - sys_memset(fe->uniq, 0, sizeof(fe->uniq)); - fe->index = 0; - refc = erts_refc_inctest(&fe->refc, 0); - if (refc < 2) /* New or pending delete */ - erts_refc_inc(&fe->refc, 1); - erts_fun_write_unlock(); - return fe; -} - -ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, byte* uniq, int index, int arity) { @@ -130,12 +109,12 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, ASSERT(is_atom(mod)); template.old_uniq = old_uniq; - template.old_index = old_index; + template.index = index; template.module = mod; erts_fun_write_lock(); fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq)); - fe->index = index; + fe->old_index = old_index; fe->arity = arity; refc = erts_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ @@ -144,13 +123,6 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, return fe; } -struct my_key { - Eterm mod; - byte* uniq; - int index; - ErlFunEntry* fe; -}; - ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index) { @@ -159,7 +131,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) ASSERT(is_atom(mod)); template.old_uniq = uniq; - template.old_index = index; + template.index = index; template.module = mod; erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); @@ -199,36 +171,33 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } +struct fun_purge_foreach_args { + BeamInstr *start; + BeamInstr *end; +}; + +static void fun_purge_foreach(ErlFunEntry *fe, struct fun_purge_foreach_args *arg) +{ + BeamInstr* addr = fe->address; + if (arg->start <= addr && addr < arg->end) { + fe->pend_purge_address = addr; + ERTS_THR_WRITE_MEMORY_BARRIER; + fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif + erts_purge_state_add_fun(fe); + } +} + void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) { - int limit; - HashBucket** bucket; - int i; + struct fun_purge_foreach_args args = {start, end}; erts_fun_read_lock(); - limit = erts_fun_table.size; - bucket = erts_fun_table.bucket; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - - while (b) { - ErlFunEntry* fe = (ErlFunEntry *) b; - BeamInstr* addr = fe->address; - - if (start <= addr && addr < end) { - fe->pend_purge_address = addr; - ERTS_THR_WRITE_MEMORY_BARRIER; - fe->address = unloaded_fun; -#ifdef HIPE - fe->pend_purge_native_address = fe->native_address; - hipe_set_closure_stub(fe); -#endif - erts_purge_state_add_fun(fe); - } - b = b->next; - } - } + hash_foreach(&erts_fun_table, (HFOREACH_FUN)fun_purge_foreach, &args); erts_fun_read_unlock(); } @@ -278,36 +247,34 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) ERTS_THR_WRITE_MEMORY_BARRIER; } +struct dump_fun_foreach_args { + fmtfn_t to; + void *to_arg; +}; + +static void +dump_fun_foreach(ErlFunEntry *fe, struct dump_fun_foreach_args *args) +{ + erts_print(args->to, args->to_arg, "=fun\n"); + erts_print(args->to, args->to_arg, "Module: %T\n", fe->module); + erts_print(args->to, args->to_arg, "Uniq: %d\n", fe->old_uniq); + erts_print(args->to, args->to_arg, "Index: %d\n",fe->old_index); + erts_print(args->to, args->to_arg, "Address: %p\n", fe->address); +#ifdef HIPE + erts_print(args->to, args->to_arg, "Native_address: %p\n", fe->native_address); +#endif + erts_print(args->to, args->to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); +} + void erts_dump_fun_entries(fmtfn_t to, void *to_arg) { - int limit; - HashBucket** bucket; - int i; + struct dump_fun_foreach_args args = {to, to_arg}; int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) erts_fun_read_lock(); - limit = erts_fun_table.size; - bucket = erts_fun_table.bucket; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - - while (b) { - ErlFunEntry* fe = (ErlFunEntry *) b; - erts_print(to, to_arg, "=fun\n"); - erts_print(to, to_arg, "Module: %T\n", fe->module); - erts_print(to, to_arg, "Uniq: %d\n", fe->old_uniq); - erts_print(to, to_arg, "Index: %d\n",fe->old_index); - erts_print(to, to_arg, "Address: %p\n", fe->address); -#ifdef HIPE - erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); -#endif - erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); - b = b->next; - } - } + hash_foreach(&erts_fun_table, (HFOREACH_FUN)dump_fun_foreach, &args); if (lock) erts_fun_read_unlock(); } @@ -315,15 +282,27 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg) static HashValue fun_hash(ErlFunEntry* obj) { - return (HashValue) (obj->old_uniq ^ obj->old_index ^ atom_val(obj->module)); + return (HashValue) (obj->old_uniq ^ obj->index ^ atom_val(obj->module)); } static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2) { - return !(obj1->module == obj2->module && + /* + * OTP 23: Use 'index' (instead of 'old_index') when comparing fun + * entries. In OTP 23, multiple make_fun2 instructions may refer to the + * the same 'index' (for the wrapper function generated for the + * 'fun F/A' syntax). + * + * This is safe when loading code compiled with OTP R15 and later, + * because since R15 (2011), the 'index' has been reliably equal + * to 'old_index'. The loader refuses to load modules compiled before + * OTP R15. + */ + + return !(obj1->module == obj2->module && obj1->old_uniq == obj2->old_uniq && - obj1->old_index == obj2->old_index); + obj1->index == obj2->index); } static ErlFunEntry* @@ -333,7 +312,7 @@ fun_alloc(ErlFunEntry* template) sizeof(ErlFunEntry)); obj->old_uniq = template->old_uniq; - obj->old_index = template->old_index; + obj->index = template->index; obj->module = template->module; erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index fb2901d866..eefc7a95bb 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -74,7 +74,6 @@ void erts_init_fun_table(void); void erts_fun_info(fmtfn_t, void *); int erts_fun_table_sz(void); -ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index f4d6ca8eb7..7f51718d1e 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -65,6 +65,8 @@ # define HARDDEBUG 1 #endif +extern BeamInstr beam_apply[2]; + /* * Returns number of elements in an array. */ @@ -937,13 +939,15 @@ garbage_collect_hibernate(Process* p, int check_long_gc) */ erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); - ASSERT(p->stop == p->hend); /* Stack must be empty. */ + ASSERT(p->stop == p->hend - 1); /* Only allow one continuation pointer. */ + ASSERT(p->stop[0] == make_cp(beam_apply+1)); /* * Do it. */ heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; + heap_size += 1; /* Reserve place for continuation pointer */ heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); @@ -969,13 +973,11 @@ garbage_collect_hibernate(Process* p, int check_long_gc) p->high_water = htop; p->htop = htop; p->hend = p->heap + heap_size; - p->stop = p->hend; + p->stop = p->hend - 1; p->heap_sz = heap_size; heap_size = actual_size = p->htop - p->heap; - if (heap_size == 0) { - heap_size = 1; /* We want a heap... */ - } + heap_size += 1; /* Reserve place for continuation pointer */ FLAGS(p) &= ~F_FORCE_GC; p->live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -991,14 +993,15 @@ garbage_collect_hibernate(Process* p, int check_long_gc) * hibernated. */ - ASSERT(p->hend - p->stop == 0); /* Empty stack */ ASSERT(actual_size < p->heap_sz); heap = ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*heap_size); sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); - p->stop = p->hend = heap + heap_size; + p->hend = heap + heap_size; + p->stop = p->hend - 1; + p->stop[0] = make_cp(beam_apply+1); offs = heap - p->heap; area = (char *) p->heap; @@ -2572,13 +2575,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) /* * The process may be garbage-collected while it is terminating. - * (fvalue contains the EXIT reason and ftrace the saved stack trace.) + * fvalue contains the EXIT reason. */ if (is_not_immed(p->fvalue)) { roots[n].v = &p->fvalue; roots[n].sz = 1; n++; } + + /* + * The raise/3 BIF will store the stacktrace in ftrace. + */ if (is_not_immed(p->ftrace)) { roots[n].v = &p->ftrace; roots[n].sz = 1; @@ -2588,7 +2595,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) /* * If a NIF or BIF has saved arguments, they need to be added */ - if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz)) + if (erts_setup_nfunc_rootset(p, &roots[n].v, &roots[n].sz)) n++; ASSERT(n <= rootset->size); @@ -3236,7 +3243,7 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); - if (erts_setup_nif_export_rootset(p, &v, &sz)) + if (erts_setup_nfunc_rootset(p, &v, &sz)) offset_heap_ptr(v, sz, offs, area, area_size); } diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 5f0352f5c0..d282eeb12c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -429,6 +429,7 @@ erl_first_process_otp(char* mod_name, int argc, char** argv) args = CONS(hp, boot_mod, args); so.flags = erts_default_spo_flags; + so.opts = NIL; res = erl_spawn_system_process(&parent, am_erl_init, am_start, args, &so); ASSERT(is_internal_pid(res)); @@ -462,6 +463,7 @@ erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq, int p so.priority = prio; so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; + so.opts = NIL; res = erl_spawn_system_process(parent, mod, am_start, NIL, &so); ASSERT(is_internal_pid(res)); @@ -484,6 +486,7 @@ Eterm erts_internal_spawn_system_process_3(BIF_ALIST_3) { ASSERT(erts_list_length(args) >= 0); so.flags = erts_default_spo_flags; + so.opts = NIL; res = erl_spawn_system_process(BIF_P, mod, func, args, &so); if (is_non_value(res)) { @@ -583,7 +586,7 @@ void erts_usage(void) ERTS_ASYNC_THREAD_MIN_STACK_SIZE, ERTS_ASYNC_THREAD_MAX_STACK_SIZE); erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); - erts_fprintf(stderr, " valid range is [0-%d]\n", + erts_fprintf(stderr, " valid range is [1-%d]\n", ERTS_MAX_NO_OF_ASYNC_THREADS); erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); @@ -774,6 +777,7 @@ early_init(int *argc, char **argv) /* int ncpu; int ncpuonln; int ncpuavail; + int ncpuquota; int schdlrs; int schdlrs_onln; int schdlrs_percentage = 100; @@ -811,7 +815,8 @@ early_init(int *argc, char **argv) /* &max_reader_groups, &ncpu, &ncpuonln, - &ncpuavail); + &ncpuavail, + &ncpuquota); ignore_break = 0; replace_intr = 0; @@ -838,9 +843,18 @@ early_init(int *argc, char **argv) /* * can initialize the allocators. */ no_schedulers = (Uint) (ncpu > 0 ? ncpu : 1); - no_schedulers_online = (ncpuavail > 0 - ? ncpuavail - : (ncpuonln > 0 ? ncpuonln : no_schedulers)); + + if (ncpuavail > 0) { + if (ncpuquota > 0) { + no_schedulers_online = MIN(ncpuquota, ncpuavail); + } else { + no_schedulers_online = ncpuavail; + } + } else if (ncpuonln > 0) { + no_schedulers_online = ncpuonln; + } else { + no_schedulers_online = no_schedulers; + } schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; @@ -1129,6 +1143,9 @@ early_init(int *argc, char **argv) /* i++; } + if (erts_async_max_threads < 1) + erts_async_max_threads = 1; + /* apply any scheduler percentages */ if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) { schdlrs = schdlrs * schdlrs_percentage / 100; diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index 2ae5b56b5c..c82d67f893 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -1078,7 +1078,7 @@ static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) { state = boxed_state; } - ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1], + ERTS_BIF_YIELD1(&bif_trap_export[BIF_iolist_to_iovec_1], state->process, state->magic_reference); } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index b391c05643..507db504db 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -84,6 +84,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "reg_tab", NULL }, { "proc_main", "pid" }, { "old_code", "address" }, + { "nif_call_tab", NULL }, #ifdef HIPE { "hipe_mfait_lock", NULL }, #endif @@ -148,6 +149,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr_x", NULL }, { "instr", NULL }, { "dyn_lock_check", NULL }, + { "nif_load", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, { "get_time", NULL }, diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 0d47b16e0b..c061e7894d 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -78,7 +78,7 @@ typedef struct { } erts_lcnt_time_t; typedef struct { - /* @brief log2 array of nano seconds occurences */ + /** @brief log2 array of nano seconds occurences */ Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; } erts_lcnt_hist_t; @@ -271,7 +271,7 @@ int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref); erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count); -/* @brief Initializes the lock info at the given index. +/** @brief Initializes the lock info at the given index. * @param id An immediate erlang term with whatever extra data you want to * identify this lock with. * @param flags The flags the lock itself was initialized with. Keep in mind @@ -300,9 +300,10 @@ void erts_lcnt_pre_thr_init(void); void erts_lcnt_post_thr_init(void); void erts_lcnt_late_init(void); -/* @brief Called after everything in the system has been initialized, including - * the schedulers. This is mainly a backwards compatibility shim for matching - * the old lcnt behavior where all lock counting was enabled by default. */ +/** @brief Called after everything in the system has been initialized, + * including the schedulers. This is mainly a backwards compatibility shim for + * matching the old lcnt behavior where all lock counting was enabled by + * default. */ void erts_lcnt_post_startup(void); void erts_lcnt_thread_setup(void); diff --git a/erts/emulator/beam/erl_lock_flags.h b/erts/emulator/beam/erl_lock_flags.h index 2db133b598..9d2216eaf6 100644 --- a/erts/emulator/beam/erl_lock_flags.h +++ b/erts/emulator/beam/erl_lock_flags.h @@ -71,10 +71,10 @@ typedef unsigned short erts_lock_flags_t; typedef unsigned short erts_lock_options_t; -/* @brief Gets the type name of the lock, honoring the RW flag if supplied. */ +/** @brief Gets the type name of the lock, honoring the RW flag if supplied. */ const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags); -/* @brief Gets a short-form description of the given lock options. (rw/r/w) */ +/** @brief Gets a short-form description of the given lock options. (rw/r/w) */ const char *erts_lock_options_get_short_desc(erts_lock_options_t options); #endif /* ERTS_LOCK_FLAGS_H__ */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 62dd85e425..9097a36e62 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -973,7 +973,13 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) { HIPE_WRAPPER_BIF_DISABLE_GC(maps_merge, 2) BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { - if (is_flatmap(BIF_ARG_1)) { + if (BIF_ARG_1 == BIF_ARG_2) { + /* Merging upon itself always returns itself */ + if (is_map(BIF_ARG_1)) { + return BIF_ARG_1; + } + BIF_P->fvalue = BIF_ARG_1; + } else if (is_flatmap(BIF_ARG_1)) { if (is_flatmap(BIF_ARG_2)) { BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); } else if (is_hashmap(BIF_ARG_2)) { @@ -1008,6 +1014,9 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { n1 = flatmap_get_size(mp1); n2 = flatmap_get_size(mp2); + if (n1 == 0) return nodeB; + if (n2 == 0) return nodeA; + need = MAP_HEADER_FLATMAP_SZ + 1 + 2 * (n1 + n2); hp = HAlloc(p, need); @@ -1127,6 +1136,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) mp = (flatmap_t*)flatmap_val(flat); n = flatmap_get_size(mp); + if (n == 0) return tree; ks = flatmap_get_keys(mp); vs = flatmap_get_values(mp); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 7901aa668c..3ac2530068 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -456,6 +456,24 @@ erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, } /** + * + * @brief Send one message from *NOT* a local process. + * + * But with a token! + */ +void +erts_queue_message_token(Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg, Eterm from, Eterm token) +{ + ASSERT(is_not_internal_pid(from)); + ERL_MESSAGE_TERM(mp) = msg; + ERL_MESSAGE_FROM(mp) = from; + ERL_MESSAGE_TOKEN(mp) = token; + queue_messages(receiver, receiver_locks, mp, &mp->next, 1); +} + + +/** * @brief Send one message from a local process. * * It is up to the caller of this function to set the @@ -674,7 +692,7 @@ erts_send_message(Process* sender, * Make sure we don't use the heap between those instances. */ if (have_seqtrace(stoken)) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 38a2d567d3..6d551a5108 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -447,6 +447,7 @@ void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, ErlHeapFragment *, Eterm, Eterm); void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); +void erts_queue_message_token(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm, Eterm); void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm); void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks, ErtsMessage*, ErtsMessage**, Uint); diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c index 43028be39d..7b3010bab2 100644 --- a/erts/emulator/beam/erl_monitor_link.c +++ b/erts/emulator/beam/erl_monitor_link.c @@ -540,6 +540,7 @@ erts_mon_link_dist_create(Eterm nodename) mld->links = NULL; mld->monitors = NULL; mld->orig_name_monitors = NULL; + mld->dist_pend_spawn_exit = NULL; return mld; } @@ -551,6 +552,7 @@ erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld) ERTS_ML_ASSERT(!mld->links); ERTS_ML_ASSERT(!mld->monitors); ERTS_ML_ASSERT(!mld->orig_name_monitors); + ERTS_ML_ASSERT(!mld->dist_pend_spawn_exit); erts_mtx_destroy(&mld->mtx); erts_free(ERTS_ALC_T_ML_DIST, mld); @@ -834,10 +836,29 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name) Uint rsz, osz, tsz; Eterm *hp; ErlOffHeap oh; - Uint16 name_flag = is_nil(name) ? ((Uint16) 0) : ERTS_ML_FLG_NAME; + Uint16 name_flag; + Uint16 pending_flag; rsz = is_immed(ref) ? 0 : size_object(ref); - tsz = is_immed(trgt) ? 0 : size_object(trgt); + if (trgt != am_pending) { + if (is_not_immed(trgt)) + tsz = size_object(trgt); + else + tsz = 0; + pending_flag = (Uint16) 0; + name_flag = is_nil(name) ? ((Uint16) 0) : ERTS_ML_FLG_NAME; + } + else { + /* Pending spawn_request() */ + pending_flag = ERTS_ML_FLG_SPAWN_PENDING; + /* Prepare for storage of exteral pid */ + tsz = EXTERNAL_THING_HEAD_SIZE + 1; + /* name contains tag */ + + /* Not by name */ + name_flag = (Uint16) 0; + + } if (type == ERTS_MON_TYPE_RESOURCE) osz = 0; else @@ -851,6 +872,16 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name) hp = &mdep->heap[0]; + if (pending_flag) { + /* Make room for the future pid... */ +#ifdef DEBUG + int i; + for (i = 0; i < EXTERNAL_THING_HEAD_SIZE + 1; i++) + hp[i] = THE_NON_VALUE; +#endif + hp += EXTERNAL_THING_HEAD_SIZE + 1; + } + mdp = &mdep->md; ERTS_ML_ASSERT(((void *) mdp) == ((void *) mdep)); @@ -858,7 +889,7 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name) mdp->origin.other.item = tsz ? copy_struct(trgt, tsz, &hp, &oh) : trgt; mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin); - mdp->origin.flags = ERTS_ML_FLG_EXTENDED|name_flag; + mdp->origin.flags = ERTS_ML_FLG_EXTENDED|name_flag|pending_flag; mdp->origin.type = type; if (type == ERTS_MON_TYPE_RESOURCE) @@ -878,6 +909,25 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name) } else { mdep->u.name = name; + if (pending_flag) { + /* spawn_request() tag is in 'name' */ + if (is_not_immed(name)) { + /* + * Save the tag in its own heap fragment with a + * little trick: + * + * bp->mem[0] = The tag + * bp->mem[1] = Beginning of heap + * mdep->u.name = Countinuation pointer to + * heap fragment... + */ + Uint hsz = size_object(name)+1; + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = &bp->mem[1]; + bp->mem[0] = copy_struct(name, hsz-1, &hp, &bp->off_heap); + mdep->u.name = make_cp((void*)bp); + } + } mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitorData, ref); ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset); @@ -964,6 +1014,20 @@ erts_monitor_destroy__(ErtsMonitorData *mdp) } if (mdep->dist) erts_mon_link_dist_dec_refc(mdep->dist); + if (mdp->origin.flags & ERTS_ML_FLG_SPAWN_PENDING) { + /* + * We have the spawn_request() tag stored in + * mdep->u.name via a little trick + * (see pending_flag in erts_monitor_create()). + * If non-immediate value make sure to release + * this heap fragment as well. + */ + if (is_not_immed(mdep->u.name)) { + ErlHeapFragment *bp; + bp = (ErlHeapFragment *) cp_val(mdep->u.name); + free_message_buffer(bp); + } + } erts_free(ERTS_ALC_T_MONITOR_EXT, mdp); } } @@ -1248,11 +1312,26 @@ erts_link_create(Uint16 type, Eterm a, Eterm b) Eterm *hp; ErlOffHeap oh; - if (is_internal_pid(a)) - hsz = NC_HEAP_SIZE(b); - else - hsz = NC_HEAP_SIZE(a); - ERTS_ML_ASSERT(hsz > 0); + hsz = EXTERNAL_THING_HEAD_SIZE + 1; + if (hsz < ERTS_REF_THING_SIZE + && (is_internal_ordinary_ref(a) + || is_internal_ordinary_ref(b))) { + hsz = ERTS_REF_THING_SIZE; + } + +#ifdef DEBUG + if (is_internal_pid(a)) { + ERTS_ML_ASSERT(is_external_pid(b) + || is_internal_ordinary_ref(b)); + ERTS_ML_ASSERT(NC_HEAP_SIZE(b) <= hsz); + } + else { + ERTS_ML_ASSERT(is_internal_pid(b)); + ERTS_ML_ASSERT(is_external_pid(a) + || is_internal_ordinary_ref(a)); + ERTS_ML_ASSERT(NC_HEAP_SIZE(a) <= hsz); + } +#endif size = sizeof(ErtsLinkDataExtended) - sizeof(Eterm); size += hsz*sizeof(Eterm); diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h index 86be400c09..e10bb27077 100644 --- a/erts/emulator/beam/erl_monitor_link.h +++ b/erts/emulator/beam/erl_monitor_link.h @@ -431,9 +431,22 @@ #define ERTS_ML_FLG_IN_SUBTABLE (((Uint16) 1) << 2) #define ERTS_ML_FLG_NAME (((Uint16) 1) << 3) #define ERTS_ML_FLG_EXTENDED (((Uint16) 1) << 4) +#define ERTS_ML_FLG_SPAWN_PENDING (((Uint16) 1) << 5) +#define ERTS_ML_FLG_SPAWN_MONITOR (((Uint16) 1) << 6) +#define ERTS_ML_FLG_SPAWN_LINK (((Uint16) 1) << 7) +#define ERTS_ML_FLG_SPAWN_ABANDONED (((Uint16) 1) << 8) +#define ERTS_ML_FLG_SPAWN_NO_SMSG (((Uint16) 1) << 9) +#define ERTS_ML_FLG_SPAWN_NO_EMSG (((Uint16) 1) << 10) #define ERTS_ML_FLG_DBG_VISITED (((Uint16) 1) << 15) +#define ERTS_ML_FLGS_SPAWN (ERTS_ML_FLG_SPAWN_PENDING \ + | ERTS_ML_FLG_SPAWN_MONITOR \ + | ERTS_ML_FLG_SPAWN_LINK \ + | ERTS_ML_FLG_SPAWN_ABANDONED \ + | ERTS_ML_FLG_SPAWN_NO_SMSG \ + | ERTS_ML_FLG_SPAWN_NO_EMSG) + /* Flags that should be the same on both monitor/link halves */ #define ERTS_ML_FLGS_SAME \ (ERTS_ML_FLG_EXTENDED|ERTS_ML_FLG_NAME) @@ -478,6 +491,7 @@ typedef struct { ErtsMonLnkNode *monitors; /* Monitor double linked circular list */ ErtsMonLnkNode *orig_name_monitors; /* Origin named monitors read-black tree */ + ErtsMonLnkNode *dist_pend_spawn_exit; } ErtsMonLnkDist; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -1397,7 +1411,7 @@ erts_monitor_release(ErtsMonitor *mon) ErtsMonitorData *mdp = erts_monitor_to_data(mon); ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) > 0); - if (erts_atomic32_dec_read_nob(&mdp->refc) == 0) { + if (erts_atomic32_dec_read_mb(&mdp->refc) == 0) { ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); @@ -1412,7 +1426,7 @@ erts_monitor_release_both(ErtsMonitorData *mdp) == (mdp->target.flags & ERTS_ML_FLGS_SAME)); ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) >= 2); - if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0) { + if (erts_atomic32_add_read_mb(&mdp->refc, (erts_aint32_t) -2) == 0) { ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c index b8cf2bee0e..8263a6e9b7 100644 --- a/erts/emulator/beam/erl_nfunc_sched.c +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -30,77 +30,37 @@ #include "erl_nfunc_sched.h" #include "erl_trace.h" -NifExport * -erts_new_proc_nif_export(Process *c_p, int argc) +ErtsNativeFunc * +erts_new_proc_nfunc(Process *c_p, int argc) { + ErtsNativeFunc *nep, *old_nep; size_t size; - int i; - NifExport *nep, *old_nep; - - size = sizeof(NifExport) + (argc-1)*sizeof(Eterm); - nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size); - for (i = 0; i < ERTS_NUM_CODE_IX; i++) - nep->exp.addressv[i] = &nep->exp.beam[0]; + size = sizeof(ErtsNativeFunc) + (argc-1)*sizeof(Eterm); + nep = erts_alloc(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, size); nep->argc = -1; /* unused marker */ nep->argv_size = argc; - nep->trace = NULL; - old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); + old_nep = ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(c_p, nep); if (old_nep) { - ASSERT(!nep->trace); - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + erts_free(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, old_nep); } return nep; } void -erts_destroy_nif_export(Process *p) +erts_destroy_nfunc(Process *p) { - NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + ErtsNativeFunc *nep = ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(p, NULL); if (nep) { if (nep->m) - erts_nif_export_cleanup_nif_mod(nep); - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep); + erts_nfunc_cleanup_nif_mod(nep); + erts_free(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, nep); } } -void -erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - NifExportTrace *netp; - ASSERT(nep && nep->argc >= 0); - ASSERT(!nep->trace); - netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE, - sizeof(NifExportTrace)); - netp->applying = applying; - netp->ep = ep; - netp->cp = cp; - netp->flags = flags; - netp->flags_meta = flags_meta; - netp->I = I; - netp->meta_tracer = NIL; - erts_tracer_update(&netp->meta_tracer, meta_tracer); - nep->trace = netp; -} - -void -erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep) -{ - NifExportTrace *netp = nep->trace; - nep->trace = NULL; - erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep, - netp->cp, netp->flags, netp->flags_meta, - netp->I, netp->meta_tracer); - erts_tracer_update(&netp->meta_tracer, NIL); - erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp); -} - -NifExport * -erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, +ErtsNativeFunc * +erts_nfunc_schedule(Process *c_p, Process *dirty_shadow_proc, ErtsCodeMFA *mfa, BeamInstr *pc, BeamInstr instr, void *dfunc, void *ifunc, @@ -110,7 +70,7 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, Process *used_proc; ErtsSchedulerData *esdp; Eterm* reg; - NifExport* nep; + ErtsNativeFunc* nep; int i; ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) @@ -133,10 +93,10 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, reg = esdp->x_reg_array; if (mfa) - nep = erts_get_proc_nif_export(c_p, (int) mfa->arity); + nep = erts_get_proc_nfunc(c_p, (int) mfa->arity); else { /* If no mfa, this is not the first schedule... */ - nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); ASSERT(nep && nep->argc >= 0); } @@ -148,16 +108,15 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, for (i = 0; i < (int) mfa->arity; i++) nep->argv[i] = reg[i]; nep->pc = pc; - nep->cp = c_p->cp; nep->mfa = mfa; nep->current = c_p->current; ASSERT(argc >= 0); nep->argc = (int) mfa->arity; nep->m = NULL; - ASSERT(!erts_check_nif_export_in_area(c_p, + ASSERT(!erts_check_nfunc_in_area(c_p, (char *) nep, - (sizeof(NifExport) + (sizeof(ErtsNativeFunc) + (sizeof(Eterm) *(nep->argc-1))))); } @@ -167,14 +126,14 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, reg[i] = argv[i]; } ASSERT(is_atom(mod) && is_atom(func)); - nep->exp.info.mfa.module = mod; - nep->exp.info.mfa.function = func; - nep->exp.info.mfa.arity = (Uint) argc; - nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */ - nep->exp.beam[1] = (BeamInstr) dfunc; + nep->trampoline.info.mfa.module = mod; + nep->trampoline.info.mfa.function = func; + nep->trampoline.info.mfa.arity = (Uint) argc; + nep->trampoline.call_op = (BeamInstr) instr; /* call_bif || call_nif */ + nep->trampoline.dfunc = (BeamInstr) dfunc; nep->func = ifunc; used_proc->arity = argc; used_proc->freason = TRAP; - used_proc->i = (BeamInstr*) nep->exp.addressv[0]; + used_proc->i = (BeamInstr*)&nep->trampoline.call_op; return nep; } diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 1cb252eba5..4dae242d4f 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -25,92 +25,78 @@ #include "bif.h" #include "error.h" -typedef struct { - int applying; - Export* ep; - BeamInstr *cp; - Uint32 flags; - Uint32 flags_meta; - BeamInstr* I; - ErtsTracer meta_tracer; -} NifExportTrace; - /* - * NIF exports need a few more items than the Export struct provides, - * including the erl_module_nif* and a NIF function pointer, so the - * NifExport below adds those. The Export member must be first in the - * struct. A number of values are stored for error handling purposes - * only. + * Native function wrappers are used to schedule native functions on both + * normal and dirty schedulers. + * + * A number of values are only stored for error handling, and the fields + * following `current` can be omitted when a wrapper is statically "scheduled" + * through placement in a function stub. * - * 'argc' is >= 0 when NifExport is in use, and < 0 when not. + * 'argc' is >= 0 when ErtsNativeFunc is in use, and < 0 when not. */ typedef struct { - Export exp; + struct { + ErtsCodeInfo info; + BeamInstr call_op; /* call_bif || call_nif */ + BeamInstr dfunc; + } trampoline; + struct erl_module_nif* m; /* NIF module, or NULL if BIF */ void *func; /* Indirect NIF or BIF to execute (may be unused) */ ErtsCodeMFA *current;/* Current as set when originally called */ - NifExportTrace *trace; /* --- The following is only used on error --- */ BeamInstr *pc; /* Program counter */ - BeamInstr *cp; /* Continuation pointer */ ErtsCodeMFA *mfa; /* MFA of original call */ int argc; /* Number of arguments in original call */ int argv_size; /* Allocated size of argv */ Eterm argv[1]; /* Saved arguments from the original call */ -} NifExport; - -NifExport *erts_new_proc_nif_export(Process *c_p, int argc); -void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); -void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep); -void erts_destroy_nif_export(Process *p); -NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, - ErtsCodeMFA *mfa, BeamInstr *pc, - BeamInstr instr, - void *dfunc, void *ifunc, - Eterm mod, Eterm func, - int argc, const Eterm *argv); -void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */ -ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra); -ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, - Uint* nobj); -ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, - char *start, Uint size); -ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep, - Eterm result); -ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, - Eterm *reg, ErtsCodeMFA **nif_mfa); -ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result, - int applying, Export* ep, - BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); +} ErtsNativeFunc; + +ErtsNativeFunc *erts_new_proc_nfunc(Process *c_p, int argc); +void erts_destroy_nfunc(Process *p); +ErtsNativeFunc *erts_nfunc_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv); +void erts_nfunc_cleanup_nif_mod(ErtsNativeFunc *ep); /* erl_nif.c */ +ERTS_GLB_INLINE ErtsNativeFunc *erts_get_proc_nfunc(Process *c_p, int extra); +ERTS_GLB_INLINE int erts_setup_nfunc_rootset(Process* proc, Eterm** objv, + Uint* nobj); +ERTS_GLB_INLINE int erts_check_nfunc_in_area(Process *p, + char *start, Uint size); +ERTS_GLB_INLINE void erts_nfunc_restore(Process *c_p, ErtsNativeFunc *ep, + Eterm result); +ERTS_GLB_INLINE void erts_nfunc_restore_error(Process* c_p, + BeamInstr **pc, + Eterm *reg, + ErtsCodeMFA **nif_mfa); ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE NifExport * -erts_get_proc_nif_export(Process *c_p, int argc) +ERTS_GLB_INLINE ErtsNativeFunc * +erts_get_proc_nfunc(Process *c_p, int argc) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); if (!nep || (nep->argc < 0 && nep->argv_size < argc)) - return erts_new_proc_nif_export(c_p, argc); + return erts_new_proc_nfunc(c_p, argc); return nep; } /* * If a process has saved arguments, they need to be part of the GC * rootset. The function below is called from setup_rootset() in - * erl_gc.c. Any exception term saved in the NifExport is also made + * erl_gc.c. Any exception term saved in the ErtsNativeFunc is also made * part of the GC rootset here; it always resides in rootset[0]. */ ERTS_GLB_INLINE int -erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) +erts_setup_nfunc_rootset(Process* proc, Eterm** objv, Uint* nobj) { - NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ErtsNativeFunc* ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); if (!ep || ep->argc <= 0) return 0; @@ -121,18 +107,16 @@ erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) } /* - * Check if nif export points into code area... + * Check if native func wrapper points into code area... */ ERTS_GLB_INLINE int -erts_check_nif_export_in_area(Process *p, char *start, Uint size) +erts_check_nfunc_in_area(Process *p, char *start, Uint size) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(p); if (!nep || nep->argc < 0) return 0; if (ErtsInArea(nep->pc, start, size)) return 1; - if (ErtsInArea(nep->cp, start, size)) - return 1; if (ErtsInArea(nep->mfa, start, size)) return 1; if (ErtsInArea(nep->current, start, size)) @@ -141,7 +125,7 @@ erts_check_nif_export_in_area(Process *p, char *start, Uint size) } ERTS_GLB_INLINE void -erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) +erts_nfunc_restore(Process *c_p, ErtsNativeFunc *ep, Eterm result) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); ERTS_LC_ASSERT(!(c_p->static_flags @@ -151,43 +135,21 @@ erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) c_p->current = ep->current; ep->argc = -1; /* Unused nif-export marker... */ - if (ep->trace) - erts_nif_export_restore_trace(c_p, result, ep); } ERTS_GLB_INLINE void -erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, +erts_nfunc_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, ErtsCodeMFA **nif_mfa) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); int ix; ASSERT(nep); *pc = nep->pc; - c_p->cp = nep->cp; *nif_mfa = nep->mfa; for (ix = 0; ix < nep->argc; ix++) reg[ix] = nep->argv[ix]; - erts_nif_export_restore(c_p, nep, THE_NON_VALUE); -} - -ERTS_GLB_INLINE int -erts_nif_export_check_save_trace(Process *c_p, Eterm result, - int applying, Export* ep, - BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - if (is_non_value(result) && c_p->freason == TRAP) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); - if (nep && nep->argc >= 0) { - erts_nif_export_save_trace(c_p, nep, applying, ep, - cp, flags, flags_meta, - I, meta_tracer); - return 1; - } - } - return 0; + erts_nfunc_restore(c_p, nep, THE_NON_VALUE); } ERTS_GLB_INLINE Process * @@ -210,10 +172,10 @@ erts_proc_shadow2real(Process *c_p) #if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) #define ERTS_NFUNC_SCHED_INTERNALS__ -#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ - (ASSERT(BeamIsOpCode(*(I), op_apply_bif) || \ - BeamIsOpCode(*(I), op_call_nif)), \ - ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) +#define ERTS_I_BEAM_OP_TO_NFUNC(I) \ + (ASSERT(BeamIsOpCode(*(I), op_call_bif_W) || \ + BeamIsOpCode(*(I), op_call_nif_WWW)), \ + ((ErtsNativeFunc *) (((char *) (I)) - offsetof(ErtsNativeFunc, trampoline.call_op)))) #include "erl_message.h" diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 27959ba2bb..71165f5ff8 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -74,12 +74,25 @@ * 'handle'. */ struct erl_module_nif { + erts_refc_t refc; /* References to this struct + * +1 erl_module_instance (loaded Erlang code) + * +1 "dlopen" (loaded native code) + * +1 scheduled load finisher + * +1 for each owned resource type + */ + erts_mtx_t load_mtx; /* protects load finish from unload */ + struct ErtsNifFinish_* finish; + ErtsThrPrgrLaterOp lop; + void* priv_data; void* handle; /* "dlopen" */ struct enif_entry_t entry; - erts_refc_t rt_cnt; /* number of resource types */ - erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ - Module* mod; /* Can be NULL if orphan with dtor-resources left */ + erts_refc_t dynlib_refc; /* References to loaded native code + +1 erl_module_instance + +1 for each owned resource type with callbacks + +1 for each ongoing dirty NIF call + */ + Module* mod; /* Can be NULL if purged and dynlib_refc > 0 */ ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; @@ -309,10 +322,10 @@ void erts_post_nif(ErlNifEnv* env) /* - * Initialize a NifExport struct. Create it if needed and store it in the + * Initialize a ErtsNativeFunc struct. Create it if needed and store it in the * proc. The direct_fp function is what will be invoked by op_call_nif, and * the indirect_fp function, if not NULL, is what the direct_fp function - * will call. If the allocated NifExport isn't enough to hold all of argv, + * will call. If the allocated ErtsNativeFunc isn't enough to hold all of argv, * allocate a larger one. Save 'current' and registers if first time this * call is scheduled. */ @@ -321,7 +334,7 @@ static ERTS_INLINE ERL_NIF_TERM schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - NifExport *ep; + ErtsNativeFunc *ep; Process *c_p, *dirty_shadow_proc; execution_state(env, &c_p, NULL); @@ -332,16 +345,16 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, + ep = erts_nfunc_schedule(c_p, dirty_shadow_proc, c_p->current, - c_p->cp, - BeamOpCodeAddr(op_call_nif), + cp_val(c_p->stop[0]), + BeamOpCodeAddr(op_call_nif_WWW), direct_fp, indirect_fp, mod, func_name, argc, (const Eterm *) argv); if (!ep->m) { /* First time this call is scheduled... */ - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + erts_refc_inc(&env->mod_nif->dynlib_refc, 2); ep->m = env->mod_nif; } return (ERL_NIF_TERM) THE_NON_VALUE; @@ -356,7 +369,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * { int exiting; ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg; - NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsNativeFunc *nep = ERTS_I_BEAM_OP_TO_NFUNC(I); ErtsCodeMFA *codemfa = erts_code_to_codemfa(I); NativeFunPtr dirty_nif = (NativeFunPtr) I[1]; ErlNifEnv env; @@ -364,7 +377,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * #ifdef DEBUG erts_aint32_t state = erts_atomic32_read_nob(&c_p->state); - ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + ASSERT(nep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p)); ASSERT(!c_p->scheduler_data); ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) @@ -815,7 +828,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, } #endif if (have_seqtrace(stoken)) { - seq_trace_update_send(c_p); + seq_trace_update_serial(c_p); seq_trace_output(stoken, msg, SEQ_TRACE_SEND, rp->common.id, c_p); } @@ -2189,7 +2202,10 @@ int enif_vsnprintf(char* buffer, size_t size, const char *format, va_list ap) ** Memory managed (GC'ed) "resource" objects ** ***********************************************************/ -/* dummy node in circular list */ +/* + * Sentinel node in circular list of all resource types. + * List protected by code_write_permission. + */ struct enif_resource_type_t resource_type_list; static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) @@ -2209,16 +2225,26 @@ static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) #define in_area(ptr,start,nbytes) \ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) -static ERTS_INLINE int rt_have_callbacks(ErlNifResourceType* rt) +static ERTS_INLINE int rt_have_callbacks(ErlNifResourceTypeInit* rti) { - return rt->dtor != NULL; + return rti->dtor != NULL; +} + +static void deref_nifmod(struct erl_module_nif* lib) +{ + if (erts_refc_dectest(&lib->refc, 0) == 0) { + ASSERT(lib->handle == NULL); + ASSERT(lib->mod == NULL); + erts_free(ERTS_ALC_T_NIF, lib); + } } -static void close_lib(struct erl_module_nif* lib) +static void close_dynlib(struct erl_module_nif* lib) { ASSERT(lib != NULL); + ASSERT(lib->mod == NULL); ASSERT(lib->handle != NULL); - ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0); + ASSERT(erts_refc_read(&lib->dynlib_refc,0) == 0); if (lib->entry.unload != NULL) { struct enif_msg_environment_t msg_env; @@ -2228,24 +2254,56 @@ static void close_lib(struct erl_module_nif* lib) } if (!erts_is_static_nif(lib->handle)) erts_sys_ddll_close(lib->handle); + lib->handle = NULL; + deref_nifmod(lib); } static void steal_resource_type(ErlNifResourceType* type) { struct erl_module_nif* lib = type->owner; - if (rt_have_callbacks(type) - && erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0 - && lib->mod == NULL) { - /* last type with destructor gone, close orphan lib */ - - close_lib(lib); - } - if (erts_refc_dectest(&lib->rt_cnt, 0) == 0 - && lib->mod == NULL) { - erts_free(ERTS_ALC_T_NIF, lib); + if (rt_have_callbacks(&type->fn_real) + && erts_refc_dectest(&lib->dynlib_refc, 0) == 0) { + /* last resource type with callbacks gone, close purged lib */ + close_dynlib(lib); } + deref_nifmod(lib); +} + +static erts_rwmtx_t erts_nif_call_tab_lock; + +static void resource_dtor_during_takeover(ErlNifEnv* env, void* obj) +{ + ErtsResource* resource = DATA_TO_RESOURCE(obj); + ErlNifResourceType* rt = resource->type; + + erts_rwmtx_rlock(&erts_nif_call_tab_lock); + if (rt->fn_real.dtor) + rt->fn_real.dtor(env, obj); + erts_rwmtx_runlock(&erts_nif_call_tab_lock); +} +static void resource_stop_during_takeover(ErlNifEnv* env, void* obj, + ErlNifEvent e, int is_direct_call) +{ + ErtsResource* resource = DATA_TO_RESOURCE(obj); + ErlNifResourceType* rt = resource->type; + + erts_rwmtx_rlock(&erts_nif_call_tab_lock); + ASSERT(rt->fn_real.stop); + rt->fn_real.stop(env, obj, e, is_direct_call); + erts_rwmtx_runlock(&erts_nif_call_tab_lock); +} +static void resource_down_during_takeover(ErlNifEnv* env, void* obj, + ErlNifPid* pid, ErlNifMonitor* mon) +{ + ErtsResource* resource = DATA_TO_RESOURCE(obj); + ErlNifResourceType* rt = resource->type; + + erts_rwmtx_rlock(&erts_nif_call_tab_lock); + ASSERT(rt->fn_real.down); + rt->fn_real.down(env, obj, pid, mon); + erts_rwmtx_runlock(&erts_nif_call_tab_lock); } static void resource_dtor_nop(ErlNifEnv* env, void* obj) @@ -2278,7 +2336,7 @@ ErlNifResourceType* open_resource_type(ErlNifEnv* env, ErlNifResourceFlags op = flags; Eterm module_am, name_am; - ASSERT(erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); module_am = make_atom(env->mod_nif->mod->module); name_am = enif_make_atom(env, name_str); @@ -2292,7 +2350,8 @@ ErlNifResourceType* open_resource_type(ErlNifEnv* env, erts_refc_init(&type->refc, 1); op = ERL_NIF_RT_CREATE; #ifdef DEBUG - type->dtor = (void*)1; + type->fn.dtor = (void*)1; + type->fn_real.dtor = (void*)1; type->owner = (void*)2; type->prev = (void*)3; type->next = (void*)4; @@ -2355,14 +2414,16 @@ enif_open_resource_type_x(ErlNifEnv* env, env->mod_nif->entry.sizeof_ErlNifResourceTypeInit); } -static void commit_opened_resource_types(struct erl_module_nif* lib) +static void prepare_opened_rt(struct erl_module_nif* lib) { - while (opened_rt_list) { - struct opened_resource_type* ort = opened_rt_list; + struct opened_resource_type* ort = opened_rt_list; + while (ort) { ErlNifResourceType* type = ort->type; if (ort->op == ERL_NIF_RT_CREATE) { + type->fn = ort->new_callbacks; + type->fn_real = ort->new_callbacks; type->prev = &resource_type_list; type->next = resource_type_list.next; type->next->prev = type; @@ -2370,20 +2431,55 @@ static void commit_opened_resource_types(struct erl_module_nif* lib) } else { /* ERL_NIF_RT_TAKEOVER */ steal_resource_type(type); + + /* + * Prepare for atomic change of callbacks with lock-wrappers + */ + type->fn.dtor = resource_dtor_during_takeover; + type->fn.stop = resource_stop_during_takeover; + type->fn.down = resource_down_during_takeover; } + type->owner = lib; - type->owner = lib; - type->dtor = ort->new_callbacks.dtor; - type->stop = ort->new_callbacks.stop; - type->down = ort->new_callbacks.down; + if (rt_have_callbacks(&ort->new_callbacks)) + erts_refc_inc(&lib->dynlib_refc, 2); + erts_refc_inc(&lib->refc, 2); - if (rt_have_callbacks(type)) { - erts_refc_inc(&lib->rt_dtor_cnt, 1); - } - erts_refc_inc(&lib->rt_cnt, 1); + ort = ort->next; + } +} - opened_rt_list = ort->next; - erts_free(ERTS_ALC_T_TMP, ort); +/* + * Do atomic change from old to new callbacks + */ +static void commit_opened_rt(void) +{ + struct opened_resource_type* ort = opened_rt_list; + + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_nif_call_tab_lock)); + + while (ort) { + if (ort->op == ERL_NIF_RT_TAKEOVER) { + ort->type->fn_real = ort->new_callbacks; + } + ort = ort->next; + } +} + +/* + * Cleanup and let callbacks be called directly without locking + */ +static void cleanup_opened_rt(void) +{ + struct opened_resource_type* ort = opened_rt_list; + + while (opened_rt_list) { + ort = opened_rt_list; + if (ort->op == ERL_NIF_RT_TAKEOVER) { + ort->type->fn = ort->new_callbacks; + } + opened_rt_list = ort->next; + erts_free(ERTS_ALC_T_TMP, ort); } } @@ -2494,7 +2590,7 @@ static void run_resource_dtor(void* vbin) ErtsMonitor *root; Uint refc; - ASSERT(type->down); + ASSERT(type->fn.down); erts_mtx_lock(&rm->lock); ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0); kill = !rmon_is_dying(rm); @@ -2523,10 +2619,10 @@ static void run_resource_dtor(void* vbin) erts_mtx_destroy(&rm->lock); } - if (type->dtor != NULL) { + if (type->fn.dtor != NULL) { struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, type->owner, NULL); - type->dtor(&msg_env.env, resource->data); + type->fn.dtor(&msg_env.env, resource->data); post_nif_noproc(&msg_env); } if (erts_refc_dectest(&type->refc, 0) == 0) { @@ -2543,9 +2639,9 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, int is_direct_call) { struct enif_msg_environment_t msg_env; - ASSERT(resource->type->stop); + ASSERT(resource->type->fn.stop); pre_nif_noproc(&msg_env, resource->type->owner, NULL); - resource->type->stop(&msg_env.env, resource->data, e, is_direct_call); + resource->type->fn.stop(&msg_env.env, resource->data, e, is_direct_call); post_nif_noproc(&msg_env); } @@ -2556,7 +2652,7 @@ void erts_nif_demonitored(ErtsResource* resource) int free_me; ASSERT(rmp); - ASSERT(resource->type->down); + ASSERT(resource->type->fn.down); erts_mtx_lock(&rmp->lock); free_me = ((rmon_refc_dec_read(rmp) == 0) & !!rmon_is_dying(rmp)); @@ -2590,7 +2686,7 @@ void erts_fire_nif_monitor(ErtsMonitor *tmon) omon = &mdp->origin; ASSERT(rmp); - ASSERT(resource->type->down); + ASSERT(resource->type->fn.down); erts_mtx_lock(&rmp->lock); @@ -2617,7 +2713,7 @@ void erts_fire_nif_monitor(ErtsMonitor *tmon) erts_ref_to_driver_monitor(mdp->ref, &nif_monitor); nif_pid.pid = omon->other.item; pre_nif_noproc(&msg_env, resource->type->owner, NULL); - resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); + resource->type->fn.down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); post_nif_noproc(&msg_env); erts_bin_release(&bin->binary); @@ -2634,7 +2730,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) ErtsResource* resource; size_t monitors_offs; - if (type->down) { + if (type->fn.down) { /* Put ErtsResourceMonitors after user data and properly aligned */ monitors_offs = ((data_sz + ERTS_ALLOC_ALIGN_BYTES - 1) & ~((size_t)ERTS_ALLOC_ALIGN_BYTES - 1)); @@ -2656,7 +2752,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) erts_refc_init(&resource->nif_refc, 1); #endif erts_refc_inc(&resource->type->refc, 2); - if (type->down) { + if (type->fn.down) { resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs); erts_mtx_init(&resource->monitors->lock, "resource_monitors", NIL, ERTS_LOCK_FLAGS_CATEGORY_GENERIC); @@ -2838,25 +2934,25 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) } static ERTS_INLINE void -nif_export_cleanup_nif_mod(NifExport *ep) +nfunc_cleanup_nif_mod(ErtsNativeFunc *ep) { - if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL) - close_lib(ep->m); + if (erts_refc_dectest(&ep->m->dynlib_refc, 0) == 0) + close_dynlib(ep->m); ep->m = NULL; } void -erts_nif_export_cleanup_nif_mod(NifExport *ep) +erts_nfunc_cleanup_nif_mod(ErtsNativeFunc *ep) { - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); } static ERTS_INLINE void -nif_export_restore(Process *c_p, NifExport *ep, Eterm res) +nfunc_restore(Process *c_p, ErtsNativeFunc *ep, Eterm res) { - erts_nif_export_restore(c_p, ep, res); + erts_nfunc_restore(c_p, ep, res); ASSERT(ep->m); - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); } @@ -2873,15 +2969,15 @@ static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NifExport* ep; + ErtsNativeFunc* ep; execution_state(env, &proc, NULL); ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); ASSERT(ep); - nif_export_restore(proc, ep, argv[0]); + nfunc_restore(proc, ep, argv[0]); return argv[0]; } @@ -2893,21 +2989,22 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ERL_NIF_TERM ret; Process* proc; - NifExport* ep; + ErtsNativeFunc* ep; Eterm exception; execution_state(env, &proc, NULL); ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); ASSERT(ep); exception = argv[0]; /* argv overwritten by restore below... */ - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); ret = enif_raise_exception(env, exception); - /* Restore orig info for error and clear nif export in handle_error() */ - proc->freason |= EXF_RESTORE_NIF; + /* Restore orig info for error and clear native func wrapper in + * handle_error() */ + proc->freason |= EXF_RESTORE_NFUNC; return ret; } @@ -2944,7 +3041,7 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, int argc, const ERL_NIF_TERM argv[]) { Process *proc; - NifExport *ep; + ErtsNativeFunc *ep; Eterm mod, func; NativeFunPtr fp; @@ -2954,12 +3051,11 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, * Called in order to schedule statically determined * dirty NIF calls... * - * Note that 'current' does not point into a NifExport - * structure; only a structure with similar - * parts (located in code). + * Note that 'current' does not point into a ErtsNativeFunc + * structure; only a structure with similar parts (located in code). */ - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + ep = ErtsContainerStruct(proc->current, ErtsNativeFunc, trampoline.info.mfa); mod = proc->current->module; func = proc->current->function; fp = (NativeFunPtr) ep->func; @@ -2987,7 +3083,6 @@ static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } - /* * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It * calls the actual NIF, restores original NIF MFA if necessary, and @@ -2998,12 +3093,12 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { Process* proc; NativeFunPtr fp; - NifExport* ep; + ErtsNativeFunc* ep; ERL_NIF_TERM result; execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + ep = ErtsContainerStruct(proc->current, ErtsNativeFunc, trampoline.info.mfa); fp = ep->func; ASSERT(ep); ASSERT(!env->exception_thrown); @@ -3016,20 +3111,20 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) result = (*fp)(env, argc, argv); - ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc)); + ASSERT(ep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc)); if (is_value(result) || proc->freason != TRAP) { /* Done (not rescheduled)... */ ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); if (!env->exception_thrown) - nif_export_restore(proc, ep, result); + nfunc_restore(proc, ep, result); else { - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); /* * Restore orig info for error and clear nif * export in handle_error() */ - proc->freason |= EXF_RESTORE_NIF; + proc->freason |= EXF_RESTORE_NFUNC; } } @@ -3439,14 +3534,14 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor == NIF_RESOURCE_DTOR); ASSERT(erts_refc_read(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->binary.intern.refc, 0) != 0); - ASSERT(!rsrc->monitors == !rsrc->type->down); + ASSERT(!rsrc->monitors == !rsrc->type->fn.down); rm = rsrc->monitors; if (!rm) { - ASSERT(!rsrc->type->down); + ASSERT(!rsrc->type->fn.down); return -1; } - ASSERT(rsrc->type->down); + ASSERT(rsrc->type->fn.down); if (target_pid->pid == am_undefined) return 1; @@ -3996,7 +4091,7 @@ void erts_add_taint(Eterm mod_atom) struct tainted_module_t *first, *t; ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) - || erts_thr_progress_is_blocking()); + || erts_has_code_write_permission()); first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint); for (t=first ; t; t=t->next) { @@ -4008,7 +4103,7 @@ void erts_add_taint(Eterm mod_atom) if (t != NULL) { t->module_atom = mod_atom; t->next = first; - erts_atomic_set_nob(&first_taint, (erts_aint_t)t); + erts_atomic_set_relb(&first_taint, (erts_aint_t)t); } } @@ -4019,7 +4114,7 @@ Eterm erts_nif_taints(Process* p) Eterm list = NIL; Eterm* hp; - first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint); + first = (struct tainted_module_t*) erts_atomic_read_acqb(&first_taint); for (t=first ; t!=NULL; t=t->next) { cnt++; } @@ -4091,6 +4186,8 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src) bytes += src->num_of_funcs * sizeof(ErlNifFunc); lib = erts_alloc(ERTS_ALC_T_NIF, bytes); + erts_mtx_init(&lib->load_mtx, "nif_load", NIL, + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); dst = &lib->entry; sys_memcpy(dst, src, offsetof(ErlNifEntry, vm_variant)); @@ -4132,8 +4229,89 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src) return lib; }; +/* load_nif/2 is implemented as an instruction as it needs to know where it + * was called from, and it's a pain to get that information in a BIF. + * + * This is a small stub that rejects apply(erlang, load_nif, [Path, Args]). */ +BIF_RETTYPE load_nif_2(BIF_ALIST_2) { + if (BIF_P->flags & F_HIPE_MODE) { + BIF_RET(load_nif_error(BIF_P, "notsup", + "Calling load_nif from HiPE compiled modules " + "not supported")); + } + + BIF_RET(load_nif_error(BIF_P, "bad_lib", + "load_nif/2 must be explicitly called from the NIF " + "module. It cannot be called through apply/3.")); +} + +typedef struct { + HashBucket bucket; + ErtsCodeInfo* code_info; + ErtsCodeMFA mfa; + BeamInstr beam[4]; +} ErtsNifBeamStub; + +typedef struct ErtsNifFinish_ { + int nstubs_hashed; /* can be less than 'num_of_funcs' if load failed */ + ErtsNifBeamStub beam_stubv[1]; +} ErtsNifFinish; + +#define sizeof_ErtsNifFinish(N) \ + (offsetof(ErtsNifFinish, beam_stubv) + (N)*sizeof(ErtsNifBeamStub)) + +static void load_nif_1st_finisher(void* vlib); +static void load_nif_2nd_finisher(void* vlib); +static void load_nif_3rd_finisher(void* vlib); +static void release_beam_stubs(struct erl_module_nif* lib); +static void erase_hashed_stubs(ErtsNifFinish*); + +struct hash erts_nif_call_tab; + +static HashValue nif_call_hash(ErtsNifBeamStub* obj) +{ + return ((HashValue)obj->code_info / sizeof(BeamInstr)); +} + +static int nif_call_cmp(ErtsNifBeamStub* tmpl, ErtsNifBeamStub* obj) +{ + return tmpl->code_info != obj->code_info; +} + +static ErtsNifBeamStub* nif_call_alloc(ErtsNifBeamStub* tmpl) +{ + return tmpl; +} + +static void nif_call_free(ErtsNifBeamStub* obj) +{ +} + +static void nif_call_table_init(void) +{ + HashFunctions f; + + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; + + erts_rwmtx_init_opt(&erts_nif_call_tab_lock, &rwmtx_opt, "nif_call_tab", + NIL, (ERTS_LOCK_FLAGS_PROPERTY_STATIC | + ERTS_LOCK_FLAGS_CATEGORY_GENERIC)); + + f.hash = (H_FUN) nif_call_hash; + f.cmp = (HCMP_FUN) nif_call_cmp; + f.alloc = (HALLOC_FUN) nif_call_alloc; + f.free = (HFREE_FUN) nif_call_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_NIF, &erts_nif_call_tab, "nif_call_tab", 100, f); +} + +static void patch_call_nif_early(ErlNifEntry*, struct erl_module_instance*); -BIF_RETTYPE load_nif_2(BIF_ALIST_2) +Eterm erts_load_nif(Process *c_p, BeamInstr *I, Eterm filename, Eterm args) { static const char bad_lib[] = "bad_lib"; static const char upgrade[] = "upgrade"; @@ -4156,42 +4334,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; - if (BIF_P->flags & F_HIPE_MODE) { - ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " - "modules not supported"); - BIF_RET(ret); - } - encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ /* since lib_name is used in error messages */ encoding = ERL_FILENAME_UTF8; } - lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0, + lib_name = erts_convert_filename_to_encoding(filename, NULL, 0, ERTS_ALC_T_TMP, 1, 0, encoding, NULL, 0); if (!lib_name) { - BIF_ERROR(BIF_P, BADARG); - } - - if (!erts_try_seize_code_write_permission(BIF_P)) { - erts_free(ERTS_ALC_T_TMP, lib_name); - ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); + return THE_NON_VALUE; } - /* Block system (is this the right place to do it?) */ - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_thr_progress_block(); - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - /* Find calling module */ - ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current->module == am_erlang - && BIF_P->current->function == am_load_nif - && BIF_P->current->arity == 2); - caller = find_function_from_pc(BIF_P->cp); + caller = find_function_from_pc(I); ASSERT(caller != NULL); mod_atom = caller->module; ASSERT(is_atom(mod_atom)); @@ -4211,7 +4368,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = &module_p->curr; prev_mi = &module_p->old; if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { - ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " + ret = load_nif_error(c_p, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); goto error; } else if (module_p->on_load) { @@ -4225,52 +4382,52 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } if (this_mi->nif != NULL) { - ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + ret = load_nif_error(c_p,"reload","NIF library already loaded" " (reload disallowed since OTP 20)."); } else if (init_func == NULL && (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { - ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); + ret = load_nif_error(c_p, "load_failed", "%s: '%s'", slogan, errdesc.str); } else { - ret = load_nif_error(BIF_P, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); + ret = load_nif_error(c_p, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); } } else if (init_func == NULL && erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) { - ret = load_nif_error(BIF_P, bad_lib, "Failed to find library init" + ret = load_nif_error(c_p, bad_lib, "Failed to find library init" " function: '%s'", errdesc.str); } else if ((taint ? erts_add_taint(mod_atom) : 0, (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { - ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); + ret = load_nif_error(c_p, bad_lib, "Library init-call unsuccessful"); } else if (entry->major > ERL_NIF_MAJOR_VERSION || (entry->major == ERL_NIF_MAJOR_VERSION && entry->minor > ERL_NIF_MINOR_VERSION)) { char* fmt = "That '%T' NIF library needs %s or newer. Either try to" " recompile the NIF lib or use a newer erts runtime."; - ret = load_nif_error(BIF_P, bad_lib, fmt, mod_atom, entry->min_erts); + ret = load_nif_error(c_p, bad_lib, fmt, mod_atom, entry->min_erts); } else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ char* fmt = "That old NIF library (%d.%d) is not compatible with this " "erts runtime (%d.%d). Try recompile the NIF lib."; - ret = load_nif_error(BIF_P, bad_lib, fmt, entry->major, entry->minor, + ret = load_nif_error(c_p, bad_lib, fmt, entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { - ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " + ret = load_nif_error(c_p, bad_lib, "Library (%s) not compiled for " "this vm variant (%s).", entry->vm_variant, ERL_NIF_VM_VARIANT); } else if (!erts_is_atom_str((char*)entry->name, mod_atom, 1)) { - ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not" + ret = load_nif_error(c_p, bad_lib, "Library module name '%s' does not" " match calling module '%T'", entry->name, mod_atom); } else { @@ -4278,37 +4435,66 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) entry = &lib->entry; /* Use a guaranteed modern lib entry from now on */ lib->handle = handle; - erts_refc_init(&lib->rt_cnt, 0); - erts_refc_init(&lib->rt_dtor_cnt, 0); + erts_refc_init(&lib->refc, 2); /* Erlang code + NIF code */ + erts_refc_init(&lib->dynlib_refc, 1); ASSERT(opened_rt_list == NULL); lib->mod = module_p; - for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { + lib->finish = erts_alloc(ERTS_ALC_T_NIF, + sizeof_ErtsNifFinish(entry->num_of_funcs)); + lib->finish->nstubs_hashed = 0; + + erts_rwmtx_rwlock(&erts_nif_call_tab_lock); + for (i=0; i < entry->num_of_funcs; i++) { ErtsCodeInfo** ci_pp; + ErtsCodeInfo* ci; ErlNifFunc* f = &entry->funcs[i]; + ErtsNifBeamStub* stub = &lib->finish->beam_stubv[i]; + + stub->code_info = NULL; /* end marker in case we fail */ if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { - ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", + ret = load_nif_error(c_p,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); + break; } - else if (f->flags != 0 && - f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && - f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) { - ret = load_nif_error(BIF_P, bad_lib, - "Illegal flags field value %d for NIF %T:%s/%u", + ci = *ci_pp; + + if (f->flags != 0 && + f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && + f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) { + + ret = load_nif_error(c_p, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", f->flags, mod_atom, f->name, f->arity); + break; } - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) - < BEAM_NIF_MIN_FUNC_SZ) - { - ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" - " in module (%T:%s/%u too small)", - mod_atom, f->name, f->arity); - } - /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", - mod_atom, f->name, f->arity);*/ + + ASSERT(erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) + >= BEAM_NATIVE_MIN_FUNC_SZ); + + stub->code_info = ci; + stub->mfa = ci->mfa; + if (hash_put(&erts_nif_call_tab, stub) != stub) { + ret = load_nif_error(c_p, bad_lib, "Duplicate NIF entry for %T:%s/%u", + mod_atom, f->name, f->arity); + break; + } + lib->finish->nstubs_hashed++; + + stub->beam[0] = BeamOpCodeAddr(op_call_nif_WWW); + stub->beam[2] = (BeamInstr) lib; + if (f->flags) { + stub->beam[3] = (BeamInstr) f->fptr; + stub->beam[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + (BeamInstr) static_schedule_dirty_io_nif : + (BeamInstr) static_schedule_dirty_cpu_nif; + } + else + stub->beam[1] = (BeamInstr) f->fptr; } + erts_rwmtx_rwunlock(&erts_nif_call_tab_lock); + ASSERT(lib->finish->nstubs_hashed == lib->entry.num_of_funcs); } if (ret != am_ok) { @@ -4324,67 +4510,71 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ void* prev_old_data = prev_mi->nif->priv_data; if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + ret = load_nif_error(c_p, upgrade, "Upgrade not supported by this NIF library."); goto error; } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + /* + * Go single scheduler during upgrade callback. + * Todo: Fix better solution with suspending callers and waiting for + * all calls to return (including dirty). + * Note that erts_thr_progress_block() will not block dirty NIFs. + */ + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_pre_nif(&env, c_p, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, args); erts_post_nif(&env); + erts_thr_progress_unblock(); if (veto) { prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); + ret = load_nif_error(c_p, upgrade, "Library upgrade-call unsuccessful (%d).", veto); } } else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_pre_nif(&env, c_p, lib, NULL); + veto = entry->load(&env, &lib->priv_data, args); erts_post_nif(&env); if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); + ret = load_nif_error(c_p, "load", "Library load-call unsuccessful (%d).", veto); } } - if (ret == am_ok) { - commit_opened_resource_types(lib); + if (ret == am_ok) { /* - ** Everything ok, patch the beam code with op_call_nif - */ - + * Everything ok, make NIF code callable. + */ this_mi->nif = lib; - for (i=0; i < entry->num_of_funcs; i++) - { - ErlNifFunc* f = &entry->funcs[i]; - ErtsCodeInfo* ci; - BeamInstr *code_ptr; + prepare_opened_rt(lib); + /* + * The call table lock will make sure all NIFs and callbacks in module + * are made accessible atomically. + */ + erts_rwmtx_rwlock(&erts_nif_call_tab_lock); + commit_opened_rt(); + patch_call_nif_early(entry, this_mi); + erts_rwmtx_rwunlock(&erts_nif_call_tab_lock); - erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); - code_ptr = erts_codeinfo_to_code(ci); + cleanup_opened_rt(); - if (ci->u.gen_bp == NULL) { - code_ptr[0] = BeamOpCodeAddr(op_call_nif); - } - else { /* Function traced, patch the original instruction word */ - GenericBp* g = ci->u.gen_bp; - ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)); - g->orig_instr = BeamOpCodeAddr(op_call_nif); - } - if (f->flags) { - code_ptr[3] = (BeamInstr) f->fptr; - code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? - (BeamInstr) static_schedule_dirty_io_nif : - (BeamInstr) static_schedule_dirty_cpu_nif; - } - else - code_ptr[1] = (BeamInstr) f->fptr; - code_ptr[2] = (BeamInstr) lib; - } + /* + * Now we wait thread progress, to make sure no process is still + * executing the beam code of the NIFs, before we can patch in the + * final fast multi word call_nif_WWW instructions. + */ + erts_refc_inc(&lib->refc, 2); + erts_schedule_thr_prgr_later_op(load_nif_1st_finisher, lib, + &lib->lop); } else { error: rollback_opened_resource_types(); ASSERT(ret != am_ok); if (lib != NULL) { + if (lib->finish != NULL) { + erase_hashed_stubs(lib->finish); + erts_free(ERTS_ALC_T_NIF, lib->finish); + } erts_free(ERTS_ALC_T_NIF, lib); } if (handle != NULL && !erts_is_static_nif(handle)) { @@ -4393,25 +4583,208 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_sys_ddll_free_error(&errdesc); } - erts_thr_progress_unblock(); - erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); BIF_RET(ret); } +/* + * Write 'call_nif_early' as the first beam instruction for all NIFs + * which will make them callable. + * + * The 'call_nif_early' is a one word beam instruction which uses a lock + * protected hash lookup to get its "arguments". This guarantees an atomically + * safe publication of all NIFs in the module. + */ +static void patch_call_nif_early(ErlNifEntry* entry, + struct erl_module_instance* this_mi) +{ + int i; + + ERTS_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_nif_call_tab_lock)); + + for (i=0; i < entry->num_of_funcs; i++) + { + ErlNifFunc* f = &entry->funcs[i]; + BeamInstr volatile *code_ptr; + ErtsCodeInfo* ci; + Eterm f_atom; + + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); + ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + code_ptr = erts_codeinfo_to_code(ci); + + if (ci->u.gen_bp) { + /* + * Function traced, patch the original instruction word + * Code write permission protects against racing breakpoint writes. + */ + GenericBp* g = ci->u.gen_bp; + g->orig_instr = BeamOpCodeAddr(op_call_nif_early); + if (BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)) + continue; + } + else + ASSERT(!BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)); + code_ptr[0] = BeamOpCodeAddr(op_call_nif_early); + } +} + +BeamInstr* erts_call_nif_early(Process* c_p, ErtsCodeInfo* ci) +{ + ErtsNifBeamStub* bs; + ErtsNifBeamStub tmpl; + tmpl.code_info = ci; + + erts_rwmtx_rlock(&erts_nif_call_tab_lock); + bs = (ErtsNifBeamStub*) hash_get(&erts_nif_call_tab, &tmpl); + erts_rwmtx_runlock(&erts_nif_call_tab_lock); + + ASSERT(bs); + return &bs->beam[0]; +} + +static void load_nif_1st_finisher(void* vlib) +{ + struct erl_module_nif* lib = (struct erl_module_nif*) vlib; + ErtsNifFinish* fin; + int i; + + erts_mtx_lock(&lib->load_mtx); + fin = lib->finish; + if (fin) { + for (i=0; i < lib->entry.num_of_funcs; i++) { + BeamInstr* code_ptr; + code_ptr = erts_codeinfo_to_code(fin->beam_stubv[i].code_info); + + code_ptr[1] = fin->beam_stubv[i].beam[1]; /* called function */ + code_ptr[2] = fin->beam_stubv[i].beam[2]; /* erl_module_nif */ + if (lib->entry.funcs[i].flags) + code_ptr[3] = fin->beam_stubv[i].beam[3]; /* real NIF */ + } + } + erts_mtx_unlock(&lib->load_mtx); + + if (fin) { + /* + * A second thread progress to get a memory barrier between the + * arguments of call_nif_WWW (written above) and the instruction word + * itself. + */ + erts_schedule_thr_prgr_later_op(load_nif_2nd_finisher, lib, + &lib->lop); + } + else { /* Unloaded */ + deref_nifmod(lib); + } +} + +static void load_nif_2nd_finisher(void* vlib) +{ + struct erl_module_nif* lib = (struct erl_module_nif*) vlib; + ErtsNifFinish* fin; + int i; + + /* + * We seize code write permission only to avoid any trace breakpoints + * to change while we patch the op_call_nif_WWW instruction. + */ + if (!erts_try_seize_code_write_permission_aux(load_nif_2nd_finisher, vlib)) { + return; + } + + erts_mtx_lock(&lib->load_mtx); + fin = lib->finish; + if (fin) { + for (i=0; i < lib->entry.num_of_funcs; i++) { + ErtsCodeInfo *ci = fin->beam_stubv[i].code_info; + BeamInstr volatile *code_ptr = erts_codeinfo_to_code(ci); + + if (ci->u.gen_bp) { + /* + * Function traced, patch the original instruction word + */ + GenericBp* g = ci->u.gen_bp; + ASSERT(g->orig_instr == BeamOpCodeAddr(op_call_nif_early)); + g->orig_instr = BeamOpCodeAddr(op_call_nif_WWW); + if (BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)) + continue; + } + ASSERT(code_ptr[0] == BeamOpCodeAddr(op_call_nif_early)); + code_ptr[0] = BeamOpCodeAddr(op_call_nif_WWW); + } + } + erts_mtx_unlock(&lib->load_mtx); + + erts_release_code_write_permission(); + + if (fin) { + UWord bytes = sizeof_ErtsNifFinish(lib->entry.num_of_funcs); + /* + * A third and final thread progress, to make sure no one is executing + * the call_nif_early instructions anymore, before we can deallocate + * the beam stubs. + */ + erts_schedule_thr_prgr_later_cleanup_op(load_nif_3rd_finisher, lib, + &lib->lop, + bytes); + } + else { /* Unloaded */ + deref_nifmod(lib); + } +} + +static void load_nif_3rd_finisher(void* vlib) +{ + struct erl_module_nif* lib = (struct erl_module_nif*) vlib; + + release_beam_stubs(lib); + deref_nifmod(lib); +} + +static void release_beam_stubs(struct erl_module_nif* lib) +{ + ErtsNifFinish* fin; + + erts_mtx_lock(&lib->load_mtx); + fin = lib->finish; + lib->finish = NULL; + erts_mtx_unlock(&lib->load_mtx); + + if (fin) { + erase_hashed_stubs(fin); + erts_free(ERTS_ALC_T_NIF, fin); + } +} + +static void erase_hashed_stubs(ErtsNifFinish* fin) +{ + int i; + + erts_rwmtx_rwlock(&erts_nif_call_tab_lock); + for (i=0; i < fin->nstubs_hashed; i++) { + void* erased = hash_erase(&erts_nif_call_tab, &fin->beam_stubv[i]); + ASSERT(erased); (void) erased; + } + erts_rwmtx_rwunlock(&erts_nif_call_tab_lock); +} + void erts_unload_nif(struct erl_module_nif* lib) { ErlNifResourceType* rt; ErlNifResourceType* next; - ASSERT(erts_thr_progress_is_blocking()); + ASSERT(lib != NULL); ASSERT(lib->mod != NULL); + ERTS_LC_ASSERT(erts_has_code_write_permission()); erts_tracer_nif_clear(); + release_beam_stubs(lib); + for (rt = resource_type_list.next; rt != &resource_type_list; rt = next) { @@ -4423,25 +4796,19 @@ erts_unload_nif(struct erl_module_nif* lib) rt->next = NULL; rt->prev = NULL; if (erts_refc_dectest(&rt->refc, 0) == 0) { - if (rt_have_callbacks(rt)) { - erts_refc_dec(&lib->rt_dtor_cnt, 0); - } - erts_refc_dec(&lib->rt_cnt, 0); + if (rt_have_callbacks(&rt->fn_real)) + erts_refc_dec(&lib->dynlib_refc, 1); + erts_refc_dec(&lib->refc, 1); erts_free(ERTS_ALC_T_NIF, rt); } } } - if (erts_refc_read(&lib->rt_dtor_cnt, 0) == 0) { - close_lib(lib); - if (erts_refc_read(&lib->rt_cnt, 0) == 0) { - erts_free(ERTS_ALC_T_NIF, lib); - return; - } - } - else { - ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0); - } - lib->mod = NULL; /* orphan lib */ + lib->mod = NULL; /* purged Elang module */ + + if (erts_refc_dectest(&lib->dynlib_refc, 0) == 0) + close_dynlib(lib); + + deref_nifmod(lib); } void erl_nif_init() @@ -4451,11 +4818,13 @@ void erl_nif_init() resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; - resource_type_list.dtor = NULL; + resource_type_list.fn.dtor = NULL; + resource_type_list.fn_real.dtor = NULL; resource_type_list.owner = NULL; resource_type_list.module = THE_NON_VALUE; resource_type_list.name = THE_NON_VALUE; + nif_call_table_init(); } int erts_nif_get_funcs(struct erl_module_nif* mod, diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 51e6a4dc40..abf833f318 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -176,9 +176,10 @@ dist_table_alloc(void *dep_tmpl) erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL); dep->connection_id = 0; dep->state = ERTS_DE_STATE_IDLE; - dep->flags = 0; + dep->pending_nodedown = 0; + dep->suspended_nodeup = NULL; + dep->dflags = 0; dep->opts = 0; - dep->version = 0; dep->mld = NULL; @@ -201,7 +202,6 @@ dist_table_alloc(void *dep_tmpl) erts_port_task_handle_init(&dep->dist_cmd); dep->send = NULL; dep->cache = NULL; - dep->transcode_ctx = NULL; dep->sequences = NULL; /* Link in */ @@ -634,7 +634,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) else { ASSERT(dep->state != ERTS_DE_STATE_IDLE); ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); - if (dep->flags & DFLAG_PUBLISHED) { + if (dep->dflags & DFLAG_PUBLISHED) { ASSERT(erts_no_of_visible_dist_entries > 0); erts_no_of_visible_dist_entries--; head = &erts_visible_dist_entries; @@ -658,7 +658,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) dep->next->prev = dep->prev; dep->state = ERTS_DE_STATE_IDLE; - dep->flags = 0; + dep->dflags = 0; dep->opts = 0; dep->prev = NULL; dep->cid = NIL; @@ -700,7 +700,7 @@ erts_set_dist_entry_pending(DistEntry *dep) erts_no_of_not_connected_dist_entries--; dep->state = ERTS_DE_STATE_PENDING; - dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC); + dep->dflags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_PENDING_CONNECT); dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; ASSERT(!dep->mld); @@ -719,7 +719,7 @@ erts_set_dist_entry_pending(DistEntry *dep) } void -erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) +erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint64 flags) { erts_aint32_t set_qflgs; @@ -731,6 +731,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); ASSERT(dep->state == ERTS_DE_STATE_PENDING); + ASSERT(!dep->pending_nodedown); ASSERT(is_internal_port(cid) || is_internal_pid(cid)); if(dep->prev) { @@ -749,7 +750,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) erts_no_of_pending_dist_entries--; dep->state = ERTS_DE_STATE_CONNECTED; - dep->flags = flags & ~DFLAG_NO_MAGIC; + dep->dflags = flags & ~DFLAG_PENDING_CONNECT; dep->cid = cid; erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) cid); @@ -976,7 +977,7 @@ static void print_node(void *venp, void *vpndp) if(pndp->sysname == NIL) { erts_print(pndp->to, pndp->to_arg, "Name: %T ", enp->sysname); } - erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); + erts_print(pndp->to, pndp->to_arg, " %u", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", erts_refc_read(&enp->refc, 0)); @@ -1019,7 +1020,7 @@ void erts_print_node_info(fmtfn_t to, /* ----------------------------------------------------------------------- */ void -erts_set_this_node(Eterm sysname, Uint creation) +erts_set_this_node(Eterm sysname, Uint32 creation) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2)); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index ffaafbbbea..f426f46d53 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -23,6 +23,7 @@ typedef struct dist_entry_ DistEntry; typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf; +typedef struct ErtsDistOutputBufsContainer_ ErtsDistOutputBufsContainer; void erts_ref_dist_entry(DistEntry *dep); void erts_deref_dist_entry(DistEntry *dep); @@ -95,26 +96,21 @@ enum dist_entry_state { struct ErtsDistOutputBuf_ { #ifdef DEBUG Uint dbg_pattern; - byte *ext_startp; - byte *alloc_endp; #endif ErtsDistOutputBuf *next; Binary *bin; - /* Pointers to the distribution header, - if NULL the distr header is in the extp */ - byte *hdrp; - byte *hdr_endp; - /* Pointers to the ctl + payload */ - byte *extp; - byte *ext_endp; - /* Start of payload and hopefull_flags, used by transcode */ - Uint hopefull_flags; - byte *msg_start; - /* start of the ext buffer, this is not always the same as extp - as the atom cache handling can use less then the allotted buffer. - This value is needed to calculate the size of this output buffer.*/ - byte *ext_start; + /* + * iov[0] reserved for driver + * iov[1] reserved for distribution header + * iov[2 ... vsize-1] data + */ + ErlIOVec *eiov; +}; +struct ErtsDistOutputBufsContainer_ { + Sint fragments; + byte *extp; + ErtsDistOutputBuf obuf[1]; /* longer if fragmented... */ }; typedef struct { @@ -147,10 +143,11 @@ struct dist_entry_ { NIL == free */ Uint32 connection_id; /* Connection id incremented on connect */ enum dist_entry_state state; - Uint32 flags; /* Distribution flags, like hidden, + int pending_nodedown; + Process* suspended_nodeup; + Uint64 dflags; /* Distribution flags, like hidden, atom cache etc. */ Uint32 opts; - unsigned long version; /* Protocol version */ ErtsMonLnkDist *mld; /* Monitors and links */ @@ -173,8 +170,6 @@ struct dist_entry_ { ErtsThrPrgrLaterOp later_op; - struct transcode_context* transcode_ctx; - struct dist_sequences *sequences; /* Ongoing distribution sequences */ }; @@ -261,10 +256,10 @@ Uint erts_dist_table_size(void); void erts_dist_table_info(fmtfn_t, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_pending(DistEntry *); -void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); +void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint64); ErlNode *erts_find_or_insert_node(Eterm, Uint32, Eterm); void erts_schedule_delete_node(ErlNode *); -void erts_set_this_node(Eterm, Uint); +void erts_set_this_node(Eterm, Uint32); Uint erts_node_table_size(void); void erts_init_node_tables(int); void erts_node_table_info(fmtfn_t, void *); diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index f8d82a8f98..efccb5fcb6 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -51,7 +51,7 @@ * Note that not all signal are handled using this functionality! */ -#define ERTS_SIG_Q_OP_MAX 13 +#define ERTS_SIG_Q_OP_MAX 14 #define ERTS_SIG_Q_OP_EXIT 0 /* Exit signal due to bif call */ #define ERTS_SIG_Q_OP_EXIT_LINKED 1 /* Exit signal due to link break*/ @@ -66,7 +66,8 @@ #define ERTS_SIG_Q_OP_IS_ALIVE 10 #define ERTS_SIG_Q_OP_PROCESS_INFO 11 #define ERTS_SIG_Q_OP_SYNC_SUSPEND 12 -#define ERTS_SIG_Q_OP_RPC ERTS_SIG_Q_OP_MAX +#define ERTS_SIG_Q_OP_RPC 13 +#define ERTS_SIG_Q_OP_DIST_SPAWN_REPLY ERTS_SIG_Q_OP_MAX #define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5) @@ -137,6 +138,14 @@ typedef struct { } ErtsSigDistLinkOp; typedef struct { + Eterm message; + Eterm ref; + Eterm result; + ErtsLink *link; + Eterm *patch_point; +} ErtsDistSpawnReplySigData; + +typedef struct { ErtsSignalCommon common; Uint flags_on; Uint flags_off; @@ -215,6 +224,8 @@ static int handle_trace_change_state(Process *c_p, ErtsMessage ***next_nm_sig); static void getting_unlinked(Process *c_p, Eterm unlinker); static void getting_linked(Process *c_p, Eterm linker); +static void linking(Process *c_p, Eterm to); + static void group_leader_reply(Process *c_p, Eterm to, Eterm ref, int success); static int stretch_limit(Process *c_p, ErtsSigRecvTracing *tp, @@ -322,6 +333,17 @@ get_exit_signal_data(ErtsMessage *xsig) + xsig->hfrag.used_size); } +static ERTS_INLINE ErtsDistSpawnReplySigData * +get_dist_spawn_reply_data(ErtsMessage *sig) +{ + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + ASSERT(sig->hfrag.alloc_size > sig->hfrag.used_size); + ASSERT((sig->hfrag.alloc_size - sig->hfrag.used_size)*sizeof(UWord) + >= sizeof(ErtsDistSpawnReplySigData)); + return (ErtsDistSpawnReplySigData *) (char *) (&sig->hfrag.mem[0] + + sig->hfrag.used_size); +} + static ERTS_INLINE void destroy_trace_info(ErtsSigTraceInfo *ti) { @@ -997,7 +1019,7 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, seq_trace = c_p && have_seqtrace(token); if (seq_trace) - seq_trace_update_send(c_p); + seq_trace_update_serial(c_p); #ifdef USE_VM_PROBES utag_sz = 0; @@ -1751,6 +1773,104 @@ erts_proc_sig_send_sync_suspend(Process *c_p, Eterm to, Eterm tag, Eterm reply) } } +int +erts_proc_sig_send_dist_spawn_reply(Eterm node, + Eterm ref, + Eterm to, + ErtsLink *lnk, + Eterm result, + Eterm token) +{ + Uint hsz, ref_sz, result_sz, token_sz; + ErtsDistSpawnReplySigData *datap; + Eterm msg, ref_copy, result_copy, res_type, + token_copy, *hp, *hp_start, *patch_point; + ErlHeapFragment *hfrag; + ErlOffHeap *ohp; + ErtsMessage *mp; + + ASSERT(is_atom(node)); + + /* + * A respons message to a spawn_request() operation + * looks like this: + * {Tag, Ref, ok|error, Pid|ErrorAtom} + * + * Tag is stored in its own heap fragment in the + * (pending) monitor struct and can be attached + * when creating the resulting message on + * reception of this signal. + */ + + hsz = ref_sz = size_object(ref); + hsz += 5 /* 4-tuple */; + if (is_atom(result)) { + res_type = am_error; + result_sz = 0; + } + else { + ASSERT(is_external_pid(result)); + res_type = am_ok; + result_sz = size_object(result); + hsz += result_sz; + } + + token_sz = is_immed(token) ? 0 : size_object(token); + hsz += token_sz; + + hsz += sizeof(ErtsDistSpawnReplySigData)/sizeof(Eterm); + + mp = erts_alloc_message(hsz, &hp); + hp_start = hp; + hfrag = &mp->hfrag; + mp->next = NULL; + ohp = &hfrag->off_heap; + + ref_copy = copy_struct(ref, ref_sz, &hp, ohp); + result_copy = (is_atom(result) + ? result + : copy_struct(result, result_sz, &hp, ohp)); + msg = TUPLE4(hp, + am_undefined, + ref_copy, + res_type, + result_copy); + + patch_point = &hp[1]; + ASSERT(*patch_point == am_undefined); + + hp += 5; + + token_copy = (!token_sz + ? token + : copy_struct(token, token_sz, &hp, ohp)); + + hfrag->used_size = hp - hp_start; + + datap = (ErtsDistSpawnReplySigData *) (char *) hp; + datap->message = msg; + datap->ref = ref_copy; + datap->result = result_copy; + datap->link = lnk; + datap->patch_point = patch_point; + + ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_DIST_SPAWN_REPLY, + ERTS_SIG_Q_TYPE_UNDEFINED, + 0); + ERL_MESSAGE_FROM(mp) = node; + ERL_MESSAGE_TOKEN(mp) = token_copy; + if (!proc_queue_signal(NULL, to, (ErtsSignal *) mp, + ERTS_SIG_Q_OP_DIST_SPAWN_REPLY)) { + mp->next = NULL; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(mp) = msg; + erts_cleanup_messages(mp); + return 0; + } + + return !0; +} + Eterm erts_proc_sig_send_rpc_request(Process *c_p, Eterm to, @@ -1901,6 +2021,7 @@ is_alive_response(Process *c_p, ErtsMessage *mp, int is_alive) } } + static ERTS_INLINE void adjust_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing, int setup) { @@ -2303,17 +2424,14 @@ static int convert_to_down_message(Process *c_p, ErtsMessage *sig, ErtsMonitorData *mdp, + ErtsMonitor **omon, Uint16 mon_type, ErtsMessage ***next_nm_sig) { - /* - * Create a 'DOWN' message and replace the signal - * with it... - */ int cnt = 0; Eterm node = am_undefined; ErtsMessage *mp; - ErtsProcLocks locks; + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; Uint hsz; Eterm *hp, ref, from, type, reason; ErlOffHeap *ohp; @@ -2322,96 +2440,170 @@ convert_to_down_message(Process *c_p, ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) == (mdp->target.flags & ERTS_ML_FLGS_SAME)); - hsz = 6; /* 5-tuple */ + /* reason is mdp->target.other.item */ + reason = mdp->target.other.item; + ASSERT(is_immed(reason)); + ASSERT(&mdp->origin == *omon); + + if (mdp->origin.flags & ERTS_ML_FLG_SPAWN_PENDING) { + /* + * Create a spawn_request() error message and replace + * the signal with it... + */ + Eterm tag; + ErtsMonitorDataExtended *mdep; - if (mdp->origin.flags & ERTS_ML_FLG_NAME) - hsz += 3; /* reg name 2-tuple */ - else { - ASSERT(is_pid(mdp->origin.other.item) - || is_internal_port(mdp->origin.other.item)); - hsz += NC_HEAP_SIZE(mdp->origin.other.item); - } + /* Should only happen when connection breaks... */ + ASSERT(reason == am_noconnection); - ASSERT(is_ref(mdp->ref)); - hsz += NC_HEAP_SIZE(mdp->ref); + if (mdp->origin.flags & (ERTS_ML_FLG_SPAWN_ABANDONED + | ERTS_ML_FLG_SPAWN_NO_EMSG)) { + /* + * Operation has been been abandoned or + * error message has been disabled... + */ + erts_monitor_release(*omon); + *omon = NULL; + return 1; + } - locks = ERTS_PROC_LOCK_MAIN; + cnt += 4; - /* reason is mdp->target.other.item */ - reason = mdp->target.other.item; - ASSERT(is_immed(reason)); + mdep = (ErtsMonitorDataExtended *) mdp; + hsz = 5; /* 4-tuple */ - mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + ASSERT(is_ref(mdp->ref)); + hsz += NC_HEAP_SIZE(mdp->ref); - if (locks != ERTS_PROC_LOCK_MAIN) - erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); - cnt += 4; + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + + ref = STORE_NC(&hp, ohp, mdp->ref); + + /* + * The tag to patch into the resulting message + * is stored in mdep->u.name via a little trick + * (see pending_flag in erts_monitor_create()). + */ + if (is_immed(mdep->u.name)) + tag = mdep->u.name; + else { + ErlHeapFragment *tag_hfrag; + tag_hfrag = (ErlHeapFragment *) cp_val(mdep->u.name); + tag = tag_hfrag->mem[0]; + /* Save heap fragment of tag in message... */ + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + tag_hfrag->next = mp->hfrag.next; + mp->hfrag.next = tag_hfrag; + } + else { + tag_hfrag->next = mp->data.heap_frag; + mp->data.heap_frag = tag_hfrag; + } + } + + /* Restore to normal monitor */ + mdep->u.name = NIL; + mdp->origin.flags &= ~ERTS_ML_FLGS_SPAWN; - ref = STORE_NC(&hp, ohp, mdp->ref); + ERL_MESSAGE_FROM(mp) = am_undefined; + ERL_MESSAGE_TERM(mp) = TUPLE4(hp, tag, ref, am_error, reason); - if (!(mdp->origin.flags & ERTS_ML_FLG_NAME)) { - from = STORE_NC(&hp, ohp, mdp->origin.other.item); } else { - ErtsMonitorDataExtended *mdep; - ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); - mdep = (ErtsMonitorDataExtended *) mdp; - ASSERT(is_atom(mdep->u.name)); - if (mdep->dist) - node = mdep->dist->nodename; - else - node = erts_this_dist_entry->sysname; - from = TUPLE2(hp, mdep->u.name, node); - hp += 3; - } + /* + * Create a 'DOWN' message and replace the signal + * with it... + */ - ASSERT(mdp->origin.type == mon_type); - switch (mon_type) { - case ERTS_MON_TYPE_PORT: - type = am_port; - if (mdp->origin.other.item == am_undefined) { - /* failed by name... */ - ERL_MESSAGE_FROM(mp) = am_system; - } + hsz = 6; /* 5-tuple */ + + if (mdp->origin.flags & ERTS_ML_FLG_NAME) + hsz += 3; /* reg name 2-tuple */ else { - ASSERT(is_internal_port(mdp->origin.other.item)); - ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; + ASSERT(is_pid(mdp->origin.other.item) + || is_internal_port(mdp->origin.other.item)); + hsz += NC_HEAP_SIZE(mdp->origin.other.item); } - break; - case ERTS_MON_TYPE_PROC: - type = am_process; - if (mdp->origin.other.item == am_undefined) { - /* failed by name... */ - ERL_MESSAGE_FROM(mp) = am_system; + + ASSERT(is_ref(mdp->ref)); + hsz += NC_HEAP_SIZE(mdp->ref); + + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + + cnt += 4; + + ref = STORE_NC(&hp, ohp, mdp->ref); + + if (!(mdp->origin.flags & ERTS_ML_FLG_NAME)) { + from = STORE_NC(&hp, ohp, mdp->origin.other.item); } else { - ASSERT(is_internal_pid(mdp->origin.other.item)); - ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; - } - break; - case ERTS_MON_TYPE_DIST_PROC: - type = am_process; - if (node == am_undefined) { ErtsMonitorDataExtended *mdep; ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); mdep = (ErtsMonitorDataExtended *) mdp; - ASSERT(mdep->dist); - node = mdep->dist->nodename; + ASSERT(is_atom(mdep->u.name)); + if (mdep->dist) + node = mdep->dist->nodename; + else + node = erts_this_dist_entry->sysname; + from = TUPLE2(hp, mdep->u.name, node); + hp += 3; + } + + ASSERT(mdp->origin.type == mon_type); + switch (mon_type) { + case ERTS_MON_TYPE_PORT: + type = am_port; + if (mdp->origin.other.item == am_undefined) { + /* failed by name... */ + ERL_MESSAGE_FROM(mp) = am_system; + } + else { + ASSERT(is_internal_port(mdp->origin.other.item)); + ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; + } + break; + case ERTS_MON_TYPE_PROC: + type = am_process; + if (mdp->origin.other.item == am_undefined) { + /* failed by name... */ + ERL_MESSAGE_FROM(mp) = am_system; + } + else { + ASSERT(is_internal_pid(mdp->origin.other.item)); + ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; + } + break; + case ERTS_MON_TYPE_DIST_PROC: + type = am_process; + if (node == am_undefined) { + ErtsMonitorDataExtended *mdep; + ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); + mdep = (ErtsMonitorDataExtended *) mdp; + ASSERT(mdep->dist); + node = mdep->dist->nodename; + } + ASSERT(is_atom(node) && node != am_undefined); + ERL_MESSAGE_FROM(mp) = node; + break; + default: + ERTS_INTERNAL_ERROR("Unexpected monitor type"); + type = am_undefined; + ERL_MESSAGE_FROM(mp) = am_undefined; + break; } - ASSERT(is_atom(node) && node != am_undefined); - ERL_MESSAGE_FROM(mp) = node; - break; - default: - ERTS_INTERNAL_ERROR("Unexpected monitor type"); - type = am_undefined; - ERL_MESSAGE_FROM(mp) = am_undefined; - break; - } - ERL_MESSAGE_TERM(mp) = TUPLE5(hp, am_DOWN, ref, - type, from, reason); - hp += 6; + ERL_MESSAGE_TERM(mp) = TUPLE5(hp, am_DOWN, ref, + type, from, reason); + hp += 6; + + } ERL_MESSAGE_TOKEN(mp) = am_undefined; /* Replace original signal with the exit message... */ @@ -3154,6 +3346,308 @@ erts_proc_sig_handle_pending_suspend(Process *c_p) ERTS_PROC_SET_PENDING_SUSPEND(c_p, NULL); } +static int +handle_dist_spawn_reply(Process *c_p, ErtsSigRecvTracing *tracing, + ErtsMessage *sig, ErtsMessage ***next_nm_sig) +{ + + ErtsDistSpawnReplySigData *datap = get_dist_spawn_reply_data(sig); + ErtsMonitorDataExtended *mdep; + Eterm msg = datap->message; + Eterm result = datap->result; + ErtsMonitor *omon; + int adjust_monitor; + ErlHeapFragment *tag_hfrag = NULL; + int convert_to_message = !0; + int cnt = 1; + + ASSERT(is_atom(result) || is_external_pid(result)); + ASSERT(is_atom(result) || size_object(result) == EXTERNAL_THING_HEAD_SIZE + 1); + + omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), datap->ref); + + if (!omon || !(omon->flags & ERTS_ML_FLG_SPAWN_PENDING)) { + /* Stale reply; remove link that was setup... */ + ErtsLink *lnk = datap->link; + if (lnk) { + ErtsLinkData *ldp; + ErtsLink *dlnk = erts_link_to_other(lnk, &ldp); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(lnk); + } + remove_nm_sig(c_p, sig, next_nm_sig); + sig->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(sig) = msg; + sig->next = NULL;; + erts_cleanup_messages(sig); + return ++cnt; + } + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon); + +#ifdef DEBUG + { + Eterm *tp; + int i; + ASSERT(erts_monitor_is_in_table(omon)); + ASSERT(omon->flags & ERTS_ML_FLG_SPAWN_PENDING); + if (is_atom(result)) { + ASSERT(!datap->link); + } + else { + ASSERT(!datap->link || (omon->flags & ERTS_ML_FLG_SPAWN_LINK)); + ASSERT(!(omon->flags & ERTS_ML_FLG_SPAWN_LINK) || datap->link); + } + ASSERT(omon->other.item == am_pending); + ASSERT(is_tuple_arity(datap->message, 4)); + tp = tuple_val(datap->message); + ASSERT(tp[1] == am_undefined); /* patch point */ + ASSERT(is_internal_ref(tp[2])); + ASSERT((tp[3] == am_ok && is_external_pid(tp[4])) + || (tp[3] == am_error && is_atom(tp[4]))); + for (i = 0; i < EXTERNAL_THING_HEAD_SIZE + 1; i++) { + ASSERT(is_non_value(mdep->heap[i])); + } + } +#endif + + /* + * The tag to patch into the resulting message + * is stored in mdep->u.name via a little trick + * (see pending_flag in erts_monitor_create()). + */ + if (is_immed(mdep->u.name)) { + tag_hfrag = NULL; + *datap->patch_point = mdep->u.name; + } + else { + tag_hfrag = (ErlHeapFragment *) cp_val(mdep->u.name); + *datap->patch_point = tag_hfrag->mem[0]; + } + mdep->u.name = NIL; /* Restore to normal monitor */ + + if (is_atom(result)) { /* Spawn error; cleanup... */ + /* Dist code should not have created a link on failure... */ + + ASSERT(is_not_atom(result) || !datap->link); + /* delete monitor structure... */ + adjust_monitor = 0; + if (omon->flags & (ERTS_ML_FLG_SPAWN_ABANDONED + | ERTS_ML_FLG_SPAWN_NO_EMSG)) + convert_to_message = 0; + } + else if (omon->flags & ERTS_ML_FLG_SPAWN_ABANDONED) { + /* + * Spawn operation has been abandoned and + * link option was passed. Send exit signal + * with exit reason 'abandoned'... + */ + DistEntry *dep; + ErtsMonLnkDist *dist; + ErtsMonitorDataExtended *mdep; + ErtsLink *lnk; + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon); + dist = mdep->dist; + + ASSERT(omon->flags & ERTS_ML_FLG_SPAWN_LINK); + + lnk = datap->link; + if (lnk) { + ErtsLinkData *ldp; + ErtsLink *dlnk; + dlnk = erts_link_to_other(lnk, &ldp); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(lnk); + } + + ASSERT(is_external_pid(result)); + dep = external_pid_dist_entry(result); + + if (dep != erts_this_dist_entry && dist->nodename == dep->sysname) { + ErtsDSigSendContext ctx; + int code = erts_dsig_prepare(&ctx, dep, c_p, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == ctx.connection_id) { + code = erts_dsig_send_exit_tt(&ctx, + c_p->common.id, + result, + am_abandoned, + SEQ_TRACE_TOKEN(c_p)); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + break; + default: + break; + } + } + /* delete monitor structure... */ + adjust_monitor = 0; + /* drop message... */ + convert_to_message = 0; + } + else { + /* Success... */ + ASSERT(is_external_pid(result)); + + if (omon->flags & ERTS_ML_FLG_SPAWN_NO_SMSG) + convert_to_message = 0; + + if (datap->link) { + cnt++; + erts_link_tree_insert(&ERTS_P_LINKS(c_p), datap->link); + if (tracing->procs) + linking(c_p, result); + } + + adjust_monitor = !!(omon->flags & ERTS_ML_FLG_SPAWN_MONITOR); + if (adjust_monitor) { + /* + * Insert the actual pid of spawned process + * in origin part of monitor... + */ + ErlOffHeap oh; + ErtsMonitorDataExtended *mdep; + Eterm *hp; + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon); + hp = &(mdep)->heap[0]; + omon->flags &= ~ERTS_ML_FLGS_SPAWN; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = mdep->uptr.ohhp; + omon->other.item = copy_struct(result, + EXTERNAL_THING_HEAD_SIZE + 1, + &hp, &oh); + mdep->uptr.ohhp = oh.first; + cnt += 2; + } + } + + if (!adjust_monitor) { + /* + * Delete monitor; either spawn error + * or no monitor requested... + */ + ErtsMonitorData *mdp = erts_monitor_to_data(omon); + + omon->flags &= ~ERTS_ML_FLGS_SPAWN; + + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon); + + if (erts_monitor_dist_delete(&mdp->target)) + erts_monitor_release_both(mdp); + else + erts_monitor_release(omon); + cnt += 2; + } + + if (convert_to_message) { + convert_prepared_sig_to_msg(c_p, sig, msg, next_nm_sig); + if (tag_hfrag) { + /* Save heap fragment of tag in message... */ + tag_hfrag->next = sig->hfrag.next; + sig->hfrag.next = tag_hfrag; + } + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); + } + else { + remove_nm_sig(c_p, sig, next_nm_sig); + sig->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(sig) = msg; + sig->next = NULL;; + erts_cleanup_messages(sig); + if (tag_hfrag) { + tag_hfrag->next = NULL; + free_message_buffer(tag_hfrag); + } + } + return cnt; +} + +static int +handle_dist_spawn_reply_exiting(Process *c_p, + ErtsMessage *sig, + ErtsMonitor **pend_spawn_mon_pp, + Eterm reason) +{ + ErtsDistSpawnReplySigData *datap = get_dist_spawn_reply_data(sig); + Eterm result = datap->result; + Eterm msg = datap->message; + ErtsMonitorData *mdp; + ErtsMonitor *omon; + int cnt = 1; + + ASSERT(is_atom(result) || is_external_pid(result)); + ASSERT(is_atom(result) || size_object(result) == EXTERNAL_THING_HEAD_SIZE + 1); + + omon = erts_monitor_tree_lookup(*pend_spawn_mon_pp, datap->ref); + if (!omon) { + /* May happen when connection concurrently close... */ + ErtsLink *lnk = datap->link; + if (lnk) { + ErtsLinkData *ldp; + ErtsLink *dlnk = erts_link_to_other(lnk, &ldp); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(lnk); + } + cnt++; + } + else { + ASSERT(omon->flags & ERTS_ML_FLG_SPAWN_PENDING); + ASSERT(!datap->link || is_external_pid(result)); + + erts_monitor_tree_delete(pend_spawn_mon_pp, omon); + mdp = erts_monitor_to_data(omon); + + if (!erts_dist_pend_spawn_exit_delete(&mdp->target)) + mdp = NULL; /* Connection closed/closing... */ + cnt++; + + if (is_external_pid(result)) { + if ((omon->flags & ERTS_ML_FLG_SPAWN_MONITOR) && mdp) { + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + erts_proc_exit_dist_demonitor(c_p, + external_pid_dist_entry(result), + mdep->dist->connection_id, + datap->ref, + result); + cnt++; + } + ASSERT(!datap->link || (omon->flags & ERTS_ML_FLG_SPAWN_LINK)); + ASSERT(!(omon->flags & ERTS_ML_FLG_SPAWN_LINK) || datap->link); + + if (datap->link) { + /* This link exit *should* have actual reason... */ + ErtsProcExitContext pectxt = {c_p, reason}; + /* unless operation has been abandoned... */ + if (omon->flags & ERTS_ML_FLG_SPAWN_ABANDONED) + pectxt.reason = am_abandoned; + erts_proc_exit_handle_link(datap->link, (void *) &pectxt, -1); + cnt++; + } + } + if (mdp) + erts_monitor_release_both(mdp); + else + erts_monitor_release(omon); + cnt++; + } + sig->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(sig) = msg; + sig->next = NULL; + erts_cleanup_messages(sig); + cnt++; + return cnt; +} + /* * Called in order to handle incoming signals. */ @@ -3266,7 +3760,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, omon = &mdp->origin; erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon); - cnt += convert_to_down_message(c_p, sig, mdp, + cnt += convert_to_down_message(c_p, sig, mdp, &omon, type, next_nm_sig); } break; @@ -3282,13 +3776,13 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, xsigd->u.ref); if (omon) { ASSERT(erts_monitor_is_origin(omon)); + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), + omon); if (omon->type == ERTS_MON_TYPE_DIST_PROC) { mdp = erts_monitor_to_data(omon); if (erts_monitor_dist_delete(&mdp->target)) tmon = &mdp->target; } - erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), - omon); cnt += convert_prepared_down_message(c_p, sig, xsigd->message, next_nm_sig); @@ -3585,6 +4079,13 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, break; } + + case ERTS_SIG_Q_OP_DIST_SPAWN_REPLY: { + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + cnt += handle_dist_spawn_reply(c_p, &tracing, sig, next_nm_sig); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } default: ERTS_INTERNAL_ERROR("Unknown signal"); @@ -3780,7 +4281,9 @@ stretch_limit(Process *c_p, ErtsSigRecvTracing *tp, int -erts_proc_sig_handle_exit(Process *c_p, Sint *redsp) +erts_proc_sig_handle_exit(Process *c_p, Sint *redsp, + ErtsMonitor **pend_spawn_mon_pp, + Eterm reason) { int cnt; Sint limit; @@ -3862,7 +4365,8 @@ erts_proc_sig_handle_exit(Process *c_p, Sint *redsp) break; case ERTS_SIG_Q_OP_MONITOR: { - ErtsProcExitContext pectxt = {c_p, am_noproc, NULL, NULL, NIL}; + ErtsProcExitContext pectxt = {c_p, am_noproc, NULL, NULL, + NULL, NULL, NIL, 0}; erts_proc_exit_handle_monitor((ErtsMonitor *) sig, (void *) &pectxt, -1); cnt += 4; @@ -3914,6 +4418,13 @@ erts_proc_sig_handle_exit(Process *c_p, Sint *redsp) break; } + case ERTS_SIG_Q_OP_DIST_SPAWN_REPLY: { + cnt += handle_dist_spawn_reply_exiting(c_p, sig, + pend_spawn_mon_pp, + reason); + break; + } + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: destroy_trace_info((ErtsSigTraceInfo *) sig); break; @@ -3986,6 +4497,7 @@ clear_seq_trace_token(ErtsMessage *sig) break; case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: + case ERTS_SIG_Q_OP_DIST_SPAWN_REPLY: ERTS_CLEAR_SEQ_TOKEN(sig); break; @@ -4060,6 +4572,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig) case ERTS_SIG_Q_OP_SYNC_SUSPEND: case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: case ERTS_SIG_Q_OP_IS_ALIVE: + case ERTS_SIG_Q_OP_DIST_SPAWN_REPLY: size = ((ErtsMessage *) sig)->hfrag.alloc_size; size *= sizeof(Eterm); size += sizeof(ErtsMessage) - sizeof(Eterm); @@ -4328,6 +4841,13 @@ getting_linked(Process *c_p, Eterm linker) am_getting_linked, linker); } +static void +linking(Process *c_p, Eterm to) +{ + trace_proc(c_p, ERTS_PROC_LOCK_MAIN, c_p, + am_link, to); +} + static ERTS_INLINE void handle_message_enqueued_tracing(Process *c_p, ErtsSigRecvTracing *tracing, diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h index 2789179b34..b0a5d0dac3 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.h +++ b/erts/emulator/beam/erl_proc_sig_queue.h @@ -706,6 +706,14 @@ erts_proc_sig_send_rpc_request(Process *c_p, Eterm (*func)(Process *, void *, int *, ErlHeapFragment **), void *arg); +int +erts_proc_sig_send_dist_spawn_reply(Eterm node, + Eterm ref, + Eterm to, + ErtsLink *lnk, + Eterm result, + Eterm token); + /* * End of send operations of currently supported process signals. */ @@ -787,7 +795,9 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, * queue. */ int -erts_proc_sig_handle_exit(Process *c_p, Sint *redsp); +erts_proc_sig_handle_exit(Process *c_p, Sint *redsp, + ErtsMonitor **pend_spawn_mon_pp, + Eterm reason); /** * diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index e9ed4a7407..16d8230533 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -44,7 +44,6 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" -#include "lttng-wrapper.h" #include "erl_ptab.h" #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API @@ -708,10 +707,10 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks - = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks - = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NFUNC_TRAP_WRAPPER].get_locks + = ERTS_PSD_NFUNC_TRAP_WRAPPER_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NFUNC_TRAP_WRAPPER].set_locks + = ERTS_PSD_NFUNC_TRAP_WRAPPER_SET_LOCKS; erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].get_locks = ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS; @@ -6495,8 +6494,8 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, ASSERT(!(state & (ERTS_PSFLG_DIRTY_IO_PROC |ERTS_PSFLG_DIRTY_CPU_PROC)) - || (BeamIsOpCode(*p->i, op_call_nif) - || BeamIsOpCode(*p->i, op_apply_bif))); + || (BeamIsOpCode(*p->i, op_call_nif_WWW) + || BeamIsOpCode(*p->i, op_call_bif_W))); a = state; @@ -9554,7 +9553,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) erts_runq_unlock(rq); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - LTTNG2(scheduler_poll, esdp->no, 1); erts_check_io(esdp->ssi->psi, ERTS_POLL_NO_TIMEOUT); ERTS_MSACC_POP_STATE_M(); @@ -11100,8 +11098,13 @@ erts_set_gc_state(Process *c_p, int enable) ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); if (!enable) { - c_p->flags |= F_DISABLE_GC; - return 0; + /* Strictly speaking it's not illegal to disable the GC when it's + * already disabled, but we risk enabling the GC prematurely if (for + * example) a BIF were to blindly disable it when trapping and then + * re-enable it before returning its result. */ + ASSERT(!(c_p->flags & F_DISABLE_GC)); + c_p->flags |= F_DISABLE_GC; + return 0; } c_p->flags &= ~F_DISABLE_GC; @@ -11482,6 +11485,180 @@ alloc_process(ErtsRunQueue *rq, int bound, erts_aint32_t state) return p; } +int +erts_parse_spawn_opts(ErlSpawnOpts *sop, Eterm opts_list, Eterm *tag, + int message_opt) +{ + /* + * Returns: + * - 0 on success + * - <0 on badopt + * - >0 on badarg (not prober list) + */ + int result = 0; + Eterm ap = opts_list; + + if (tag) + *tag = am_spawn_reply; + /* + * Store default values for options. + */ + sop->multi_set = 0; + sop->flags = erts_default_spo_flags; + sop->min_heap_size = H_MIN_SIZE; + sop->min_vheap_size = BIN_VH_MIN_SIZE; + sop->max_heap_size = H_MAX_SIZE; + sop->max_heap_flags = H_MAX_FLAGS; + sop->priority = PRIORITY_NORMAL; + sop->max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); + sop->scheduler = 0; + + /* + * Walk through the option list. + */ + while (is_list(ap)) { + Eterm arg = CAR(list_val(ap)); + ap = CDR(list_val(ap)); + if (arg == am_link) { + if (sop->flags & SPO_LINK) + sop->multi_set = !0; + sop->flags |= SPO_LINK; + } else if (arg == am_monitor) { + if (sop->flags & SPO_MONITOR) + sop->multi_set = !0; + sop->flags |= SPO_MONITOR; + } else if (is_tuple(arg)) { + Eterm* tp2 = tuple_val(arg); + Eterm val; + if (*tp2 != make_arityval(2)) { + result = -1; + continue; + } + arg = tp2[1]; + val = tp2[2]; + if (arg == am_priority) { + if (sop->flags & SPO_PRIORITY) + sop->multi_set = !0; + sop->flags |= SPO_PRIORITY; + if (val == am_max) + sop->priority = PRIORITY_MAX; + else if (val == am_high) + sop->priority = PRIORITY_HIGH; + else if (val == am_normal) + sop->priority = PRIORITY_NORMAL; + else if (val == am_low) + sop->priority = PRIORITY_LOW; + else + result = -1; + } else if (arg == am_message_queue_data) { + if (sop->flags & (SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ)) + sop->multi_set = !0; + switch (val) { + case am_on_heap: + sop->flags &= ~SPO_OFF_HEAP_MSGQ; + sop->flags |= SPO_ON_HEAP_MSGQ; + break; + case am_off_heap: + sop->flags &= ~SPO_ON_HEAP_MSGQ; + sop->flags |= SPO_OFF_HEAP_MSGQ; + break; + default: + result = -1; + break; + } + } else if (arg == am_min_heap_size && is_small(val)) { + Sint min_heap_size = signed_val(val); + if (sop->flags & SPO_MIN_HEAP_SIZE) + sop->multi_set = !0; + sop->flags |= SPO_MIN_HEAP_SIZE; + if (min_heap_size < 0) { + result = -1; + } else if (min_heap_size < H_MIN_SIZE) { + sop->min_heap_size = H_MIN_SIZE; + } else { + sop->min_heap_size = erts_next_heap_size(min_heap_size, 0); + } + } else if (arg == am_max_heap_size) { + if (sop->flags & SPO_MAX_HEAP_SIZE) + sop->multi_set = !0; + sop->flags |= SPO_MAX_HEAP_SIZE; + if (!erts_max_heap_size(val, &sop->max_heap_size, &sop->max_heap_flags)) + result = -1; + } else if (arg == am_min_bin_vheap_size && is_small(val)) { + Sint min_vheap_size = signed_val(val); + if (sop->flags & SPO_MIN_VHEAP_SIZE) + sop->multi_set = !0; + sop->flags |= SPO_MIN_VHEAP_SIZE; + if (min_vheap_size < 0) { + result = -1; + } else if (min_vheap_size < BIN_VH_MIN_SIZE) { + sop->min_vheap_size = BIN_VH_MIN_SIZE; + } else { + sop->min_vheap_size = erts_next_heap_size(min_vheap_size, 0); + } + } else if (arg == am_fullsweep_after && is_small(val)) { + Sint max_gen_gcs = signed_val(val); + if (sop->flags & SPO_MAX_GEN_GCS) + sop->multi_set = !0; + sop->flags |= SPO_MAX_GEN_GCS; + if (max_gen_gcs < 0) { + result = -1; + } else { + sop->max_gen_gcs = max_gen_gcs; + } + } else if (arg == am_scheduler && is_small(val)) { + Sint scheduler = signed_val(val); + if (sop->flags & SPO_SCHEDULER) + sop->multi_set = !0; + sop->flags |= SPO_SCHEDULER; + if (scheduler < 0 || erts_no_schedulers < scheduler) + result = -1; + else + sop->scheduler = (int) scheduler; + } else if (arg == am_reply) { + if (!message_opt) + result = -1; + else if (val == am_error_only) { + sop->flags |= SPO_NO_SMSG; + sop->flags &= ~SPO_NO_EMSG; + } + else if (val == am_success_only) { + sop->flags &= ~SPO_NO_SMSG; + sop->flags |= SPO_NO_EMSG; + } + else if (val == am_no) { + sop->flags |= SPO_NO_SMSG; + sop->flags |= SPO_NO_EMSG; + } + else if (val == am_yes) { + sop->flags &= ~SPO_NO_SMSG; + sop->flags &= ~SPO_NO_EMSG; + } + else + result = -1; + } else if (arg == am_reply_tag) { + if (!tag) + result = -1; + else + *tag = val; + } else { + result = -1; + } + } else { + result = -1; + } + } + if (is_not_nil(ap)) { + return 1; + } + + if (sop->max_heap_size != 0 && sop->max_heap_size < sop->min_heap_size) { + result = -1; + } + + return result; +} + Eterm erl_create_process(Process* parent, /* Parent of process (default group leader). */ Eterm mod, /* Tagged atom for module. */ @@ -11501,6 +11678,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + Eterm node_token_heap[6]; + Eterm group_leader, parent_id, spawn_ref, token; #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); @@ -11508,8 +11687,25 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_literal_area_t litarea; INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif - - erts_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); + + if (!parent) { + token = so->token; + group_leader = so->group_leader; + parent_id = so->parent_id; + spawn_ref = so->mref; + } + else { + token = SEQ_TRACE_TOKEN(parent); + erts_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); + group_leader = parent->group_leader; + parent_id = parent->common.id; + if (so->flags & (SPO_MONITOR | SPO_ASYNC)) + spawn_ref = so->mref = erts_make_ref(parent); + else if (have_seqtrace(token)) + spawn_ref = erts_make_ref(parent); + else + spawn_ref = THE_NON_VALUE; + } /* * Check for errors. @@ -11520,6 +11716,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } + if (arity > MAX_SMALL) { + so->error_code = SYSTEM_LIMIT; + goto error; + } + if (so->flags & SPO_USE_ARGS) { if (so->scheduler) { int ix = so->scheduler-1; @@ -11544,14 +11745,21 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ASSERT((qs_flags & FS_ON_HEAP_MSGQ) || (qs_flags & FS_OFF_HEAP_MSGQ)); - if (!rq) - rq = erts_get_runq_proc(parent, NULL); + if (!rq) { + if (parent) + rq = erts_get_runq_proc(parent, NULL); + else { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp->type == ERTS_SCHED_NORMAL); + rq = esdp->run_queue; + } + } + ASSERT(rq); p = alloc_process(rq, bound, state); /* All proc locks are locked by this thread on success */ if (!p) { - erts_send_error_to_logger_str(parent->group_leader, - "Too many processes\n"); + erts_send_error_to_logger_str(group_leader, "Too many processes\n"); so->error_code = SYSTEM_LIMIT; goto error; } @@ -11561,7 +11769,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #else arg_size = size_object_litopt(args, &litarea); #endif - heap_need = arg_size; + heap_need = arg_size + 1; /* Reserve place for continuation pointer */ p->flags = flags; p->sig_qs.flags = qs_flags; @@ -11585,18 +11793,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->u.initial.module = mod; - p->u.initial.function = func; - p->u.initial.arity = (Uint) arity; - /* * Must initialize binary lists here before copying binaries to process. */ p->off_heap.first = NULL; p->off_heap.overhead = 0; - heap_need += - IS_CONST(parent->group_leader) ? 0 : NC_HEAP_SIZE(parent->group_leader); + if (is_not_immed(group_leader)) + heap_need += NC_HEAP_SIZE(group_leader); if (heap_need < p->min_heap_size) { sz = heap_need = p->min_heap_size; @@ -11611,7 +11815,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->old_hend = p->old_htop = p->old_heap = NULL; p->high_water = p->heap; p->gen_gcs = 0; - p->stop = p->hend = p->heap + sz; + p->hend = p->heap + sz; + p->stop = p->hend - 1; /* Reserve place for continuation pointer */ p->htop = p->heap; p->heap_sz = sz; p->abandoned_heap = NULL; @@ -11629,7 +11834,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; - p->cp = (BeamInstr *) beam_apply+1; + p->stop[0] = make_cp(beam_apply + 1); p->arg_reg = p->def_arg_reg; p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); @@ -11655,16 +11860,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ERTS_P_MONITORS(p) = NULL; ERTS_P_LT_MONITORS(p) = NULL; - ASSERT(is_pid(parent->group_leader)); + ASSERT(is_pid(group_leader)); - if (parent->group_leader == ERTS_INVALID_PID) + if (group_leader == ERTS_INVALID_PID) p->group_leader = p->common.id; else { /* Needs to be done after the heap has been set up */ p->group_leader = - IS_CONST(parent->group_leader) - ? parent->group_leader - : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); + IS_CONST(group_leader) + ? group_leader + : STORE_NC(&p->htop, &p->off_heap, group_leader); } erts_get_default_proc_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p)); @@ -11692,14 +11897,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->mbuf_sz = 0; erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; - p->seq_trace_lastcnt = 0; - p->seq_trace_clock = 0; - SEQ_TRACE_TOKEN(p) = NIL; #ifdef USE_VM_PROBES DT_UTAG(p) = NIL; DT_UTAG_FLAGS(p) = 0; #endif - p->parent = (parent->common.id == ERTS_INVALID_PID + p->parent = (!parent || parent->common.id == ERTS_INVALID_PID ? NIL : parent->common.id); @@ -11715,7 +11917,107 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->fp_exception = 0; #endif - if (IS_TRACED(parent)) { + /* seq_trace is handled before regular tracing as the latter may touch the + * trace token. */ + if (!have_seqtrace(token)) { + SEQ_TRACE_TOKEN(p) = NIL; + p->seq_trace_lastcnt = 0; + p->seq_trace_clock = 0; + } + else { + Eterm tmp_heap[9]; /* 8-tuple */ + Eterm seq_msg; + Uint token_sz; + Eterm *hp; + + if (parent) { + seq_trace_update_serial(parent); + token = SEQ_TRACE_TOKEN(parent); + ASSERT(SEQ_TRACE_T_ARITY(token) == 5); + sys_memcpy(&node_token_heap[0], + (void *) tuple_val(token), + sizeof(Eterm)*6); + token = make_tuple(&node_token_heap[0]); + } + + ASSERT(SEQ_TRACE_T_ARITY(token) == 5); + ASSERT(is_immed(SEQ_TRACE_T_FLAGS(token))); + ASSERT(is_immed(SEQ_TRACE_T_SERIAL(token))); + ASSERT(is_immed(SEQ_TRACE_T_LASTCNT(token))); + + token_sz = size_object(token); + + hp = HAlloc(p, token_sz); + SEQ_TRACE_TOKEN(p) = copy_struct(token, token_sz, &hp, &MSO(p)); + + ASSERT((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) == + (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)); + + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + + if (parent) { + /* Simulate spawn_request message... */ + Eterm tmp_heap2[4]; + Eterm mfa; + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + mfa = TUPLE3(&tmp_heap2[0], mod, func, make_small(arity)) ; + seq_msg = TUPLE8(&tmp_heap[0], am_spawn_request, + spawn_ref, parent_id, group_leader, + mfa, so->opts, so->tag, args); + seq_trace_output(token, seq_msg, SEQ_TRACE_SEND, + p->common.id, parent); + seq_trace_output(token, seq_msg, SEQ_TRACE_RECEIVE, + p->common.id, parent); + + /* The counters behave the same way on spawning as they do on messages; + * we don't inherit our parent's lastcnt. */ + p->seq_trace_clock = unsigned_val(SEQ_TRACE_T_SERIAL(token)); + p->seq_trace_lastcnt = p->seq_trace_clock; + + } + else { + /* + * The spawn request is presented as two messages + * in dist case. It is sent as one signal over the + * distribution with the argument list as payload. + * The payload will be delivered as an ordinary + * message of its own, as the first message to the + * newly created process (in order to decode it in + * in the newly created process). For more info see + * erts_internal:dist_spawn_init() in erts_interal.erl. + * We expose these as two messages when seq-tracing + * in order not having to decode the argument list + * here. The remote node has passed a token with + * serial bumped twice, i.e., the first message should + * use a serial of one less than in the actual token; + * adjust serial and then restore it for use with + * the argument list message... + */ + Eterm serial; + Uint serial_num; + ASSERT(eq(SEQ_TRACE_T_SENDER(token), parent_id)); + serial = SEQ_TRACE_T_SERIAL(token); + serial_num = unsigned_val(serial); + serial_num--; + SEQ_TRACE_T_SERIAL(token) = make_small(serial_num); + + seq_msg = TUPLE6(&tmp_heap[0], am_spawn_request, + spawn_ref, parent_id, group_leader, + so->mfa, so->opts); + seq_trace_output(token, seq_msg, SEQ_TRACE_RECEIVE, + p->common.id, p); + + /* as on receive... */ + p->seq_trace_clock = serial_num; + p->seq_trace_lastcnt = serial_num; + + /* Restore serial for the argument list message... */ + SEQ_TRACE_T_SERIAL(token) = serial; + } + } + + if (parent && IS_TRACED(parent)) { if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); erts_tracer_replace(&p->common, ERTS_TRACER(parent)); @@ -11736,9 +12038,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } } if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + /* The locks may already be released if seq_trace is enabled as + * well. */ + if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) + == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); if (so->flags & SPO_LINK) trace_proc(parent, locks, parent, am_link, p->common.id); @@ -11751,49 +12058,187 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* This happens when parent was not traced, but child is */ locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + if (parent) + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); } - trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args); + trace_proc_spawn(p, am_spawned, parent_id, mod, func, args); if (so->flags & SPO_LINK) - trace_proc(p, locks, p, am_getting_linked, parent->common.id); + trace_proc(p, locks, p, am_getting_linked, parent_id); } /* * Check if this process should be initially linked to its parent. */ - if (so->flags & SPO_LINK) { - ErtsLink *lnk; - ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_PROC, - parent->common.id, - p->common.id); - lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(parent), &ldp->a); - if (lnk) { - /* - * This should more or less never happen, but could - * potentially happen if pid:s wrap... - */ - erts_link_release(lnk); + if (parent) { + /* Node local spawn... */ + + if (so->flags & SPO_LINK) { + ErtsLink *lnk; + ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_PROC, + parent->common.id, + p->common.id); + lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(parent), &ldp->a); + if (lnk) { + /* + * This should more or less never happen, but could + * potentially happen if pid:s wrap... + */ + erts_link_release(lnk); + } + erts_link_tree_insert(&ERTS_P_LINKS(p), &ldp->b); } - erts_link_tree_insert(&ERTS_P_LINKS(p), &ldp->b); - } - /* - * Test whether this process should be initially monitored by its parent. - */ - if (so->flags & SPO_MONITOR) { - Eterm mref = erts_make_ref(parent); - ErtsMonitorData *mdp = erts_monitor_create(ERTS_MON_TYPE_PROC, - mref, - parent->common.id, - p->common.id, - NIL); - erts_monitor_tree_insert(&ERTS_P_MONITORS(parent), &mdp->origin); - erts_monitor_list_insert(&ERTS_P_LT_MONITORS(p), &mdp->target); - so->mref = mref; + /* + * Test whether this process should be initially monitored by its parent. + */ + if (so->flags & SPO_MONITOR) { + ErtsMonitorData *mdp = erts_monitor_create(ERTS_MON_TYPE_PROC, + spawn_ref, + parent->common.id, + p->common.id, + NIL); + erts_monitor_tree_insert(&ERTS_P_MONITORS(parent), &mdp->origin); + erts_monitor_list_insert(&ERTS_P_LT_MONITORS(p), &mdp->target); + } + + ASSERT(locks & ERTS_PROC_LOCK_MSGQ); + + if (so->flags & SPO_ASYNC) { /* spawn_request() */ + Eterm *tp; + + ASSERT(is_tuple_arity(so->mfa, 3)); + tp = tuple_val(so->mfa); + ASSERT(is_atom(tp[1])); + ASSERT(is_atom(tp[2])); + ASSERT(is_small(tp[3])); + p->u.initial.module = tp[1]; + p->u.initial.function = tp[2]; + p->u.initial.arity = (Uint) unsigned_val(tp[3]); + + ASSERT(is_value(so->tag)); + if (have_seqtrace(token)) { + seq_trace_update_serial(p); + token = SEQ_TRACE_TOKEN(p); + } + + if (!(so->flags & SPO_NO_SMSG)) { + /* + * Ensure spawn reply success message reach parent before + * any down or exit signals from child... + */ + erts_send_local_spawn_reply(parent, locks, p, + so->tag, spawn_ref, + p->common.id, token); + } + } + else { /* synchronous spawn */ + + p->u.initial.module = mod; + p->u.initial.function = func; + p->u.initial.arity = (Uint) arity; + + if (have_seqtrace(token)) { + /* Simulate spawn reply message... */ + Eterm tmp_heap[5]; + Eterm seq_msg; + Uint serial; + + seq_trace_update_serial(p); + token = SEQ_TRACE_TOKEN(p); + serial = SEQ_TRACE_T_SERIAL(token); + seq_msg = TUPLE4(&tmp_heap[0], so->tag, spawn_ref, + am_ok, p->common.id); + seq_trace_output(token, seq_msg, SEQ_TRACE_SEND, + parent_id, parent); + + /* Update parent as if receive... */ + parent->seq_trace_lastcnt = serial; + parent->seq_trace_clock = serial; + seq_trace_output(token, seq_msg, SEQ_TRACE_RECEIVE, + parent_id, parent); + } + + } + + erts_proc_unlock(p, locks); + erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + } + else { + /* Distributed spawn */ + ErtsDSigSendContext ctx; + int code; + Eterm *tp; + + ASSERT(is_tuple_arity(so->mfa, 3)); + tp = tuple_val(so->mfa); + ASSERT(is_atom(tp[1])); + ASSERT(is_atom(tp[2])); + ASSERT(is_small(tp[3])); + p->u.initial.module = tp[1]; + p->u.initial.function = tp[2]; + p->u.initial.arity = (Uint) unsigned_val(tp[3]); + + ASSERT(locks & ERTS_PROC_LOCK_MSGQ); + /* + * Pass the (on external format) encoded argument list as + * *first* message to the process. Note that this message + * *must* be first in the message queue of the newly + * spawned process! + */ + erts_queue_dist_message(p, locks, so->edep, so->ede_hfrag, + token, parent_id); - erts_proc_unlock(p, locks); + erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); + + if (so->flags & SPO_LINK) { + ErtsLinkData *ldp; + ldp = erts_link_create(ERTS_LNK_TYPE_DIST_PROC, + parent_id, p->common.id); + code = erts_link_dist_insert(&ldp->a, so->dist_entry->mld); + ASSERT(code); + erts_link_tree_insert(&ERTS_P_LINKS(p), &ldp->b); + } + + if (so->flags & SPO_MONITOR) { + ErtsMonitorData *mdp; + mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, + spawn_ref, parent_id, + p->common.id, NIL); + code = erts_monitor_dist_insert(&mdp->origin, so->dist_entry->mld); + ASSERT(code); (void)code; + erts_monitor_tree_insert(&ERTS_P_MONITORS(p), &mdp->target); + } + + if (have_seqtrace(token)) { + Eterm tmp_heap[5]; + Eterm seq_msg; + seq_trace_update_serial(p); + seq_msg = TUPLE4(&tmp_heap[0], so->tag, + spawn_ref, am_ok, p->common.id); + token = SEQ_TRACE_TOKEN(p); + seq_trace_output(token, seq_msg, SEQ_TRACE_SEND, parent_id, p); + } + + code = erts_dsig_prepare(&ctx, so->dist_entry, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + int dsflags = 0; + if (so->flags & SPO_LINK) + dsflags |= ERTS_DIST_SPAWN_FLAG_LINK; + if (so->flags & SPO_MONITOR) + dsflags |= ERTS_DIST_SPAWN_FLAG_MONITOR; + code = erts_dsig_send_spawn_reply(&ctx, spawn_ref, + parent_id, + make_small(dsflags), + p->common.id, + token); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + + erts_proc_unlock(p, locks & ERTS_PROC_LOCK_MAIN); + } res = p->common.id; @@ -11801,8 +12246,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ - erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); - schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); @@ -11821,11 +12264,65 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). error: - erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + if (parent) + erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); return res; } +void +erts_send_local_spawn_reply(Process *parent, ErtsProcLocks parent_locks, + Process *child, Eterm tag, Eterm ref, + Eterm result, Eterm token) +{ + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm *hp; + Eterm msg, ref_copy, ref_sz, tag_copy, tag_sz, + token_copy, token_sz, type; + ErtsProcLocks locks = parent_locks; + + type = child ? am_ok : am_error; + + if (have_seqtrace(token) && child) + token_sz = size_object(token); + else { + token_copy = token = NIL; + token_sz = 0; + } + + ref_sz = size_object(ref); + tag_sz = is_immed(tag) ? 0 : size_object(tag); + mp = erts_alloc_message_heap(parent, &locks, + 5 + tag_sz + ref_sz + token_sz, + &hp, &ohp); + ref_copy = copy_struct(ref, ref_sz, &hp, ohp); + tag_copy = is_immed(tag) ? tag : copy_struct(tag, tag_sz, &hp, ohp); + msg = TUPLE4(hp, tag_copy, ref_copy, type, result); + hp += 5; + + if (have_seqtrace(token)) { + token_copy = copy_struct(token, token_sz, &hp, ohp); + seq_trace_output(token_copy, msg, SEQ_TRACE_SEND, + parent->common.id, parent); + } + + if (!child) { /* error reply */ + ASSERT(is_atom(result)); + erts_queue_message(parent, parent_locks, mp, msg, + erts_this_dist_entry->sysname); + } + else { /* success reply */ + ASSERT(child->common.id == result); + ERL_MESSAGE_TOKEN(mp) = token_copy; + erts_queue_proc_message(child, parent, parent_locks, mp, msg); + } + + ASSERT((parent_locks & locks) == parent_locks); + if (locks != parent_locks) + erts_proc_unlock(parent, locks & ~parent_locks); +} + /* * Initiates a pseudo process that can be used * for arithmetic BIFs. @@ -11902,7 +12399,6 @@ void erts_init_empty_process(Process *p) p->u.initial.function = 0; p->u.initial.arity = 0; p->catches = 0; - p->cp = NULL; p->i = NULL; p->current = NULL; @@ -11980,7 +12476,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->bif_timers == NULL); ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); - ASSERT(p->cp == NULL); ASSERT(p->i == NULL); ASSERT(p->current == NULL); @@ -12041,7 +12536,7 @@ delete_process(Process* p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); - erts_destroy_nif_export(p); + erts_destroy_nfunc(p); /* Cleanup psd */ @@ -12261,6 +12756,156 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) return reds_consumed; } +static int +proc_exit_handle_pend_spawn_monitors(ErtsMonitor *mon, void *vctxt, Sint reds) +{ + ErtsProcExitContext *ctxt = (ErtsProcExitContext *) vctxt; + Process *c_p = ctxt->c_p; + Eterm reason = ctxt->reason; + Eterm item, *hp; + Uint item_sz; + int code; + ErtsDSigSendContext ctx; + ErtsMonitorData *mdp; + ErtsMonLnkDist *dist; + DistEntry *dep; + ErtsHeapFactory factory; + Sint reds_consumed = 0; + + ASSERT(c_p->flags & F_DISABLE_GC); + ASSERT(erts_monitor_is_origin(mon) && mon->type == ERTS_MON_TYPE_DIST_PROC); + ASSERT(ctxt->dist_state == NIL); + ASSERT(!ctxt->wait_pend_spawn_monitor); + ASSERT(!ctxt->yield); + + ASSERT(mon->flags & ERTS_ML_FLG_SPAWN_PENDING); + + mdp = erts_monitor_to_data(mon); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + + if (!(mon->flags & ERTS_ML_FLG_SPAWN_LINK)) { + /* Just cleanup... */ + if (!erts_dist_pend_spawn_exit_delete(&mdp->target)) { + mdp = NULL; + } + goto done; + } + + code = erts_dist_pend_spawn_exit_parent_wait(c_p, + ERTS_PROC_LOCK_MAIN, + mon); + if (code == 0) { + /* Connection closing; cleanup... */ + mdp = NULL; + goto done; + } + + if (code < 0) { + /* We got suspended need to wait for spawn-reply... */ + ctxt->wait_pend_spawn_monitor = mon; + ctxt->yield = !0; + return reds; + } + + ASSERT(is_external_pid(mon->other.item) || is_atom(mon->other.item)); + + /* If other.item is an atom the spawn failed... */ + + if (is_not_external_pid(mon->other.item)) + goto done; /* Cleanup */ + + if (mon->flags & ERTS_ML_FLG_SPAWN_ABANDONED) + reason = am_abandoned; + + /* Send exit signal... */ + dep = external_pid_dist_entry(mon->other.item); + + code = erts_dsig_prepare(&ctx, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 0, 0); + + ctx.reds = (Sint) (reds * TERM_TO_BINARY_LOOP_FACTOR); + + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + break; + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: + if (dist->connection_id != ctx.connection_id) + break; + erts_factory_proc_init(&factory, c_p); + item_sz = size_object(mon->other.item); + hp = erts_produce_heap(&factory, item_sz, 0); + item = copy_struct(mon->other.item, item_sz, &hp, factory.off_heap); + erts_factory_close(&factory); + code = erts_dsig_send_exit_tt(&ctx, + c_p->common.id, + item, + reason, + SEQ_TRACE_TOKEN(c_p)); + reds_consumed = reds - (ctx.reds / TERM_TO_BINARY_LOOP_FACTOR); + switch (code) { + case ERTS_DSIG_SEND_YIELD: + reds_consumed = reds; /* force yield */ + ctxt->yield = 1; + break; + case ERTS_DSIG_SEND_CONTINUE: + ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); + reds_consumed = reds; /* force yield */ + ctxt->yield = 1; + break; + case ERTS_DSIG_SEND_OK: + break; + case ERTS_DSIG_SEND_TOO_LRG: + erts_kill_dist_connection(dep, dist->connection_id); + break; + default: + ASSERT(! "Invalid dsig send exit result"); + break; + } + break; + default: + ASSERT(! "Invalid dsig prep exit result"); + break; + } + +done: + + if (mdp) + erts_monitor_release_both(mdp); + else + erts_monitor_release(mon); + + return reds_consumed; +} + +void +erts_proc_exit_dist_demonitor(Process *c_p, DistEntry *dep, Uint32 conn_id, + Eterm ref, Eterm watched) +{ + ErtsDSigSendContext ctx; + int code; + + ASSERT(is_internal_ref(ref)); + ASSERT(is_atom(watched) || is_external_pid(watched)); + + code = erts_dsig_prepare(&ctx, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 1, 1, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (conn_id == ctx.connection_id) { + code = erts_dsig_send_demonitor(&ctx, + c_p->common.id, + watched, + ref); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + default: + break; + } +} + int erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) { @@ -12381,10 +13026,15 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) case ERTS_MON_TYPE_DIST_PROC: { ErtsMonLnkDist *dist; DistEntry *dep; - ErtsDSigSendContext ctx; - int code; Eterm watched; + if (mon->flags & ERTS_ML_FLG_SPAWN_PENDING) { + if (!erts_dist_pend_spawn_exit_parent_setup(mon)) + break; /* Drop it... */ + erts_monitor_tree_insert(&ctxt->pend_spawn_monitors, mon); + return 1; + } + mdp = erts_monitor_to_data(mon); dist = ((ErtsMonitorDataExtended *) mdp)->dist; ASSERT(dist); @@ -12398,21 +13048,8 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) ASSERT(is_external_pid(watched)); dep = external_pid_dist_entry(watched); } - code = erts_dsig_prepare(&ctx, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 1, 1, 0); - switch (code) { - case ERTS_DSIG_PREP_CONNECTED: - case ERTS_DSIG_PREP_PENDING: - if (dist->connection_id == ctx.connection_id) { - code = erts_dsig_send_demonitor(&ctx, - c_p->common.id, - watched, - mdp->ref); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - default: - break; - } + erts_proc_exit_dist_demonitor(c_p, dep, dist->connection_id, + mdp->ref, watched); if (!erts_monitor_dist_delete(&mdp->target)) mdp = NULL; res = 100; @@ -12676,6 +13313,7 @@ enum continue_exit_phase { ERTS_CONTINUE_EXIT_HANDLE_PROC_SIG, ERTS_CONTINUE_EXIT_DIST_LINKS, ERTS_CONTINUE_EXIT_DIST_MONITORS, + ERTS_CONTINUE_EXIT_DIST_PEND_SPAWN_MONITORS, ERTS_CONTINUE_EXIT_DONE, }; @@ -12925,6 +13563,8 @@ restart: trap_state->pectxt.reason = trap_state->reason; trap_state->pectxt.dist_links = NULL; trap_state->pectxt.dist_monitors = NULL; + trap_state->pectxt.pend_spawn_monitors = NULL; + trap_state->pectxt.wait_pend_spawn_monitor = NULL; trap_state->pectxt.dist_state = NIL; trap_state->pectxt.yield = 0; @@ -12981,8 +13621,11 @@ restart: case ERTS_CONTINUE_EXIT_HANDLE_PROC_SIG: { Sint r = reds; - if (!erts_proc_sig_handle_exit(p, &r)) + if (!erts_proc_sig_handle_exit(p, &r, + &trap_state->pectxt.pend_spawn_monitors, + trap_state->reason)) { goto yield; + } reds -= r; @@ -13047,6 +13690,31 @@ restart: if (reds <= 0 || trap_state->pectxt.yield) goto yield; + trap_state->phase = ERTS_CONTINUE_EXIT_DIST_PEND_SPAWN_MONITORS; + } + case ERTS_CONTINUE_EXIT_DIST_PEND_SPAWN_MONITORS: { + + if (is_not_nil(trap_state->pectxt.dist_state)) + goto continue_dist_send; + + if (trap_state->pectxt.wait_pend_spawn_monitor) { + ErtsMonitor *mon = trap_state->pectxt.wait_pend_spawn_monitor; + trap_state->pectxt.wait_pend_spawn_monitor = NULL; + reds -= (proc_exit_handle_pend_spawn_monitors( + mon, (void *) &trap_state->pectxt, reds)); + if (reds <= 0 || trap_state->pectxt.yield) + goto yield; + } + + reds = erts_monitor_tree_foreach_delete_yielding( + &trap_state->pectxt.pend_spawn_monitors, + proc_exit_handle_pend_spawn_monitors, + (void *) &trap_state->pectxt, + &trap_state->yield_state, + reds); + if (reds <= 0 || trap_state->pectxt.yield) + goto yield; + trap_state->phase = ERTS_CONTINUE_EXIT_DONE; } case ERTS_CONTINUE_EXIT_DONE: { @@ -13266,9 +13934,6 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Program counter: %p (", p->i); print_function_from_pc(to, to_arg, p->i); erts_print(to, to_arg, ")\n"); - erts_print(to, to_arg, "CP: %p (", p->cp); - print_function_from_pc(to, to_arg, p->cp); - erts_print(to, to_arg, ")\n"); state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS @@ -13545,9 +14210,6 @@ static void print_current_process_info(fmtfn_t to, void *to_arg, erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); print_function_from_pc(to, to_arg, p->i); erts_print(to, to_arg, ")\n"); - erts_print(to, to_arg, "Current Process CP: %p (", p->cp); - print_function_from_pc(to, to_arg, p->cp); - erts_print(to, to_arg, ")\n"); /* Getting this stacktrace can segfault if we are very very unlucky if called while a process is being garbage collected. @@ -13579,7 +14241,7 @@ static void print_current_process_info(fmtfn_t to, void *to_arg, * * A BIF that calls this should make sure to schedule out to never come back: * erts_halt(code); - * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); + * ERTS_BIF_YIELD1(&bif_trap_export[BIF_erlang_halt_1], BIF_P, NIL); */ void erts_halt(int code) { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index d311122381..f38008004f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -812,7 +812,7 @@ erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_SCHED_ID 2 #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 -#define ERTS_PSD_NIF_TRAP_EXPORT 5 +#define ERTS_PSD_NFUNC_TRAP_WRAPPER 5 #define ERTS_PSD_ETS_OWNED_TABLES 6 #define ERTS_PSD_ETS_FIXED_TABLES 7 #define ERTS_PSD_DIST_ENTRY 8 @@ -849,8 +849,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NFUNC_TRAP_WRAPPER_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NFUNC_TRAP_WRAPPER_SET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS @@ -975,7 +975,6 @@ struct process { unsigned max_arg_reg; /* Maximum number of argument registers available. */ Eterm def_arg_reg[6]; /* Default array for argument registers. */ - BeamInstr* cp; /* (untagged) Continuation pointer (for threaded code). */ BeamInstr* i; /* Program counter for threaded code. */ Sint catches; /* Number of catches on stack */ Sint fcalls; /* @@ -1326,12 +1325,46 @@ void erts_check_for_holes(Process* p); * Possible flags for the flags field in ErlSpawnOpts below. */ -#define SPO_LINK 1 -#define SPO_USE_ARGS 2 -#define SPO_MONITOR 4 -#define SPO_SYSTEM_PROC 8 -#define SPO_OFF_HEAP_MSGQ 16 -#define SPO_ON_HEAP_MSGQ 32 +#define SPO_IX_LINK 0 +#define SPO_IX_MONITOR 1 +#define SPO_IX_SYSTEM_PROC 2 +#define SPO_IX_OFF_HEAP_MSGQ 3 +#define SPO_IX_ON_HEAP_MSGQ 4 +#define SPO_IX_MIN_HEAP_SIZE 5 +#define SPO_IX_MIN_VHEAP_SIZE 6 +#define SPO_IX_PRIORITY 7 +#define SPO_IX_MAX_GEN_GCS 8 +#define SPO_IX_MAX_HEAP_SIZE 9 +#define SPO_IX_SCHEDULER 10 +#define SPO_IX_ASYNC 11 +#define SPO_IX_NO_SMSG 12 +#define SPO_IX_NO_EMSG 13 + +#define SPO_NO_INDICES (SPO_IX_ASYNC+1) + +#define SPO_LINK (1 << SPO_IX_LINK) +#define SPO_MONITOR (1 << SPO_IX_MONITOR) +#define SPO_SYSTEM_PROC (1 << SPO_IX_SYSTEM_PROC) +#define SPO_OFF_HEAP_MSGQ (1 << SPO_IX_OFF_HEAP_MSGQ) +#define SPO_ON_HEAP_MSGQ (1 << SPO_IX_ON_HEAP_MSGQ) +#define SPO_MIN_HEAP_SIZE (1 << SPO_IX_MIN_HEAP_SIZE) +#define SPO_MIN_VHEAP_SIZE (1 << SPO_IX_MIN_VHEAP_SIZE) +#define SPO_PRIORITY (1 << SPO_IX_PRIORITY) +#define SPO_MAX_GEN_GCS (1 << SPO_IX_MAX_GEN_GCS) +#define SPO_MAX_HEAP_SIZE (1 << SPO_IX_MAX_HEAP_SIZE) +#define SPO_SCHEDULER (1 << SPO_IX_SCHEDULER) +#define SPO_ASYNC (1 << SPO_IX_ASYNC) +#define SPO_NO_SMSG (1 << SPO_IX_NO_SMSG) +#define SPO_NO_EMSG (1 << SPO_IX_NO_EMSG) + +#define SPO_MAX_FLAG SPO_NO_EMSG + +#define SPO_USE_ARGS \ + (SPO_MIN_HEAP_SIZE \ + | SPO_PRIORITY \ + | SPO_MAX_GEN_GCS \ + | SPO_MAX_HEAP_SIZE \ + | SPO_SCHEDULER) extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags); @@ -1341,7 +1374,22 @@ extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags); typedef struct { int flags; int error_code; /* Error code returned from create_process(). */ - Eterm mref; /* Monitor ref returned (if SPO_MONITOR was given). */ + Eterm mref; /* Monitor ref returned (if SPO_MONITOR was given). + (output if local; input if distributed) */ + + int multi_set; + + Eterm tag; /* If SPO_ASYNC */ + Eterm opts; /* Option list for seq-trace... */ + + /* Input fields used for distributed spawn only */ + Eterm parent_id; + Eterm group_leader; + Eterm mfa; + DistEntry *dist_entry; + ErtsDistExternal *edep; + ErlHeapFragment *ede_hfrag; + Eterm token; /* * The following items are only initialized if the SPO_USE_ARGS flag is set. @@ -1354,6 +1402,7 @@ typedef struct { Uint max_heap_size; /* Maximum heap size in words */ Uint max_heap_flags; /* Maximum heap flags (kill | log) */ int scheduler; + } ErlSpawnOpts; /* @@ -1541,6 +1590,7 @@ extern int erts_system_profile_ts_type; #define SEQ_TRACE_SEND (1 << 0) #define SEQ_TRACE_RECEIVE (1 << 1) #define SEQ_TRACE_PRINT (1 << 2) +/* (This three-bit gap contains the timestamp.) */ #define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3 @@ -1852,6 +1902,12 @@ Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); Process *erts_schedule(ErtsSchedulerData *, Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); +int erts_parse_spawn_opts(ErlSpawnOpts *sop, Eterm opts_list, Eterm *tag, + int success_message_opt); +void +erts_send_local_spawn_reply(Process *parent, ErtsProcLocks parent_locks, + Process *child, Eterm tag, Eterm ref, + Eterm result, Eterm token); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_set_self_exiting(Process *, Eterm); void erts_do_exit_process(Process*, Eterm); @@ -1882,12 +1938,17 @@ typedef struct { Eterm reason; ErtsLink *dist_links; ErtsMonitor *dist_monitors; + ErtsMonitor *pend_spawn_monitors; + ErtsMonitor *wait_pend_spawn_monitor; Eterm dist_state; int yield; } ErtsProcExitContext; int erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds); int erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt, Sint reds); +void erts_proc_exit_dist_demonitor(Process *c_p, DistEntry *dep, Uint32 conn_id, + Eterm ref, Eterm watched); + Eterm erts_get_process_priority(erts_aint32_t state); Eterm erts_set_process_priority(Process *p, Eterm prio); @@ -1917,6 +1978,7 @@ Uint erts_debug_nbalance(void); #define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS (1 << 0) #define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS (1 << 1) #define ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK (1 << 2) +#define ERTS_DEBUG_WAIT_COMPLETED_THREAD_PROGRESS (1 << 3) int erts_debug_wait_completed(Process *c_p, int flags); @@ -2089,10 +2151,10 @@ erts_psd_set(Process *p, int ix, void *data) #define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \ ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) -#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \ - erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT) -#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ - erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#define ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(P) \ + erts_psd_get((P), ERTS_PSD_NFUNC_TRAP_WRAPPER) +#define ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(P, NTE) \ + erts_psd_set((P), ERTS_PSD_NFUNC_TRAP_WRAPPER, (void *) (NTE)) #define ERTS_PROC_GET_DIST_ENTRY(P) \ ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) @@ -2696,7 +2758,9 @@ ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); void erts_aux_thread_poke(void); ERTS_GLB_INLINE Uint32 erts_sched_local_random_hash_64_to_32_shift(Uint64 key); ERTS_GLB_INLINE Uint32 erts_sched_local_random(Uint additional_seed); - +#ifdef DEBUG +ERTS_GLB_INLINE float erts_sched_local_random_float(Uint additional_seed); +#endif #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -2746,6 +2810,20 @@ Uint32 erts_sched_local_random(Uint additional_seed) return erts_sched_local_random_hash_64_to_32_shift(seed); } +#ifdef DEBUG + +/* + * This function returns a random float between 0.0 and 1.0. + */ +ERTS_GLB_INLINE +float erts_sched_local_random_float(Uint additional_seed) +{ + Uint32 rnd = erts_sched_local_random(additional_seed); + return (float)(((double)rnd)/((double)ERTS_UINT32_MAX)); +} + +#endif /* #ifdef DEBUG */ + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index cc483a2148..f1f066be46 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -830,7 +830,7 @@ trace_receive(Process* receiver, } int -seq_trace_update_send(Process *p) +seq_trace_update_serial(Process *p) { ErtsTracer seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); @@ -853,6 +853,18 @@ seq_trace_update_send(Process *p) return 1; } +void +erts_seq_trace_update_node_token(Eterm token) +{ + Eterm serial; + Uint serial_num; + SEQ_TRACE_T_SENDER(token) = erts_this_dist_entry->sysname; + serial = SEQ_TRACE_T_SERIAL(token); + serial_num = unsigned_val(serial); + serial_num++; + SEQ_TRACE_T_SERIAL(token) = make_small(serial_num); +} + /* Send a sequential trace message to the sequential tracer. * p is the caller (which contains the trace token), @@ -929,6 +941,9 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, #undef LOCAL_HEAP_SIZE } + + + /* Send {trace_ts, Pid, return_to, {Mod, Func, Arity}, Timestamp} * or {trace, Pid, return_to, {Mod, Func, Arity}} */ diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index af38ef52db..2c5adf3198 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -142,12 +142,6 @@ void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp); -Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); -Eterm -erts_bif_trace_epilogue(Process *p, Eterm result, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); #define ERTS_CHK_PEND_TRACE_MSGS(ESDP) \ @@ -163,7 +157,10 @@ seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom)) void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm receiver, Process *process, Eterm exitfrom); -int seq_trace_update_send(Process *process); +/* Bump the sequence number if tracing is enabled; must be used before sending + * send trace messages. */ +int seq_trace_update_serial(Process *process); +void erts_seq_trace_update_node_token(Eterm token); Eterm erts_seq_trace(Process *process, Eterm atom_type, Eterm atom_true_or_false, diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 430ac305c5..895f8e5e27 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -70,6 +70,7 @@ int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_hash2(Eterm); +Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm, Uint32 salt); @@ -130,15 +131,28 @@ Sint erts_cmp_compound(Eterm, Eterm, int, int); #define CMP_GT(a,b) ((a) != (b) && CMP((a),(b)) > 0) #define CMP_EQ_ACTION(X,Y,Action) \ - if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); } + if ((X) != (Y)) { EQ_SPEC((X),(Y),!=,Action); } #define CMP_NE_ACTION(X,Y,Action) \ - if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),==,Action,1); } -#define CMP_GE_ACTION(X,Y,Action) \ - if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action,0); } + if ((X) == (Y)) { Action; } else { EQ_SPEC((X),(Y),==,Action); } + +#define EQ_SPEC(X,Y,Op,Action) \ + if (is_both_immed(X, Y)) { \ + if (X Op Y) { Action; }; \ + } else if (is_float(X) && is_float(Y)) { \ + FloatDef af, bf; \ + GET_DOUBLE(X, af); \ + GET_DOUBLE(Y, bf); \ + if (af.fd Op bf.fd) { Action; }; \ + } else { \ + if (erts_cmp_compound(X,Y,0,1) Op 0) { Action; }; \ + } + +#define CMP_GE_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action); } #define CMP_LT_ACTION(X,Y,Action) \ - if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action,0); } + if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action); } -#define CMP_SPEC(X,Y,Op,Action,EqOnly) \ +#define CMP_SPEC(X,Y,Op,Action) \ if (is_atom(X) && is_atom(Y)) { \ if (erts_cmp_atoms(X, Y) Op 0) { Action; }; \ } else if (is_both_small(X, Y)) { \ @@ -149,7 +163,26 @@ Sint erts_cmp_compound(Eterm, Eterm, int, int); GET_DOUBLE(Y, bf); \ if (af.fd Op bf.fd) { Action; }; \ } else { \ - if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; }; \ + if (erts_cmp_compound(X,Y,0,0) Op 0) { Action; }; \ + } + +/* + * When either operand for is_lt or is_ge is a literal, that literal is + * almost always an integer and almost never an atom. Therefore, only + * special case the comparison of small integers before calling the + * general compare function. + */ + +#define CMP_GE_LITERAL_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_LITERAL_SPEC((X),(Y),<,Action); } +#define CMP_LT_LITERAL_ACTION(X,Y,Action) \ + if ((X) == (Y)) { Action; } else { CMP_LITERAL_SPEC((X),(Y),>=,Action); } + +#define CMP_LITERAL_SPEC(X,Y,Op,Action) \ + if (is_both_small(X, Y)) { \ + if (signed_val(X) Op signed_val(Y)) { Action; }; \ + } else { \ + if (erts_cmp_compound(X,Y,0,0) Op 0) { Action; }; \ } #define erts_float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1)) diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index 8792138d53..2af2af2b37 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -176,7 +176,7 @@ provider erlang { * Fired whenever a user function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from * @param depth the stack depth */ probe function__return(char *p, char *mfa, int depth); @@ -193,7 +193,7 @@ provider erlang { * Fired whenever a Built In Function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from */ probe bif__return(char *p, char *mfa); @@ -209,7 +209,7 @@ provider erlang { * Fired whenever a Native Function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from */ probe nif__return(char *p, char *mfa); @@ -617,29 +617,6 @@ provider erlang { probe driver__stop_select(char *name); - /* Async driver pool */ - - /** - * Show the post-add length of the async driver thread pool member's queue. - * - * NOTE: The port name is not available: additional lock(s) must - * be acquired in order to get the port name safely in an SMP - * environment. The same is true for the aio__pool_get probe. - * - * @param port the Port (string form) - * @param new queue length - */ - probe aio_pool__add(char *, int); - - /** - * Show the post-get length of the async driver thread pool member's queue. - * - * @param port the Port (string form) - * @param new queue length - */ - probe aio_pool__get(char *, int); - - /* * The set of probes called by the erlang tracer nif backend. In order * to receive events on these you both have to enable tracing in erlang @@ -678,16 +655,6 @@ provider erlang { */ /* - * NOTE: For file_drv_return + SMP + R14B03 (and perhaps other - * releases), the sched-thread-id will be the same as the - * work-thread-id: erl_async.c's async_main() function - * will call the asynchronous invoke function and then - * immediately call the drivers ready_async function while - * inside the same I/O worker pool thread. - * For R14B03's source, see erl_async.c lines 302-317. - */ - -/* * The set of probes for use by Erlang code ... moved to here from * lib/runtime_tools/c_src/dtrace_user.d until a more portable solution * is found. This move pollutes the Erlang VM with functions that are diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h index 9b93d77f6e..213830e6fd 100644 --- a/erts/emulator/beam/erlang_lttng.h +++ b/erts/emulator/beam/erlang_lttng.h @@ -30,21 +30,6 @@ #include <lttng/tracepoint.h> -/* Schedulers */ - -TRACEPOINT_EVENT( - org_erlang_otp, - scheduler_poll, - TP_ARGS( - int, id, - int, runnable - ), - TP_FIELDS( - ctf_integer(int, scheduler, id) - ctf_integer(int, runnable, runnable) - ) -) - #ifndef LTTNG_CARRIER_STATS #define LTTNG_CARRIER_STATS typedef struct { @@ -292,35 +277,6 @@ TRACEPOINT_EVENT( ) ) -/* Async pool */ - -TRACEPOINT_EVENT( - org_erlang_otp, - aio_pool_get, - TP_ARGS( - char*, port, - int, length - ), - TP_FIELDS( - ctf_string(port, port) - ctf_integer(int, length, length) - ) -) - -TRACEPOINT_EVENT( - org_erlang_otp, - aio_pool_put, - TP_ARGS( - char*, port, - int, length - ), - TP_FIELDS( - ctf_string(port, port) - ctf_integer(int, length, length) - ) -) - - /* Memory Allocator */ TRACEPOINT_EVENT( diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 64c08b1570..cc80fd09df 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -66,13 +66,13 @@ #define EXF_OFFSET EXTAG_BITS #define EXF_BITS 7 -#define EXF_PANIC (1<<(0+EXF_OFFSET)) /* ignore catches */ -#define EXF_THROWN (1<<(1+EXF_OFFSET)) /* nonlocal return */ -#define EXF_LOG (1<<(2+EXF_OFFSET)) /* write to logger on termination */ -#define EXF_NATIVE (1<<(3+EXF_OFFSET)) /* occurred in native code */ -#define EXF_SAVETRACE (1<<(4+EXF_OFFSET)) /* save stack trace in internal form */ -#define EXF_ARGLIST (1<<(5+EXF_OFFSET)) /* has arglist for top of trace */ -#define EXF_RESTORE_NIF (1<<(6+EXF_OFFSET)) /* restore original bif/nif */ +#define EXF_PANIC (1<<(0+EXF_OFFSET)) /* ignore catches */ +#define EXF_THROWN (1<<(1+EXF_OFFSET)) /* nonlocal return */ +#define EXF_LOG (1<<(2+EXF_OFFSET)) /* write to logger on termination */ +#define EXF_NATIVE (1<<(3+EXF_OFFSET)) /* occurred in native code */ +#define EXF_SAVETRACE (1<<(4+EXF_OFFSET)) /* save stack trace in internal form */ +#define EXF_ARGLIST (1<<(5+EXF_OFFSET)) /* has arglist for top of trace */ +#define EXF_RESTORE_NFUNC (1<<(6+EXF_OFFSET)) /* restore original bif/nif */ #define EXC_FLAGBITS (((1<<(EXF_BITS+EXF_OFFSET))-1) \ & ~((1<<(EXF_OFFSET))-1)) @@ -155,10 +155,8 @@ /* No matching try clause */ #define EXC_NOTSUP ((17 << EXC_OFFSET) | EXC_ERROR) /* Not supported */ - #define EXC_BADMAP ((18 << EXC_OFFSET) | EXC_ERROR) /* Bad map */ - #define EXC_BADKEY ((19 << EXC_OFFSET) | EXC_ERROR) /* Bad key in map */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 946ffeffb8..af1b1c2892 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -129,14 +129,17 @@ export_alloc(struct export_entry* tmpl_e) obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; - obj->beam[0] = 0; + obj->bif_number = -1; + obj->is_bif_traced = 0; + + memset(&obj->trampoline, 0, sizeof(obj->trampoline)); + if (BeamOpsAreInitialized()) { - obj->beam[0] = BeamOpCodeAddr(op_call_error_handler); + obj->trampoline.op = BeamOpCodeAddr(op_call_error_handler); } - obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { - obj->addressv[ix] = obj->beam; + obj->addressv[ix] = obj->trampoline.raw; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; @@ -196,6 +199,19 @@ init_export_table(void) } } +static struct export_entry* init_template(struct export_templ* templ, + Eterm m, Eterm f, unsigned a) +{ + templ->entry.ep = &templ->exp; + templ->entry.slot.index = -1; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; + templ->exp.bif_number = -1; + templ->exp.is_bif_traced = 0; + return &templ->entry; +} + /* * Return a pointer to the export entry for the given function, * or NULL otherwise. Notes: @@ -214,41 +230,15 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a,ErtsCodeIndex code_ix); Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { - HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); - int ix; - HashBucket* b; - - ix = hval % export_tables[code_ix].htable.size; - b = export_tables[code_ix].htable.bucket[ix]; - - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b != (HashBucket*) 0) { - Export* ep = ((struct export_entry*) b)->ep; - if (ep->info.mfa.module == m && - ep->info.mfa.function == f && - ep->info.mfa.arity == a) { - return ep; - } - b = b->next; - } + struct export_templ templ; + struct export_entry *ee = + hash_fetch(&export_tables[code_ix].htable, + init_template(&templ, m, f, a), + (H_FUN)export_hash, (HCMP_FUN)export_cmp); + if (ee) return ee->ep; return NULL; } -static struct export_entry* init_template(struct export_templ* templ, - Eterm m, Eterm f, unsigned a) -{ - templ->entry.ep = &templ->exp; - templ->entry.slot.index = -1; - templ->exp.info.mfa.module = m; - templ->exp.info.mfa.function = f; - templ->exp.info.mfa.arity = a; - return &templ->entry; -} - - /* * Find the export entry for a loaded function. * Returns a NULL pointer if the given function is not loaded, or @@ -268,8 +258,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->beam && - ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->trampoline.raw && + ! BeamIsOpCode(ee->ep->trampoline.op, op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index ae8dfa4cf8..91c4844d20 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -31,24 +31,72 @@ typedef struct export { - void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - - ErtsCodeInfo info; /* MUST be just before beam[] */ - - /* - * beam[0]: This entry is 0 unless the 'addressv' field points to it. - * Threaded code instruction to load function - * (em_call_error_handler), execute BIF (em_apply_bif), - * or a breakpoint instruction (op_i_generic_breakpoint). - * beam[1]: Function pointer to BIF function (for BIFs only), - * or pointer to threaded code if the module has an - * on_load function that has not been run yet, or pointer - * to code if function beam[0] is a breakpoint instruction. - * Otherwise: 0. - */ - BeamInstr beam[2]; + /* Pointer to code for function. */ + void* addressv[ERTS_NUM_CODE_IX]; + + /* Index into bif_table[], or -1 if not a BIF. */ + int bif_number; + /* Non-zero if this is a BIF that's traced. */ + int is_bif_traced; + + /* This is a small trampoline function that can be used for lazy code + * loading, global call tracing, and so on. It's only valid when + * addressv points to it and should otherwise be left zeroed. + * + * Needless to say, the order of the fields below is significant. */ + ErtsCodeInfo info; + union { + BeamInstr op; /* Union discriminant. */ + + struct { + BeamInstr op; /* op_i_generic_breakpoint */ + BeamInstr address; /* Address of the original function */ + } breakpoint; + + /* This is used when a module refers to (imports) a function that + * hasn't been loaded yet. Upon loading we create an export entry which + * redirects to the error_handler so that the appropriate module will + * be loaded when called (or crash). + * + * This is also used when a module has an on_load callback as we need + * to defer all calls until the callback returns. `deferred` contains + * the address of the original function in this case, and there's an + * awkward condiditon where `deferred` may be set while op is zero. See + * erlang:finish_after_on_load/2 for details. */ + struct { + BeamInstr op; /* op_call_error_handler, or 0 during the last + * phase of code loading when on_load is + * present. See above. */ + BeamInstr deferred; + } not_loaded; + + struct { + BeamInstr op; /* op_trace_jump_W */ + BeamInstr address; /* Address of the traced function */ + } trace; + + BeamInstr raw[2]; /* For use in address comparisons, should not + * be tampered directly. */ + } trampoline; } Export; +#ifdef DEBUG +#define DBG_CHECK_EXPORT(EP, CX) \ + do { \ + if((EP)->addressv[CX] == (EP)->trampoline.raw) { \ + /* The entry currently points at the trampoline, so the + * instructions must be valid. */ \ + ASSERT(((BeamIsOpCode((EP)->trampoline.op, op_i_generic_breakpoint)) && \ + (EP)->trampoline.breakpoint.address != 0) || \ + ((BeamIsOpCode((EP)->trampoline.op, op_trace_jump_W)) && \ + (EP)->trampoline.trace.address != 0) || \ + /* (EP)->trampoline.not_loaded.deferred may be zero. */ \ + (BeamIsOpCode((EP)->trampoline.op, op_call_error_handler))); \ + } \ + } while(0) +#else +#define DBG_CHECK_EXPORT(EP, CX) ((void)(EP), (void)(CX)) +#endif void init_export_table(void); void export_info(fmtfn_t, void *); @@ -71,9 +119,6 @@ extern erts_mtx_t export_staging_lock; #define export_staging_unlock() erts_mtx_unlock(&export_staging_lock) #include "beam_load.h" /* For em_* extern declarations */ -#define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ - (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif))) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 12eda60527..5d91c1b2cb 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -46,23 +46,26 @@ #include "erl_bits.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_proc_sig_queue.h" +#include "erl_trace.h" + +#define PASS_THROUGH 'p' #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff -/* MAX value for the creation field in pid, port and reference - for the local node and for the current external format. - - Larger creation values than this are allowed in external pid, port and refs - encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT. - The point here is to prepare for future upgrade to 32-bit creation. - OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes, - but do not use big creation values for the local node yet, - as we still may have to communicate with older nodes. +/* + * MAX value for the creation field in pid, port and reference + * for the old PID_EXT, PORT_EXT, REFERENCE_EXT and NEW_REFERENCE_EXT. + * Older nodes (OTP 19-22) will send us these so we must be able to decode them. + * + * From OTP 23 DFLAG_BIG_CREATION is mandatory so this node will always + * encode with new big 32-bit creations using NEW_PID_EXT, NEW_PORT_EXT + * and NEWER_REFERENCE_EXT. */ -#define ERTS_MAX_LOCAL_CREATION (3) -#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION) +#define ERTS_MAX_TINY_CREATION (3) +#define is_tiny_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_TINY_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -98,13 +101,13 @@ static Export term_to_binary_trap_export; -static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); +static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint64, struct erl_off_heap_header** off_heap); struct TTBEncodeContext_; -static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, +static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint64 dflags, struct erl_off_heap_header** off_heap, Sint *reds, byte **res); static int is_external_string(Eterm obj, Uint* lenp); -static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); -static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); +static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint64); +static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint64); struct B2TContext_t; static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*, int); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); @@ -112,19 +115,23 @@ static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*, byte t static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); -static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int level, - Uint flags, Binary *context_b); +static Eterm erts_term_to_binary_int(Process* p, Sint bif_ix, Eterm Term, Eterm opts, int level, + Uint64 dflags, Binary *context_b, int iovec, + Uint fragment_size); -static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); -struct TTBSizeContext_; -static ErtsExtSzRes encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, - Eterm obj, unsigned dflags, Sint *reds, Uint *res); +static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, Uint64); +static ErtsExtSzRes encode_size_struct_int(TTBSizeContext*, ErtsAtomCacheMap *acmp, + Eterm obj, Uint64 dflags, Sint *reds, Uint *res); static Export binary_to_term_trap_export; static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); -static Sint transcode_dist_obuf(ErtsDistOutputBuf*, DistEntry*, Uint32 dflags, Sint reds); - - +static Sint transcode_dist_obuf(ErtsDistOutputBuf*, DistEntry*, Uint64 dflags, Sint reds); +static byte *hopefull_bit_binary(TTBEncodeContext* ctx, byte **epp, Binary *pb_val, Eterm pb_term, + byte *bytes, byte bitoffs, byte bitsize, Uint sz); +static void hopefull_export(TTBEncodeContext* ctx, byte **epp, Export* exp, Uint32 dflags, + struct erl_off_heap_header** off_heap); +static void store_in_vec(TTBEncodeContext *ctx, byte *ep, Binary *ohbin, Eterm ohpb, + byte *ohp, Uint ohsz); void erts_init_external(void) { erts_init_trap_export(&term_to_binary_trap_export, @@ -221,7 +228,7 @@ erts_destroy_atom_cache_map(ErtsAtomCacheMap *acmp) } static ERTS_INLINE void -insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) +insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint64 dflags) { if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES) { int ix; @@ -237,7 +244,7 @@ insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) } static ERTS_INLINE int -get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) +get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint64 dflags) { if (!acmp) return -1; @@ -257,7 +264,7 @@ get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) } void -erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) +erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint64 dflags) { if (acmp) { int long_atoms = 0; /* !0 if one or more atoms are longer than 255. */ @@ -295,10 +302,16 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) } Uint -erts_encode_ext_dist_header_size(ErtsAtomCacheMap *acmp, Uint fragments) +erts_encode_ext_dist_header_size(TTBEncodeContext *ctx, + ErtsAtomCacheMap *acmp, + Uint fragments) { - if (!acmp) - return 0; + if (ctx->dflags & DFLAG_PENDING_CONNECT) { + /* HOPEFUL_DATA + hopefull flags + hopefull ix + payload ix */ + return 1 + 8 + 4 + 4; + } + else if (!acmp && !(ctx->dflags & DFLAG_FRAGMENTS)) + return 1; /* pass through */ else { int fix_sz = 1 /* VERSION_MAGIC */ @@ -306,48 +319,82 @@ erts_encode_ext_dist_header_size(ErtsAtomCacheMap *acmp, Uint fragments) + 1 /* dist header flags */ + 1 /* number of internal cache entries */ ; - ASSERT(acmp->hdr_sz >= 0); + if (fragments > 1) fix_sz += 8 /* sequence id */ + 8 /* number of fragments */ ; - return fix_sz + acmp->hdr_sz; + if (acmp) { + ASSERT(acmp->hdr_sz >= 0); + fix_sz += acmp->hdr_sz; + } else { + ASSERT(ctx->dflags & DFLAG_FRAGMENTS); + } + + return fix_sz; } } -byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp, +byte *erts_encode_ext_dist_header_setup(TTBEncodeContext *ctx, + byte *ctl_ext, ErtsAtomCacheMap *acmp, Uint fragments, Eterm from) { /* Maximum number of atom must be less than the maximum of a 32 bits unsigned integer. Check is done in erl_init.c, erl_start function. */ - if (!acmp) - return ctl_ext; + if (ctx->dflags & DFLAG_PENDING_CONNECT) { + byte *ep = ctl_ext; + ep -= 4; + ctx->payload_ixp = ep; + put_int32(0, ep); + ep -= 4; + ctx->hopefull_ixp = ep; + put_int32(ERTS_NO_HIX, ep); + ep -= 8; + ctx->hopefull_flagsp = ep; + put_int64(0, ep); + *--ep = HOPEFUL_DATA; + return ep; + } + else if (!acmp && !(ctx->dflags & DFLAG_FRAGMENTS)) { + byte *ep = ctl_ext; + *--ep = PASS_THROUGH; + return ep; + } else { int i; byte *ep = ctl_ext; - byte dist_hdr_flags = acmp->long_atoms ? ERTS_DIST_HDR_LONG_ATOMS_FLG : 0; - ASSERT(acmp->hdr_sz >= 0); - /* - * Write cache update instructions. Note that this is a purely - * internal format, never seen on the wire. This section is later - * rewritten by erts_encode_ext_dist_header_finalize() while updating - * the cache. We write the header backwards just before the - * actual term(s). - */ - for (i = acmp->sz-1; i >= 0; i--) { - Uint32 aval; - ASSERT(0 <= acmp->cix[i] && acmp->cix[i] < ERTS_ATOM_CACHE_SIZE); - ASSERT(i == acmp->cache[acmp->cix[i]].iix); - ASSERT(is_atom(acmp->cache[acmp->cix[i]].atom)); - - aval = (Uint32) atom_val(acmp->cache[acmp->cix[i]].atom); - ep -= 4; - put_int32(aval, ep); - ep -= 2; - put_int16(acmp->cix[i], ep); - } - --ep; - put_int8(acmp->sz, ep); + byte dist_hdr_flags = acmp && acmp->long_atoms ? ERTS_DIST_HDR_LONG_ATOMS_FLG : 0; + ASSERT(!acmp || acmp->hdr_sz >= 0); + + if (acmp) { + /* + * Write cache update instructions. Note that this is a purely + * internal format, never seen on the wire. This section is later + * rewritten by erts_encode_ext_dist_header_finalize() while updating + * the cache. We write the header backwards just before the + * actual term(s). + */ + for (i = acmp->sz-1; i >= 0; i--) { + Uint32 aval; + ASSERT(0 <= acmp->cix[i] && acmp->cix[i] < ERTS_ATOM_CACHE_SIZE); + ASSERT(i == acmp->cache[acmp->cix[i]].iix); + ASSERT(is_atom(acmp->cache[acmp->cix[i]].atom)); + + aval = (Uint32) atom_val(acmp->cache[acmp->cix[i]].atom); + ep -= 4; + put_int32(aval, ep); + ep -= 2; + put_int16(acmp->cix[i], ep); + } + --ep; + put_int8(acmp->sz, ep); + } else { + ASSERT(ctx->dflags & DFLAG_FRAGMENTS); + /* If we don't have an atom cache but are using a dist header we just put 0 + in the atom cache size slot */ + --ep; + put_int8(0, ep); + } --ep; put_int8(dist_hdr_flags, ep); if (fragments > 1) { @@ -382,11 +429,9 @@ byte *erts_encode_ext_dist_header_fragment(byte **hdrpp, } -#define PASS_THROUGH 'p' - Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, DistEntry* dep, - Uint32 dflags, + Uint64 dflags, Sint reds) { byte *ip; @@ -395,69 +440,34 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, byte dist_hdr_flags; int long_atoms; Uint64 seq_id = 0, frag_id = 0; - register byte *ep = ob->hdrp ? ob->hdrp : ob->extp; + register byte *ep = ob->eiov->iov[1].iov_base; ASSERT(dflags & DFLAG_UTF8_ATOMS); /* * The buffer can have different layouts at this point depending on * what was known when encoded: * - * Pending connection: CtrlTerm [, MsgTerm] + * Pending connection: HOPEFUL_DATA, HFlgs, HIX, PIX, CtrlTerm [, MsgTerm] * With atom cache : VERSION_MAGIC, DIST_HEADER, ..., CtrlTerm [, MsgTerm] * No atom cache : VERSION_MAGIC, CtrlTerm [, VERSION_MAGIC, MsgTerm] */ - if (ep[0] != VERSION_MAGIC || dep->transcode_ctx) { - /* - * Was encoded without atom cache toward pending connection. - */ - ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT); - - if (~dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME) - && ep[0] == SMALL_TUPLE_EXT - && ep[1] == 4 - && ep[2] == SMALL_INTEGER_EXT - && (ep[3] == DOP_MONITOR_P || - ep[3] == DOP_MONITOR_P_EXIT || - ep[3] == DOP_DEMONITOR_P)) { - /* - * Receiver does not support process monitoring. - * Suppress monitor control msg (see erts_dsig_send_monitor) - * by converting it to an empty (tick) packet. - */ - ob->ext_endp = ob->extp; - return reds; - } - if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG - | DFLAG_DIST_HDR_ATOM_CACHE)) { - reds = transcode_dist_obuf(ob, dep, dflags, reds); - if (reds < 0) - return reds; - ep = ob->extp; - } - if (dflags & DFLAG_DIST_HDR_ATOM_CACHE) { - /* - * Encoding was done without atom caching but receiver expects - * a dist header, so we prepend an empty one. - */ - *--ep = 0; /* NumberOfAtomCacheRefs */ - *--ep = DIST_HEADER; - *--ep = VERSION_MAGIC; - } - goto done; - } - else if (ep[1] != DIST_HEADER && ep[1] != DIST_FRAG_HEADER && ep[1] != DIST_FRAG_CONT) { - ASSERT(ep[1] == SMALL_TUPLE_EXT || ep[1] == LARGE_TUPLE_EXT); - ASSERT(!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)); - /* Node without atom cache, 'pass through' needed */ - *--ep = PASS_THROUGH; - goto done; + if (ep[0] == HOPEFUL_DATA) + return transcode_dist_obuf(ob, dep, dflags, reds); + + if (ep[0] == PASS_THROUGH) { + ASSERT(!(dflags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_FRAGMENTS))); + ASSERT(ob->eiov->iov[1].iov_len == 1); + return reds; } if (ep[1] == DIST_FRAG_CONT) { - ep = ob->extp; - goto done; - } else if (ep[1] == DIST_FRAG_HEADER) { + ASSERT(ep[0] == VERSION_MAGIC); + ASSERT(ob->eiov->iov[1].iov_len == 18); + return reds; + } + + if (ep[1] == DIST_FRAG_HEADER) { /* skip the seq id and frag id */ seq_id = get_int64(&ep[2]); ep += 8; @@ -481,10 +491,7 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, ip = &instr_buf[0]; sys_memcpy((void *) ip, (void *) ep, sz); ep += sz; - /* ep now points to the beginning of the control message term */ -#ifdef ERTS_DEBUG_USE_DIST_SEP - ASSERT(*ep == VERSION_MAGIC); -#endif + ASSERT(ep == (byte *) (ob->eiov->iov[1].iov_base + ob->eiov->iov[1].iov_len)); if (ci > 0) { Uint32 flgs_buf[((ERTS_DIST_HDR_ATOM_CACHE_FLAG_BYTES( ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES)-1) @@ -597,49 +604,82 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, *--ep = DIST_HEADER; } *--ep = VERSION_MAGIC; -done: - ob->extp = ep; - ASSERT((byte*)ob->bin->orig_bytes <= ob->extp && ob->extp < ob->ext_endp); + + sz = ((byte *) ob->eiov->iov[1].iov_base) - ep; + ob->eiov->size += sz; + ob->eiov->iov[1].iov_len += sz; + ob->eiov->iov[1].iov_base = ep; + return reds < 0 ? 0 : reds; } ErtsExtSzRes -erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, Uint* szp) +erts_encode_dist_ext_size(Eterm term, + ErtsAtomCacheMap *acmp, + TTBSizeContext* ctx, + Uint* szp, Sint *redsp, + Sint *vlenp, Uint *fragmentsp) { Uint sz; - ErtsExtSzRes res = encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz); - if (res == ERTS_EXT_SZ_OK) { + ErtsExtSzRes res; + + ASSERT(ctx); + ASSERT(szp); + ASSERT(vlenp); + ASSERT(fragmentsp); + + sz = *szp; + + if (!ctx->wstack.wstart) { + /* + * First call for this 'term'. We might however encode + * multiple terms and this might not be the first term + * in the sequence. 'ctx' should contain valid info about + * about previous terms regarding fragments, and vlen. + * 'szp' should contain valid info about the total size + * of previous terms. + */ + if (ctx->vlen < 0) { + /* First term as well */ + ctx->vlen = 0; + if (ctx->dflags & DFLAG_FRAGMENTS) + ctx->fragment_size = ERTS_DIST_FRAGMENT_SIZE; + } + #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) + if (!(ctx->dflags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_FRAGMENTS))) #endif sz++ /* VERSION_MAGIC */; - *szp += sz; } - return res; -} -ErtsExtSzRes -erts_encode_dist_ext_size_ctx(Eterm term, ErtsDSigSendContext *ctx, Uint* szp) -{ - Uint sz; - ErtsExtSzRes res = encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, - ctx->flags, &ctx->reds, &sz); + res = encode_size_struct_int(ctx, acmp, term, ctx->dflags, redsp, &sz); + if (res == ERTS_EXT_SZ_OK) { -#ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) -#endif - sz++ /* VERSION_MAGIC */; + Uint total_size, fragments; + + /* + * Each fragment use + * - one element for driver header + * - one element for fragment header + * - and (at least) one for data + */ + total_size = sz + ctx->extra_size; + fragments = (total_size - 1)/ctx->fragment_size + 1; - *szp += sz; + *szp = sz; + *fragmentsp = fragments; + *vlenp = ctx->vlen + 3*fragments; } + return res; } ErtsExtSzRes erts_encode_ext_size_2(Eterm term, unsigned dflags, Uint *szp) { - ErtsExtSzRes res = encode_size_struct_int(NULL, NULL, term, dflags, - NULL, szp); + ErtsExtSzRes res; + *szp = 0; + res = encode_size_struct_int(NULL, NULL, term, dflags, NULL, szp); (*szp)++ /* VERSION_MAGIC */; return res; } @@ -651,20 +691,44 @@ ErtsExtSzRes erts_encode_ext_size(Eterm term, Uint *szp) Uint erts_encode_ext_size_ets(Eterm term) { - return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAG_INTERNAL_TAGS); + return encode_size_struct2(NULL, term, + TERM_TO_BINARY_DFLAGS|DFLAG_ETS_COMPRESSED); } -int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp, - TTBEncodeContext* ctx, Sint* reds) +int erts_encode_dist_ext(Eterm term, byte **ext, Uint64 flags, ErtsAtomCacheMap *acmp, + TTBEncodeContext* ctx, Uint *fragmentsp, Sint* reds) { - if (!ctx || !ctx->wstack.wstart) { - #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) - #endif + int res; + ASSERT(ctx); + + if (!ctx->wstack.wstart) { + ctx->cptr = *ext; +#ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_PENDING_CONNECT|DFLAG_FRAGMENTS))) +#endif *(*ext)++ = VERSION_MAGIC; +#ifndef ERTS_DEBUG_USE_DIST_SEP + if (flags & DFLAG_PENDING_CONNECT) { + Sint payload_ix = ctx->vlen; + ASSERT(ctx->payload_ixp); + if (payload_ix) { + /* we potentially need a version magic on the payload... */ + (*ext)++; + ctx->cptr = *ext; + put_int32(payload_ix, ctx->payload_ixp); + } + } +#endif } - return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext); + res = enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext); + if (fragmentsp) + *fragmentsp = res == 0 ? ctx->frag_ix + 1 : ctx->frag_ix; + if (flags & DFLAG_PENDING_CONNECT) { + ASSERT(ctx->hopefull_flagsp); + put_int64(ctx->hopefull_flags, ctx->hopefull_flagsp); + } + return res; } void erts_encode_ext(Eterm term, byte **ext) @@ -681,7 +745,7 @@ void erts_encode_ext(Eterm term, byte **ext) byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap) { - return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAG_INTERNAL_TAGS, + return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAG_ETS_COMPRESSED, off_heap); } @@ -786,7 +850,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ASSERT(dep); erts_de_rlock(dep); - ASSERT(dep->flags & DFLAG_UTF8_ATOMS); + ASSERT(dep->dflags & DFLAG_UTF8_ATOMS); if ((dep->state != ERTS_DE_STATE_CONNECTED && @@ -796,7 +860,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, return ERTS_PREP_DIST_EXT_CLOSED; } - if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) { + if (!(dep->dflags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_FRAGMENTS))) { /* Skip PASS_THROUGH */ ext++; size--; @@ -826,7 +890,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, edep->data->seq_id = 0; edep->data->frag_id = 1; - if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) + if (dep->dflags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_FRAGMENTS)) edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; if (ep[1] != DIST_HEADER && ep[1] != DIST_FRAG_HEADER && ep[1] != DIST_FRAG_CONT) { @@ -836,7 +900,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, edep->data->extp = ext; } else if (ep[1] == DIST_FRAG_CONT) { - if (!(dep->flags & DFLAG_FRAGMENTS)) + if (!(dep->dflags & DFLAG_FRAGMENTS)) goto bad_hdr; edep->attab.size = 0; edep->data->extp = ext + 1 + 1 + 8 + 8; @@ -853,7 +917,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, goto bad_hdr; if (ep[1] == DIST_FRAG_HEADER) { - if (!(dep->flags & DFLAG_FRAGMENTS)) + if (!(dep->dflags & DFLAG_FRAGMENTS)) goto bad_hdr; edep->data->seq_id = get_int64(&ep[2]); edep->data->frag_id = get_int64(&ep[2+8]); @@ -1286,8 +1350,11 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) Eterm Term = tp[1]; Eterm Opts = tp[2]; Eterm bt = tp[3]; + Eterm bix = tp[4]; + Sint bif_ix = signed_val(bix); Binary *bin = erts_magic_ref2bin(bt); - Eterm res = erts_term_to_binary_int(BIF_P, Term, Opts, 0, 0,bin); + Eterm res = erts_term_to_binary_int(BIF_P, bif_ix, Term, Opts, + 0, 0,bin, 0, ~((Uint) 0)); if (is_non_value(res)) { if (erts_set_gc_state(BIF_P, 1) || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P)) { @@ -1295,10 +1362,10 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) } if (Opts == am_undefined) ERTS_BIF_ERROR_TRAPPED1(BIF_P, SYSTEM_LIMIT, - bif_export[BIF_term_to_binary_1], Term); + &bif_trap_export[bif_ix], Term); else ERTS_BIF_ERROR_TRAPPED2(BIF_P, SYSTEM_LIMIT, - bif_export[BIF_term_to_binary_2], Term, Opts); + &bif_trap_export[bif_ix], Term, Opts); } if (is_tuple(res)) { ASSERT(BIF_P->flags & F_DISABLE_GC); @@ -1316,8 +1383,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1) BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, am_undefined, - 0, TERM_TO_BINARY_DFLAGS, NULL); + Eterm res = erts_term_to_binary_int(BIF_P, BIF_term_to_binary_1, + BIF_ARG_1, am_undefined, + 0, TERM_TO_BINARY_DFLAGS, NULL, 0, + ~((Uint) 0)); if (is_non_value(res)) { ASSERT(!(BIF_P->flags & F_DISABLE_GC)); BIF_ERROR(BIF_P, SYSTEM_LIMIT); @@ -1331,22 +1400,43 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) } } -HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2) +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_iovec, 1) -BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) +BIF_RETTYPE term_to_iovec_1(BIF_ALIST_1) +{ + Eterm res = erts_term_to_binary_int(BIF_P, BIF_term_to_iovec_1, + BIF_ARG_1, am_undefined, + 0, TERM_TO_BINARY_DFLAGS, NULL, !0, + ~((Uint) 0)); + if (is_non_value(res)) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + } + if (is_tuple(res)) { + erts_set_gc_state(BIF_P, 0); + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_RET(res); + } +} + +static ERTS_INLINE int +parse_t2b_opts(Eterm opts, Uint *flagsp, int *levelp, int *iovecp, Uint *fsizep) { - Process* p = BIF_P; - Eterm Term = BIF_ARG_1; - Eterm Flags = BIF_ARG_2; int level = 0; + int iovec = 0; Uint flags = TERM_TO_BINARY_DFLAGS; - Eterm res; + Uint fsize = ~((Uint) 0); /* one fragment */ - while (is_list(Flags)) { - Eterm arg = CAR(list_val(Flags)); + while (is_list(opts)) { + Eterm arg = CAR(list_val(opts)); Eterm* tp; if (arg == am_compressed) { level = Z_DEFAULT_COMPRESSION; + } + else if (iovecp && arg == am_iovec) { + iovec = !0; } else if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) { if (tp[1] == am_minor_version && is_small(tp[2])) { switch (signed_val(tp[2])) { @@ -1360,34 +1450,66 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS; break; default: - goto error; + return 0; /* badarg */ } } else if (tp[1] == am_compressed && is_small(tp[2])) { level = signed_val(tp[2]); if (!(0 <= level && level < 10)) { - goto error; + return 0; /* badarg */ } - } else { - goto error; + } else if (fsizep) { + if (ERTS_IS_ATOM_STR("fragment", tp[1])) { + if (!term_to_Uint(tp[2], &fsize)) + return 0; /* badarg */ + } + else { + return 0; /* badarg */ + } + } + else { + return 0; /* badarg */ } } else { - error: - BIF_ERROR(p, BADARG); + return 0; /* badarg */ } - Flags = CDR(list_val(Flags)); + opts = CDR(list_val(opts)); } - if (is_not_nil(Flags)) { - goto error; + if (is_not_nil(opts)) { + return 0; /* badarg */ } - res = erts_term_to_binary_int(p, Term, BIF_ARG_2, - level, flags, NULL); + *flagsp = flags; + *levelp = level; + if (iovecp) + *iovecp = iovec; + if (fsizep) + *fsizep = fsize; + + return !0; /* ok */ +} + +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2) + +BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) +{ + int level; + Uint flags; + Eterm res; + + if (!parse_t2b_opts(BIF_ARG_2, &flags, &level, NULL, NULL)) { + BIF_ERROR(BIF_P, BADARG); + } + + res = erts_term_to_binary_int(BIF_P, BIF_term_to_binary_2, + BIF_ARG_1, BIF_ARG_2, + level, flags, NULL, 0, + ~((Uint) 0)); if (is_non_value(res)) { ASSERT(!(BIF_P->flags & F_DISABLE_GC)); BIF_ERROR(BIF_P, SYSTEM_LIMIT); } if (is_tuple(res)) { - erts_set_gc_state(p, 0); + erts_set_gc_state(BIF_P, 0); BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); } else { ASSERT(!(BIF_P->flags & F_DISABLE_GC)); @@ -1395,6 +1517,67 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_iovec, 2) + +BIF_RETTYPE term_to_iovec_2(BIF_ALIST_2) +{ + int level; + Uint flags; + Eterm res; + + if (!parse_t2b_opts(BIF_ARG_2, &flags, &level, NULL, NULL)) { + BIF_ERROR(BIF_P, BADARG); + } + + res = erts_term_to_binary_int(BIF_P, BIF_term_to_iovec_2, + BIF_ARG_1, BIF_ARG_2, + level, flags, NULL, !0, + ~((Uint) 0)); + if (is_non_value(res)) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + } + if (is_tuple(res)) { + erts_set_gc_state(BIF_P, 0); + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_RET(res); + } +} + +Eterm +erts_debug_term_to_binary(Process *p, Eterm term, Eterm opts) +{ + Eterm ret; + int level, iovec; + Uint flags; + Uint fsize; + + if (!parse_t2b_opts(opts, &flags, &level, &iovec, &fsize)) { + ERTS_BIF_PREP_ERROR(ret, p, BADARG); + } + else { + Eterm res = erts_term_to_binary_int(p, BIF_term_to_binary_2, + term, opts, level, flags, + NULL, iovec, fsize); + + if (is_non_value(res)) { + ASSERT(!(p->flags & F_DISABLE_GC)); + ERTS_BIF_PREP_ERROR(ret, p, SYSTEM_LIMIT); + } + else if (is_tuple(res)) { + erts_set_gc_state(p, 0); + ERTS_BIF_PREP_TRAP1(ret, &term_to_binary_trap_export,p,res); + } + else { + ASSERT(!(p->flags & F_DISABLE_GC)); + ERTS_BIF_PREP_RET(ret, res); + } + } + return ret; +} + enum B2TState { /* order is somewhat significant */ B2TPrepare, @@ -1804,8 +1987,8 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx) case B2TBadArg: BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1] - || ctx->bif == bif_export[BIF_binary_to_term_2]); + ASSERT(ctx->bif == &bif_trap_export[BIF_binary_to_term_1] + || ctx->bif == &bif_trap_export[BIF_binary_to_term_2]); if (is_first_call) ERTS_BIF_PREP_ERROR(ret_val, p, BADARG); @@ -1886,7 +2069,7 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) ctx.flags = 0; ctx.used_bytes = 0; ctx.trap_bin = THE_NON_VALUE; - ctx.bif = bif_export[BIF_binary_to_term_1]; + ctx.bif = &bif_trap_export[BIF_binary_to_term_1]; ctx.arg[0] = BIF_ARG_1; ctx.arg[1] = THE_NON_VALUE; return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx); @@ -1921,7 +2104,7 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) goto error; ctx.trap_bin = THE_NON_VALUE; - ctx.bif = bif_export[BIF_binary_to_term_2]; + ctx.bif = &bif_trap_export[BIF_binary_to_term_2]; ctx.arg[0] = BIF_ARG_1; ctx.arg[1] = BIF_ARG_2; return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx); @@ -1935,7 +2118,7 @@ external_size_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm Term = BIF_ARG_1; - Uint size; + Uint size = 0; switch (erts_encode_ext_size(Term, &size)) { case ERTS_EXT_SZ_SYSTEM_LIMIT: @@ -1957,7 +2140,7 @@ external_size_1(BIF_ALIST_1) Eterm external_size_2(BIF_ALIST_2) { - Uint size; + Uint size = 0; Uint flags = TERM_TO_BINARY_DFLAGS; while (is_list(BIF_ARG_2)) { @@ -2006,7 +2189,7 @@ external_size_2(BIF_ALIST_2) } static Eterm -erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint flags) +erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint64 dflags) { Eterm bin; size_t real_size; @@ -2022,7 +2205,7 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bytes = erts_alloc(ERTS_ALC_T_TMP, size); } - if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) + if ((endp = enc_term(NULL, Term, bytes, dflags, NULL)) == NULL) { erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -2067,7 +2250,7 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); bytes[0] = VERSION_MAGIC; - if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) + if ((endp = enc_term(NULL, Term, bytes+1, dflags, NULL)) == NULL) { erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -2082,8 +2265,8 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl } Eterm -erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { - Uint size; +erts_term_to_binary(Process* p, Eterm Term, int level, Uint64 flags) { + Uint size = 0; switch (encode_size_struct_int(NULL, NULL, Term, flags, NULL, &size)) { case ERTS_EXT_SZ_SYSTEM_LIMIT: return THE_NON_VALUE; @@ -2121,6 +2304,8 @@ static int ttb_context_destructor(Binary *context_bin) erts_bin_free(context->s.ec.result_bin); context->s.ec.result_bin = NULL; } + if (context->s.ec.iov) + erts_free(ERTS_ALC_T_T2B_VEC, context->s.ec.iov); break; case TTBCompress: erl_zlib_deflate_finish(&(context->s.cc.stream)); @@ -2142,8 +2327,77 @@ static int ttb_context_destructor(Binary *context_bin) return 1; } -static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int level, - Uint flags, Binary *context_b) +Uint +erts_ttb_iov_size(int use_termv, Sint vlen, Uint fragments) +{ + Uint sz; + ASSERT(vlen > 0); + ASSERT(fragments > 0); + sz = sizeof(SysIOVec)*vlen; + if (sz % sizeof(void *) != 0) + sz += sizeof(void *) - (sz % sizeof(void *)); + sz += sizeof(ErlDrvBinary *)*vlen; + if (use_termv) + sz += sizeof(Eterm)*vlen; + sz += sizeof(ErlIOVec *)*fragments; + sz += sizeof(ErlIOVec)*fragments; + if (sz % sizeof(void *) != 0) + sz += sizeof(void *) - (sz % sizeof(void *)); + return sz; +} + +ErlIOVec ** +erts_ttb_iov_init(TTBEncodeContext *ctx, int use_termv, char *ptr, + Sint vlen, Uint fragments, Uint fragment_size) +{ + int i; + ErlIOVec *feiovp; + ctx->vlen = 0; + ctx->size = 0; + + ctx->iov = (SysIOVec *) ptr; + ptr += sizeof(SysIOVec)*vlen; + if (((UWord) ptr) % sizeof(void *) != 0) + ptr += sizeof(void *) - (((UWord) ptr) % sizeof(void *)); + ASSERT(((UWord) ptr) % sizeof(void *) == 0); + + ctx->binv = (ErlDrvBinary **) ptr; + ptr += sizeof(ErlDrvBinary *)*vlen; + + if (!use_termv) + ctx->termv = NULL; + else { + ctx->termv = (Eterm *) ptr; + ptr += sizeof(Eterm)*vlen; + } + + ctx->fragment_eiovs = (ErlIOVec **) ptr; + ptr += sizeof(ErlIOVec *)*fragments; + + feiovp = (ErlIOVec *) ptr; + ptr += sizeof(ErlIOVec)*fragments; + + if (((UWord) ptr) % sizeof(void *) != 0) + ptr += sizeof(void *) - (((UWord) ptr) % sizeof(void *)); + ASSERT(((UWord) ptr) % sizeof(void *) == 0); + + for (i = 0; i < fragments; i++) + ctx->fragment_eiovs[i] = &feiovp[i]; + + ctx->frag_ix = -1; + ctx->fragment_size = fragment_size; + +#ifdef DEBUG + ctx->cptr = NULL; + ctx->debug_fragments = fragments; + ctx->debug_vlen = vlen; +#endif + return ctx->fragment_eiovs; +} + +static Eterm erts_term_to_binary_int(Process* p, Sint bif_ix, Eterm Term, Eterm opts, + int level, Uint64 dflags, Binary *context_b, + int iovec, Uint fragment_size) { Eterm *hp; Eterm res; @@ -2157,6 +2411,10 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int lev TTBContext c_buff; TTBContext *context = &c_buff; + ASSERT(bif_ix > 0 && IS_USMALL(!0, bif_ix)); + ASSERT(bif_ix == BIF_term_to_binary_1 || bif_ix == BIF_term_to_binary_2 + || bif_ix == BIF_term_to_iovec_1 || bif_ix == BIF_term_to_iovec_2); + #define EXPORT_CONTEXT() \ do { \ if (context_b == NULL) { \ @@ -2169,36 +2427,47 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int lev #define RETURN_STATE() \ do { \ - hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 1 + 3); \ + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 1 + 4); \ c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b); \ - res = TUPLE3(hp, Term, opts, c_term); \ + res = TUPLE4(hp, Term, opts, c_term, make_small(bif_ix)); \ BUMP_ALL_REDS(p); \ return res; \ } while (0); - if (context_b == NULL) { /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; - context->s.sc.wstack.wstart = NULL; - context->s.sc.flags = flags; + ERTS_INIT_TTBSizeContext(&context->s.sc, dflags); context->s.sc.level = level; + context->s.sc.fragment_size = fragment_size; + if (!level) { + context->s.sc.vlen = iovec ? 0 : -1; + context->s.sc.iovec = iovec; + } + else { + context->s.sc.vlen = -1; + context->s.sc.iovec = 0; + } } else { context = ERTS_MAGIC_BIN_DATA(context_b); - } + } + /* Initialization done, now we will go through the states */ for (;;) { switch (context->state) { case TTBSize: { - Uint size; + Uint size, fragments = 1; Binary *result_bin; - int level; - Uint flags; - /* Try for fast path */ + int level = context->s.sc.level; + Sint vlen; + iovec = context->s.sc.iovec; + fragment_size = context->s.sc.fragment_size; + size = 1; /* VERSION_MAGIC */ switch (encode_size_struct_int(&context->s.sc, NULL, Term, - context->s.sc.flags, &reds, &size)) { + context->s.sc.dflags, &reds, + &size)) { case ERTS_EXT_SZ_SYSTEM_LIMIT: BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); return THE_NON_VALUE; @@ -2209,14 +2478,23 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int lev case ERTS_EXT_SZ_OK: break; } - ++size; /* VERSION_MAGIC */ /* Move these to next state */ - flags = context->s.sc.flags; - level = context->s.sc.level; - if (size <= ERL_ONHEAP_BIN_LIMIT) { + dflags = context->s.sc.dflags; + vlen = context->s.sc.vlen; + if (vlen >= 0) { + Uint total_size = size + context->s.sc.extra_size; + fragments = (total_size - 1)/fragment_size + 1; + vlen += 3*fragments; + ASSERT(vlen); + } + else if (size <= ERL_ONHEAP_BIN_LIMIT) { /* Finish in one go */ res = erts_term_to_binary_simple(p, Term, size, - level, flags); + level, dflags); + if (iovec) { + Eterm *hp = HAlloc(p, 2); + res = CONS(hp, res, NIL); + } BUMP_REDS(p, 1); return res; } @@ -2225,37 +2503,156 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int lev result_bin->orig_bytes[0] = (byte)VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; - context->s.ec.flags = flags; + ERTS_INIT_TTBEncodeContext(&context->s.ec, dflags); context->s.ec.level = level; - context->s.ec.wstack.wstart = NULL; context->s.ec.result_bin = result_bin; + context->s.ec.iovec = iovec; + if (vlen >= 0) { + Uint sz = erts_ttb_iov_size(!0, vlen, fragments); + char *ptr = (char *) erts_alloc(ERTS_ALC_T_T2B_VEC, sz); + erts_ttb_iov_init(&context->s.ec, !0, ptr, vlen, + fragments, fragment_size); + context->s.ec.cptr = (byte *) &result_bin->orig_bytes[0]; + } break; } case TTBEncode: { - byte *endp; + byte *endp, *tmp; byte *bytes = (byte *) context->s.ec.result_bin->orig_bytes; size_t real_size; Binary *result_bin; + Sint realloc_offset; + Uint fragments; - flags = context->s.ec.flags; - if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { + dflags = context->s.ec.dflags; + if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, dflags, + NULL, &reds, &endp) < 0) { EXPORT_CONTEXT(); RETURN_STATE(); } real_size = endp - bytes; + tmp = (byte *) &context->s.ec.result_bin->orig_bytes[0]; result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); + realloc_offset = (byte *) &result_bin->orig_bytes[0] - tmp; level = context->s.ec.level; BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); if (level == 0 || real_size < 6) { /* We are done */ + Sint cbin_refc_diff; + Eterm result, rb_term, *hp, *hp_end; + Uint hsz; + int ix; + SysIOVec *iov; + Eterm *termv; return_normal: + fragments = context->s.ec.frag_ix + 1; context->s.ec.result_bin = NULL; context->alive = 0; if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } - return erts_build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), - result_bin); + if (!context->s.ec.iov) { + hsz = PROC_BIN_SIZE + (iovec ? 2 : 0); + hp = HAlloc(p, hsz); + result = erts_build_proc_bin(&MSO(p), hp, result_bin); + if (iovec) { + hp += PROC_BIN_SIZE; + result = CONS(hp, result, NIL); + } + return result; + } + iovec = context->s.ec.iovec; + ASSERT(iovec); + iov = context->s.ec.iov; + termv = context->s.ec.termv; + ASSERT(context->s.ec.vlen <= context->s.ec.debug_vlen); + ASSERT(fragments <= context->s.ec.debug_fragments); + /* first two elements should be unused */ + ASSERT(context->s.ec.vlen >= 3*fragments); + ASSERT(!iov[0].iov_base && !iov[0].iov_len); + ASSERT(!iov[1].iov_base && !iov[1].iov_len); + + hsz = (2 /* cons */ + + (PROC_BIN_SIZE > ERL_SUB_BIN_SIZE + ? PROC_BIN_SIZE + : ERL_SUB_BIN_SIZE)); /* max size per vec */ + hsz *= context->s.ec.vlen - 2*fragments; /* number of vecs */ + hp = HAlloc(p, hsz); + hp_end = hp + hsz; + rb_term = THE_NON_VALUE; + result = NIL; + ASSERT(erts_refc_read(&result_bin->intern.refc, 1) == 1); + cbin_refc_diff = -1; + for (ix = context->s.ec.vlen - 1; ix > 1; ix--) { + Eterm bin_term, pb_term; + Uint pb_size; + ProcBin *pb; + SysIOVec *iovp = &iov[ix]; + if (!iovp->iov_base) + continue; /* empty slot for header */ + pb_term = termv[ix]; + if (is_value(pb_term)) { + pb_size = binary_size(pb_term); + pb = (ProcBin *) binary_val(pb_term); + } + else { + iovp->iov_base = (void *) (((byte *) iovp->iov_base) + + realloc_offset); + pb_size = result_bin->orig_size; + if (is_non_value(rb_term)) + pb = NULL; + else { + pb = (ProcBin *) binary_val(rb_term); + pb_term = rb_term; + } + } + /* + * We intentionally avoid using sub binaries + * since the GC might convert those to heap + * binaries and by this ruin the nice preparation + * for usage of this data as I/O vector in + * nifs/drivers. + */ + if (is_value(pb_term) && iovp->iov_len == pb_size) + bin_term = pb_term; + else { + Binary *bin; + if (is_value(pb_term)) { + bin = ((ProcBin *) binary_val(pb_term))->val; + erts_refc_inc(&bin->intern.refc, 2); + } + else { + bin = result_bin; + cbin_refc_diff++; + } + pb = (ProcBin *) (char *) hp; + hp += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = (Uint) iovp->iov_len; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) pb; + pb->val = bin; + pb->bytes = (byte*) iovp->iov_base; + pb->flags = 0; + OH_OVERHEAD(&MSO(p), pb->size / sizeof(Eterm)); + bin_term = make_binary(pb); + } + result = CONS(hp, bin_term, result); + hp += 2; + } + ASSERT(hp <= hp_end); + HRelease(p, hp_end, hp); + context->s.ec.iov = NULL; + erts_free(ERTS_ALC_T_T2B_VEC, iov); + if (cbin_refc_diff) { + ASSERT(cbin_refc_diff >= -1); + if (cbin_refc_diff > 0) + erts_refc_add(&result_bin->intern.refc, + cbin_refc_diff, 1); + else + erts_bin_free(result_bin); + } + return result; } /* Continue with compression... */ /* To make absolutely sure that zlib does not barf on a reallocated context, @@ -2372,7 +2769,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, Eterm opts, int lev */ static byte* -enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) +enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint64 dflags) { int iix; int len; @@ -2380,7 +2777,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) ASSERT(is_atom(atom)); - if (dflags & DFLAG_INTERNAL_TAGS) { + if (dflags & DFLAG_ETS_COMPRESSED) { Uint aval = atom_val(atom); ASSERT(aval < (1<<24)); if (aval >= (1 << 16)) { @@ -2457,19 +2854,20 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) /* * We use this atom as sysname in local pid/port/refs - * for the ETS compressed format (DFLAG_INTERNAL_TAGS). + * for the ETS compressed format * */ #define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom static byte* -enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) +enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint64 dflags) { Uint on, os; - Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) + Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_ETS_COMPRESSED)) ? INTERNAL_LOCAL_SYSNAME : pid_node_name(pid)); Uint32 creation = pid_creation(pid); - byte* tagp = ep++; + + *ep++ = NEW_PID_EXT; /* insert atom here containing host and sysname */ ep = enc_atom(acmp, sysname, ep, dflags); @@ -2481,15 +2879,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = PID_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_pid(pid)); - *tagp = NEW_PID_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; return ep; } @@ -2609,7 +3000,7 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, if (tag == PID_EXT) { cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { return NULL; } } else { @@ -2653,7 +3044,7 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6) static byte* -enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, +enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint64 dflags, struct erl_off_heap_header** off_heap) { byte *res; @@ -2662,7 +3053,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, } static int -enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, +enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, + Uint64 dflags, struct erl_off_heap_header** off_heap, Sint *reds, byte **res) { DECLARE_WSTACK(s); @@ -2673,10 +3065,12 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm val; FloatDef f; Sint r = 0; + int use_iov = 0; if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; + use_iov = !!ctx->iov; if (ctx->wstack.wstart) { /* restore saved stacks and byte pointer */ WSTACK_RESTORE(s, &ctx->wstack); @@ -2867,28 +3261,21 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case REF_DEF: case EXTERNAL_REF_DEF: { Uint32 *ref_num; - Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) + Eterm sysname = (((dflags & DFLAG_ETS_COMPRESSED) && is_internal_ref(obj)) ? INTERNAL_LOCAL_SYSNAME : ref_node_name(obj)); Uint32 creation = ref_creation(obj); - byte* tagp = ep++; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); erts_magic_ref_save_bin(obj); + *ep++ = NEWER_REFERENCE_EXT; i = ref_no_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp, sysname, ep, dflags); - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = NEW_REFERENCE_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_ref(obj)); - *tagp = NEWER_REFERENCE_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -2898,24 +3285,17 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, } case PORT_DEF: case EXTERNAL_PORT_DEF: { - Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) + Eterm sysname = (((dflags & DFLAG_ETS_COMPRESSED) && is_internal_port(obj)) ? INTERNAL_LOCAL_SYSNAME : port_node_name(obj)); Uint32 creation = port_creation(obj); - byte* tagp = ep++; + *ep++ = NEW_PORT_EXT; ep = enc_atom(acmp, sysname, ep, dflags); j = port_number(obj); put_int32(j, ep); ep += 4; - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = PORT_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_port(obj)); - *tagp = NEW_PORT_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; break; } case LIST_DEF: @@ -3042,9 +3422,44 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint bitsize; byte* bytes; byte* data_dst; + Uint off_heap_bytesize = 0; + Uint off_heap_tail; + Eterm pb_term; + Binary *pb_val; + ASSERT(!(dflags & DFLAG_PENDING_CONNECT) || (ctx && ctx->iov)); + ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); - if (dflags & DFLAG_INTERNAL_TAGS) { + if (use_iov) { + if (bitoffs == 0) { + ProcBin* pb = (ProcBin*) binary_val(obj); + off_heap_bytesize = pb->size; + if (off_heap_bytesize <= ERL_ONHEAP_BIN_LIMIT) + off_heap_bytesize = 0; + else { + pb_term = obj; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*)pb; + pb_term = sub->orig; + pb = (ProcBin*) binary_val(pb_term); + } + if (pb->thing_word != HEADER_PROC_BIN) + off_heap_bytesize = 0; + else { + if (pb->flags) { + char* before_realloc = pb->val->orig_bytes; + erts_emasculate_writable_binary(pb); + bytes += (pb->val->orig_bytes - before_realloc); + ASSERT((byte *) &pb->val->orig_bytes[0] <= bytes + && bytes < ((byte *) &pb->val->orig_bytes[0] + + pb->val->orig_size)); + } + pb_val = pb->val; + } + } + } + } + else if (dflags & DFLAG_ETS_COMPRESSED) { ProcBin* pb = (ProcBin*) binary_val(obj); Uint bytesize = pb->size; if (pb->thing_word == HEADER_SUB_BIN) { @@ -3087,20 +3502,51 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, j = binary_size(obj); put_int32(j, ep); ep += 4; - data_dst = ep; - ep += j; + if (off_heap_bytesize) + off_heap_tail = 0; + else { + data_dst = ep; + ep += j; + } } else if (dflags & DFLAG_BIT_BINARIES) { /* Bit-level binary. */ - *ep++ = BIT_BINARY_EXT; - j = binary_size(obj); - put_int32((j+1), ep); - ep += 4; - *ep++ = bitsize; - ep[j] = 0; /* Zero unused bits at end of binary */ - data_dst = ep; - ep += j + 1; - if (ctx) - ctx->hopefull_flags |= DFLAG_BIT_BINARIES; + if (dflags & DFLAG_PENDING_CONNECT) { + j = off_heap_bytesize; + if (!j) { + pb_val = NULL; + pb_term = THE_NON_VALUE; + j = binary_size(obj); + } + data_dst = hopefull_bit_binary(ctx, &ep, pb_val, pb_term, + bytes, bitoffs, bitsize, j); + if (!data_dst) + break; /* off heap binary referred... */ + ASSERT(!off_heap_bytesize); + off_heap_tail = 0; + /* + * Trailing bits already written by hopefull_bit_binary(); + * now go copy all whole octets... + */ + bitsize = 0; + } + else { + *ep++ = BIT_BINARY_EXT; + j = binary_size(obj); + put_int32((j+1), ep); + ep += 4; + *ep++ = bitsize; + if (off_heap_bytesize) { + /* trailing bits */ + ep[0] = 0; + copy_binary_to_buffer(ep, 0, bytes + j, 0, bitsize); + off_heap_tail = 1; + } + else { + ep[j] = 0; /* Zero unused bits at end of binary */ + data_dst = ep; + ep += j + 1; + } + } } else { /* * Bit-level binary, but the receiver doesn't support it. @@ -3112,13 +3558,30 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, j = binary_size(obj); put_int32((j+1), ep); ep += 4; - ep[j] = 0; /* Zero unused bits at end of binary */ - data_dst = ep; - ep += j+1; - *ep++ = SMALL_INTEGER_EXT; - *ep++ = bitsize; + + if (off_heap_bytesize) { + /* trailing bits */ + ep[0] = 0; + copy_binary_to_buffer(ep, 0, bytes + j, 0, bitsize); + ep[1] = SMALL_INTEGER_EXT; + ep[2] = bitsize; + off_heap_tail = 3; + } + else { + ep[j] = 0; /* Zero unused bits at end of binary */ + data_dst = ep; + ep += j+1; + *ep++ = SMALL_INTEGER_EXT; + *ep++ = bitsize; + } } - if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) { + if (off_heap_bytesize) { + ASSERT(pb_val); + store_in_vec(ctx, ep, pb_val, pb_term, + bytes, off_heap_bytesize); + ep += off_heap_tail; + } + else if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) { WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs, ENC_BIN_COPY, 8*j + bitsize); } else { @@ -3130,14 +3593,15 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case EXPORT_DEF: { Export* exp = *((Export **) (export_val(obj) + 1)); - if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { + ASSERT(!(dflags & DFLAG_PENDING_CONNECT) || (ctx && ctx->iov)); + if (dflags & DFLAG_PENDING_CONNECT) + hopefull_export(ctx, &ep, exp, dflags, off_heap); + else if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); ep = enc_term(acmp, make_small(exp->info.mfa.arity), ep, dflags, off_heap); - if (ctx) - ctx->hopefull_flags |= DFLAG_EXPORT_PTR_TAG; } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -3187,11 +3651,276 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, if (ctx) { ASSERT(ctx->wstack.wstart == NULL); *reds = r; + if (use_iov) + store_in_vec(ctx, ep, NULL, THE_NON_VALUE, NULL, 0); } *res = ep; return 0; } +static ERTS_INLINE void +store_in_vec_aux(Uint *szp, + Sint *vlenp, + SysIOVec *iov, + ErlDrvBinary **binv, + Eterm *termv, + Binary *bin, + Eterm term, + byte *ptr, + Uint len, + Sint *fixp, + ErlIOVec **frag_eiovpp, + Uint frag_size) +{ + ErlDrvBinary *dbin = Binary2ErlDrvBinary(bin); + Uint size = 0; + int vlen = *vlenp; + Uint iov_len; + ErlIOVec *feiovp; + Sint fix = *fixp; + + ASSERT(((byte *) &bin->orig_bytes[0]) <= ptr); + ASSERT(ptr + len <= ((byte *) &bin->orig_bytes[0]) + bin->orig_size); + + if (fix >= 0) { + feiovp = frag_eiovpp[fix]; + ASSERT(0 < feiovp->size); + ASSERT(feiovp->size <= frag_size); + if (feiovp->size != frag_size) { + /* current fragment not full yet... */ + iov_len = frag_size - feiovp->size; + if (len < iov_len) + iov_len = len; + goto store_iov_data; + } + } + + while (len) { + /* Start new fragment... */ + + feiovp = frag_eiovpp[++fix]; + ASSERT(fix >= 0); + + if (termv) { + termv[vlen] = THE_NON_VALUE; + termv[vlen+1] = THE_NON_VALUE; + } + + feiovp->vsize = 2; + feiovp->size = 0; + feiovp->iov = &iov[vlen]; + feiovp->binv = &binv[vlen]; + + /* entry for driver header */ + iov[vlen].iov_base = NULL; + iov[vlen].iov_len = 0; + binv[vlen] = NULL; + vlen++; + + /* entry for dist header */ + iov[vlen].iov_base = NULL; + iov[vlen].iov_len = 0; + binv[vlen] = NULL; + vlen++; + + iov_len = len < frag_size ? len : frag_size; + + store_iov_data: + + ASSERT(iov_len); + + do { + Uint iov_len_left; + + if (iov_len <= MAX_SYSIOVEC_IOVLEN) + iov_len_left = 0; + else { + iov_len_left = iov_len - MAX_SYSIOVEC_IOVLEN; + iov_len = MAX_SYSIOVEC_IOVLEN; + } + + iov[vlen].iov_base = ptr; + iov[vlen].iov_len = iov_len; + binv[vlen] = dbin; + if (termv) + termv[vlen] = term; + else + erts_refc_inc(&bin->intern.refc, 2); + size += iov_len; + len -= iov_len; + ptr += iov_len; + vlen++; + feiovp->size += iov_len; + feiovp->vsize++; + + iov_len = iov_len_left; + } while (iov_len); + } + + *fixp = fix; + *vlenp = vlen; + *szp += size; +} + +static void +store_in_vec(TTBEncodeContext *ctx, + byte *ep, + Binary *ohbin, + Eterm ohpb, + byte *ohp, + Uint ohsz) +{ + byte *cp = ctx->cptr; + if (cp != ep) { + /* save data in common binary... */ + store_in_vec_aux(&ctx->size, + &ctx->vlen, + ctx->iov, + ctx->binv, + ctx->termv, + ctx->result_bin, + THE_NON_VALUE, + cp, ep - cp, + &ctx->frag_ix, + ctx->fragment_eiovs, + ctx->fragment_size); + ASSERT(ctx->vlen <= ctx->debug_vlen); + ASSERT(ctx->frag_ix <= ctx->debug_fragments); + ctx->cptr = ep; + } + if (ohbin) { + /* save off-heap binary... */ + store_in_vec_aux(&ctx->size, + &ctx->vlen, + ctx->iov, + ctx->binv, + ctx->termv, + ohbin, ohpb, ohp, ohsz, + &ctx->frag_ix, + ctx->fragment_eiovs, + ctx->fragment_size); + ASSERT(ctx->vlen <= ctx->debug_vlen); + ASSERT(ctx->frag_ix <= ctx->debug_fragments); + } +} + +static byte * +begin_hopefull_data(TTBEncodeContext *ctx, byte *ep) +{ + store_in_vec(ctx, ep, NULL, THE_NON_VALUE, NULL, 0); + ASSERT(ERTS_NO_HIX == (Uint32) get_int32(ctx->hopefull_ixp)); + put_int32(ctx->vlen, ctx->hopefull_ixp); + ctx->hopefull_ixp = ep; + put_int32(ERTS_NO_HIX, ep); + ep += 4; + ctx->cptr = ep; + return ep; +} + +static byte * +end_hopefull_data(TTBEncodeContext *ctx, byte *ep, Uint fallback_size) +{ + Uint sz; + store_in_vec(ctx, ep, NULL, THE_NON_VALUE, NULL, 0); + /* + * Reserve extra room for fallback if needed. The four + * bytes used for hopefull index can be used for + * fallback encoding... + */ + sz = ep - ctx->hopefull_ixp; + if (fallback_size > sz) { + ep += fallback_size - sz; + ctx->cptr = ep; + } + return ep; +} + +static byte * +hopefull_bit_binary(TTBEncodeContext* ctx, byte **epp, Binary *pb_val, Eterm pb_term, + byte *bytes, byte bitoffs, byte bitsize, Uint sz) +{ + byte *octets, *ep = *epp; + + ctx->hopefull_flags |= DFLAG_BIT_BINARIES; + + /* + * The fallback: + * + * SMALL_TUPLE_EXT - 1 byte + * 2 - 1 byte + * BINARY_EXT - 1 byte + * whole octet size ('sz') - 4 byte + * whole octets - 'sz' bytes + * trailing bits - 1 byte + * SMALL_INTEGER_EXT - 1 byte + * bitsize - 1 byte + */ + + /* bit binary prelude in one hopefull data element */ + ep = begin_hopefull_data(ctx, ep); + *ep++ = BIT_BINARY_EXT; + put_int32((sz+1), ep); + ep += 4; + *ep++ = bitsize; + ep = end_hopefull_data(ctx, ep, 1+1+1+4); + + /* All whole octets... */ + if (pb_val) { + octets = NULL; + store_in_vec(ctx, ep, pb_val, pb_term, bytes, sz); + } + else { + /* ... will be copied here afterwards */ + octets = ep; + ep += sz; + } + + /* copy trailing bits into new hopefull data element */ + ep = begin_hopefull_data(ctx, ep); + *ep = 0; + + copy_binary_to_buffer(ep, 0, bytes + sz, bitoffs, bitsize); + ep++; + + ep = end_hopefull_data(ctx, ep, 1+1+1); + *epp = ep; + + return octets; +} + +static void +hopefull_export(TTBEncodeContext* ctx, byte **epp, Export* exp, Uint32 dflags, + struct erl_off_heap_header** off_heap) +{ + Uint fallback_sz; + byte *ep = *epp, *mod_start; + + /* + * The fallback: + * + * SMALL_TUPLE_EXT - 1 byte + * 2 - 1 byte + * module atom... - M bytes + * function atom... - F bytes + */ + + ctx->hopefull_flags |= DFLAG_EXPORT_PTR_TAG; + + ep = begin_hopefull_data(ctx, ep); + + *ep++ = EXPORT_EXT; + mod_start = ep; + ep = enc_atom(NULL, exp->info.mfa.module, ep, dflags); + ep = enc_atom(NULL, exp->info.mfa.function, ep, dflags); + fallback_sz = 2 + (ep - mod_start); + ep = enc_term(NULL, make_small(exp->info.mfa.arity), + ep, dflags, off_heap); + + ep = end_hopefull_data(ctx, ep, fallback_sz); + + *epp = ep; +} + /** @brief Is it a list of bytes not longer than MAX_STRING_LEN? * @param lenp out: string length or number of list cells traversed * @return true/false @@ -3611,7 +4340,7 @@ dec_term_atom_common: if (tag == PORT_EXT) { cre = get_int8(ep); ep++; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } } @@ -3658,7 +4387,7 @@ dec_term_atom_common: cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } goto ref_ext_common; @@ -3672,7 +4401,7 @@ dec_term_atom_common: cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } r0 = get_int32(ep); @@ -4067,73 +4796,6 @@ dec_term_atom_common: next = &(funp->creator); break; } - case FUN_EXT: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm module; - Sint old_uniq; - Sint old_index; - unsigned num_free; - int i; - Eterm temp; - - num_free = get_int32(ep); - ep += 4; - hp += ERL_FUN_SIZE; - hp += num_free; - factory->hp = hp; - funp->thing_word = HEADER_FUN; - funp->num_free = num_free; - *objp = make_fun(funp); - - /* Creator pid */ - if ((*ep != PID_EXT && *ep != NEW_PID_EXT) - || (ep = dec_pid(edep, factory, ep+1, - &funp->creator, *ep))==NULL) { - goto error; - } - - /* Module */ - if ((ep = dec_atom(edep, ep, &module)) == NULL) { - goto error; - } - - /* Index */ - if ((ep = dec_term(edep, factory, ep, &temp, NULL, 0)) == NULL) { - goto error; - } - if (!is_small(temp)) { - goto error; - } - old_index = unsigned_val(temp); - - /* Uniq */ - if ((ep = dec_term(edep, factory, ep, &temp, NULL, 0)) == NULL) { - goto error; - } - if (!is_small(temp)) { - goto error; - } - - /* - * It is safe to link the fun into the fun list only when - * no more validity tests can fail. - */ - funp->next = factory->off_heap->first; - factory->off_heap->first = (struct erl_off_heap_header*)funp; - old_uniq = unsigned_val(temp); - - funp->fe = erts_put_fun_entry(module, old_uniq, old_index); - funp->arity = funp->fe->address[-1] - num_free; - hp = factory->hp; - - /* Environment */ - for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; - next = funp->env + i; - } - break; - } case ATOM_INTERNAL_REF2: n = get_int16(ep); ep += 2; @@ -4302,10 +4964,11 @@ error_hamt: (except for cached atoms) */ static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, - unsigned dflags) { - Uint size; + Uint64 dflags) { + Uint size = 0; ErtsExtSzRes res = encode_size_struct_int(NULL, acmp, obj, - dflags, NULL, &size); + dflags, NULL, + &size); /* * encode_size_struct2() only allowed when * we know the result will always be OK! @@ -4316,18 +4979,23 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, static ErtsExtSzRes encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, - unsigned dflags, Sint *reds, Uint *res) + Uint64 dflags, Sint *reds, Uint *res) { DECLARE_WSTACK(s); Uint m, i, arity; - Uint result = 0; + Uint result = *res; Sint r = 0; + int vlen = -1; if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->wstack.wstart) { /* restore saved stack */ + vlen = ctx->vlen; + + if (!ctx->wstack.wstart) + ctx->last_result = result; + else { /* restore saved stack */ WSTACK_RESTORE(s, &ctx->wstack); result = ctx->result; obj = ctx->obj; @@ -4346,6 +5014,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, *reds = 0; ctx->obj = obj; ctx->result = result; + ctx->vlen = vlen; WSTACK_SAVE(s, &ctx->wstack); return ERTS_EXT_SZ_YIELD; } @@ -4354,7 +5023,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result++; break; case ATOM_DEF: - if (dflags & DFLAG_INTERNAL_TAGS) { + if (dflags & DFLAG_ETS_COMPRESSED) { if (atom_val(obj) >= (1<<16)) { result += 1 + 3; } @@ -4408,30 +5077,21 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ break; case EXTERNAL_PID_DEF: - if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case PID_DEF: result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) + - 4 + 4 + 1); + 4 + 4 + 4); break; case EXTERNAL_REF_DEF: - if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case REF_DEF: ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); i = ref_no_numbers(obj); result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) + - 1 + 4*i); + 4 + 4*i); break; case EXTERNAL_PORT_DEF: - if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case PORT_DEF: result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) + - 4 + 1); + 4 + 4); break; case LIST_DEF: { int is_str = is_external_string(obj, &m); @@ -4525,39 +5185,131 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; case BINARY_DEF: { ProcBin* pb = (ProcBin*) binary_val(obj); - Uint tot_bytes = pb->size; - if (!(dflags & DFLAG_INTERNAL_TAGS)) { -#ifdef ARCH_64 - if (tot_bytes >= (Uint) 0xffffffff) { - if (pb->thing_word == HEADER_SUB_BIN) { - ErlSubBin* sub = (ErlSubBin*) pb; - tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8; - } - if (tot_bytes > (Uint) 0xffffffff) { - WSTACK_DESTROY(s); - return ERTS_EXT_SZ_SYSTEM_LIMIT; - } - } -#endif - } - else { + Uint bin_size = pb->size; + byte bitoffs = 0; + byte bitsize = 0; + if (dflags & DFLAG_ETS_COMPRESSED) { ProcBin* pb = (ProcBin*) binary_val(obj); Uint sub_extra = 0; if (pb->thing_word == HEADER_SUB_BIN) { ErlSubBin* sub = (ErlSubBin*) pb; + bitoffs = sub->bitoffs; + bitsize = sub->bitsize; pb = (ProcBin*) binary_val(sub->orig); sub_extra = 2; /* bitoffs and bitsize */ - tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8; + bin_size += (bitoffs + bitsize + 7) / 8; } if (pb->thing_word == HEADER_PROC_BIN - && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) { + && heap_bin_size(bin_size) > PROC_BIN_SIZE) { result += 1 + sub_extra + sizeof(ProcBin); break; } + } + else { +#ifdef ARCH_64 + if (bin_size >= (Uint) 0xffffffff) { + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*) pb; + bin_size += (sub->bitoffs + sub->bitsize+ 7) / 8; + } + if (bin_size > (Uint) 0xffffffff) { + WSTACK_DESTROY(s); + return ERTS_EXT_SZ_SYSTEM_LIMIT; + } + } +#endif + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*) pb; + bitoffs = sub->bitoffs; + bitsize = sub->bitsize; + pb = (ProcBin*) binary_val(sub->orig); + } + if (vlen >= 0) { + Uint csz; + if (pb->thing_word == HEADER_PROC_BIN + && bitoffs == 0 + && bin_size > ERL_ONHEAP_BIN_LIMIT) { + Uint trailing_result; + if (bitsize == 0) { + result += (1 /* BIT_BINARY_EXT */ + + 4 /* size */); + trailing_result = 0; + } + else if (dflags & DFLAG_BIT_BINARIES) { + result += (1 /* BIT_BINARY_EXT */ + + 4 /* size */ + + 1 /* trailing bitsize */); + trailing_result = 1 /* trailing bits */; + } + else { + /* sigh... */ + result += (1 /* SMALL_TUPLE_EXT */ + + 1 /* 2 tuple size */ + + 1 /* BINARY_EXT */ + + 4 /* binary size */); + trailing_result = (1 /* SMALL_INTEGER_EXT */ + + 1 /* bitsize */); + } + csz = result - ctx->last_result; + ctx->last_result = result; + result += trailing_result; + vlen += 2; /* data leading up to binary and binary */ + + /* potentially multiple elements leading up to binary */ + vlen += csz/MAX_SYSIOVEC_IOVLEN; + /* potentially multiple elements for binary */ + vlen += bin_size/MAX_SYSIOVEC_IOVLEN; + ctx->extra_size += bin_size; + + if (dflags & DFLAG_PENDING_CONNECT) { + ASSERT(dflags & DFLAG_BIT_BINARIES); + vlen += 2; /* for hopefull prolog and epilog */ + result += (4 /* for hopefull prolog (see below) */ + + 4); /* for hopefull epilog (see below) */ + ctx->last_result = result; + } + break; + } + } } - result += 1 + 4 + binary_size(obj) + - 5; /* For unaligned binary */ + + if (bitsize == 0) { + result += (1 /* BIT_BINARY_EXT */ + + 4 /* size */ + + bin_size); + } + else if (dflags & DFLAG_PENDING_CONNECT) { + Uint csz = result - ctx->last_result; + ASSERT(dflags & DFLAG_BIT_BINARIES); + /* potentially multiple elements leading up to binary */ + vlen += csz/MAX_SYSIOVEC_IOVLEN; + vlen++; /* hopefull prolog */ + /* + * Size for hopefull prolog is max of + * - fallback: 1 + 1 + 1 + 4 + * - hopfull index + bit binary prolog: 4 + 1 + 4 + 1 + */ + result += 4 + 1 + 4 + 1; + /* potentially multiple elements for binary */ + vlen += bin_size/MAX_SYSIOVEC_IOVLEN + 1; + result += bin_size; + vlen++; /* hopefull epiolog */ + /* + * Size for hopefull epiolog is max of + * - fallback: 1 + 1 + 1 + * - hopfull index + bit binary epilog: 4 + 1 + */ + result += 4 + 1; + ctx->last_result = result; + } + else if (dflags & DFLAG_BIT_BINARIES) { + result += 1 + 4 + 1 + bin_size + 1; + } + else { + /* Sigh... */ + result += 1 + 1 + 1 + 4 + bin_size + 1 + 1 + 1; + } break; } case FUN_DEF: @@ -4584,10 +5336,25 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case EXPORT_DEF: { Export* ep = *((Export **) (export_val(obj) + 1)); + Uint tmp_result = result; result += 1; result += encode_size_struct2(acmp, ep->info.mfa.module, dflags); result += encode_size_struct2(acmp, ep->info.mfa.function, dflags); result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags); + if (dflags & DFLAG_PENDING_CONNECT) { + Uint csz; + /* + * Fallback is 1 + 1 + Module size + Function size, that is, + * the hopefull index + hopefull encoding is larger... + */ + ASSERT(dflags & DFLAG_EXPORT_PTR_TAG); + csz = tmp_result - ctx->last_result; + /* potentially multiple elements leading up to hopefull entry */ + vlen += csz/MAX_SYSIOVEC_IOVLEN; + vlen++; /* hopefull entry */ + result += 4; /* hopefull index */ + ctx->last_result = result; + } } break; @@ -4630,6 +5397,14 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, if (ctx) { ASSERT(ctx->wstack.wstart == NULL); *reds = r < 0 ? 0 : r; + + if (vlen >= 0) { + Uint csz; + csz = result - ctx->last_result; + if (csz) + vlen += csz/MAX_SYSIOVEC_IOVLEN + 1; + ctx->vlen = vlen; + } } *res = result; return ERTS_EXT_SZ_OK; @@ -4898,9 +5673,6 @@ init_done: total_size = get_int32(ep); CHKSIZE(total_size); ep += 1+16+4+4; - /*FALLTHROUGH*/ - - case FUN_EXT: CHKSIZE(4); num_free = get_int32(ep); ep += 4; @@ -4911,6 +5683,12 @@ init_done: heap_size += ERL_FUN_SIZE + num_free; break; } + case FUN_EXT: + /* + * OTP 23: No longer support decoding the old fun + * representation. + */ + goto error; case ATOM_INTERNAL_REF2: SKIP(2+atom_extra_skip); atom_extra_skip = 0; @@ -4968,181 +5746,374 @@ error: #undef CHKSIZE } +#define ERTS_TRANSCODE_REDS_FACT 4 -struct transcode_context { - enum { - TRANSCODE_DEC_MSG_SIZE, - TRANSCODE_DEC_MSG, - TRANSCODE_ENC_CTL, - TRANSCODE_ENC_MSG - }state; - Eterm ctl_term; - Eterm* ctl_heap; - ErtsHeapFactory ctl_factory; - Eterm* msg_heap; - B2TContext b2t; - TTBEncodeContext ttb; -#ifdef DEBUG - ErtsDistOutputBuf* dbg_ob; -#endif -}; - -void transcode_free_ctx(DistEntry* dep) +Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, + DistEntry* dep, + Uint64 dflags, + Sint reds) { - struct transcode_context* ctx = dep->transcode_ctx; + ErlIOVec* eiov = ob->eiov; + SysIOVec* iov = eiov->iov; + byte *hdr; + Uint64 hopefull_flags; + Uint32 hopefull_ix, payload_ix; + Sint start_r, r; + Uint new_len; + byte *ep; - erts_factory_close(&ctx->ctl_factory); - erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->ctl_heap); + if (reds < 0) + return reds; - if (ctx->msg_heap) { - erts_factory_close(&ctx->b2t.u.dc.factory); - erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->msg_heap); + /* + * HOPEFUL_DATA header always present in io vector + * element 1: + * + * +---+--------------+-----------+----------+ + * |'H'|Hopefull Flags|Hopefull IX|Payload IX| + * +---+--------------+-----------+----------+ + * 1 8 4 4 + * + * Hopefull flags: Flags corresponding to actual + * hopefull encodings in this + * buffer. + * Hopefull IX: Vector index of first hopefull + * encoding. Each hopefull encoding + * is preceeded by 4 bytes containing + * next vector index of hopefull + * encoding. ERTS_NO_HIX marks the + * end. + * Payload IX: Vector index of the beginning + * of the payload if there is + * one; otherwise, zero. + */ + hdr = (byte *) iov[1].iov_base; + + ASSERT(HOPEFUL_DATA == *((byte *)iov[1].iov_base)); + ASSERT(iov[1].iov_len == 1+8+4+4); + + /* Control message always begin in vector element 2 */ + ep = iov[2].iov_base; + ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT); + + if (~dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME) + && ep[0] == SMALL_TUPLE_EXT + && ep[1] == 4 + && ep[2] == SMALL_INTEGER_EXT + && (ep[3] == DOP_MONITOR_P || + ep[3] == DOP_MONITOR_P_EXIT || + ep[3] == DOP_DEMONITOR_P)) { + /* + * Receiver does not support process monitoring. + * Suppress monitor control msg (see erts_dsig_send_monitor) + * by converting it to an empty (tick) packet. + */ + ob->eiov->vsize = 1; + ob->eiov->size = 0; + return reds; } - erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx); - dep->transcode_ctx = NULL; -} -Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, - DistEntry* dep, - Uint32 dflags, - Sint reds) -{ - Sint hsz; - byte* decp; - const int have_msg = !!ob->msg_start; - int i; - struct transcode_context* ctx = dep->transcode_ctx; + hdr++; + hopefull_flags = get_int64(hdr); - if (!ctx) { /* first call for 'ob' */ - ASSERT(!(ob->hopefull_flags & ~(Uint)(DFLAG_BIT_BINARIES | - DFLAG_EXPORT_PTR_TAG))); - if (~dflags & ob->hopefull_flags) { - /* - * Receiver does not support bitstrings and/or export funs - * and output buffer contains such message tags (hopefull_flags). - * Must transcode control and message terms to use tuple fallbacks. - */ - ctx = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, sizeof(struct transcode_context)); - dep->transcode_ctx = ctx; - #ifdef DEBUG - ctx->dbg_ob = ob; - #endif - - hsz = decoded_size(ob->extp, ob->ext_endp, 0, NULL); - ctx->ctl_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm)); - erts_factory_tmp_init(&ctx->ctl_factory, ctx->ctl_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE); - ctx->msg_heap = NULL; - - decp = dec_term(NULL, &ctx->ctl_factory, ob->extp, &ctx->ctl_term, NULL, 0); - if (have_msg) { - ASSERT(decp == ob->msg_start); (void)decp; - ctx->b2t.u.sc.ep = NULL; - ctx->b2t.state = B2TSize; - ctx->b2t.aligned_alloc = NULL; - ctx->b2t.b2ts.exttmp = 0; - ctx->state = TRANSCODE_DEC_MSG_SIZE; - } - else { - ASSERT(decp == ob->ext_endp); - ctx->state = TRANSCODE_ENC_CTL; - } + hdr += 8; + hopefull_ix = get_int32(hdr); + + if ((~dflags & DFLAG_SPAWN) + && ep[0] == SMALL_TUPLE_EXT + && ((ep[1] == 6 + && ep[2] == SMALL_INTEGER_EXT + && ep[3] == DOP_SPAWN_REQUEST) + || (ep[1] == 8 + && ep[2] == SMALL_INTEGER_EXT + && ep[3] == DOP_SPAWN_REQUEST_TT))) { + /* + * Receiver does not support distributed spawn. Convert + * this packet to an empty (tick) packet, and inform + * spawning process that this is not supported... + */ + ErtsHeapFactory factory; + Eterm ctl_msg, ref, pid, token, *tp, *hp; + Uint buf_sz; + byte *buf_start, *buf_end; + byte *ptr; + Uint hsz; + + hdr += 4; + payload_ix = get_int32(hdr); + ASSERT(payload_ix >= 3); + + if (payload_ix == 3) { + /* The whole control message is in iov[2].iov_base */ + buf_sz = (Uint) iov[2].iov_len; + buf_start = (byte *) iov[2].iov_base; + buf_end = buf_start + buf_sz; } - else if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) { - /* - * No need for full transcoding, but primitive receiver (erl_/jinterface) - * expects VERSION_MAGIC before both control and message terms. - */ - if (ob->msg_start) { - Sint ctl_bytes = ob->msg_start - ob->extp; - ASSERT(ob->extp < ob->msg_start && ob->msg_start < ob->ext_endp); - /* Move control term back 1 byte to make room */ - sys_memmove(ob->extp-1, ob->extp, ctl_bytes); - *--(ob->msg_start) = VERSION_MAGIC; - --(ob->extp); - reds -= ctl_bytes / (B2T_BYTES_PER_REDUCTION * B2T_MEMCPY_FACTOR); + else { + /* Control message over multiple buffers... */ + int ix; + buf_sz = 0; + for (ix = 2; ix < payload_ix; ix++) + buf_sz += iov[ix].iov_len; + ptr = buf_start = erts_alloc(ERTS_ALC_T_TMP, buf_sz); + buf_end = buf_start + buf_sz; + for (ix = 2; ix < payload_ix; ix++) { + sys_memcpy((void *) ptr, + (void *) iov[ix].iov_base, + iov[ix].iov_len); + ptr += iov[ix].iov_len; } - *--(ob->extp) = VERSION_MAGIC; - goto done; } - else - goto done; - } - else { /* continue after yield */ - ASSERT(ctx->dbg_ob == ob); - } - ctx->b2t.reds = reds * B2T_BYTES_PER_REDUCTION; - switch (ctx->state) { - case TRANSCODE_DEC_MSG_SIZE: - hsz = decoded_size(ob->msg_start, ob->ext_endp, 0, &ctx->b2t); - if (ctx->b2t.state == B2TSize) { - return -1; + hsz = decoded_size(buf_start, buf_end, 0, NULL); + hp = erts_alloc(ERTS_ALC_T_TMP, hsz*sizeof(Eterm)); + erts_factory_tmp_init(&factory, hp, hsz, ERTS_ALC_T_TMP); + + ptr = dec_term(NULL, &factory, buf_start, &ctl_msg, NULL, 0); + ASSERT(ptr); (void)ptr; + + ASSERT(is_tuple_arity(ctl_msg, 6) + || is_tuple_arity(ctl_msg, 8)); + tp = tuple_val(ctl_msg); + ASSERT(tp[1] == make_small(DOP_SPAWN_REQUEST) + || tp[1] == make_small(DOP_SPAWN_REQUEST_TT)); + + ref = tp[2]; + pid = tp[3]; + if (tp[1] == make_small(DOP_SPAWN_REQUEST)) + token = NIL; + else { + token = tp[8]; + erts_seq_trace_update_node_token(token); } - ASSERT(ctx->b2t.state == B2TDecodeInit); - ctx->msg_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm)); - ctx->b2t.u.dc.ep = ob->msg_start; - ctx->b2t.u.dc.res = (Eterm) NULL; - ctx->b2t.u.dc.next = &ctx->b2t.u.dc.res; - erts_factory_tmp_init(&ctx->b2t.u.dc.factory, - ctx->msg_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE); - ctx->b2t.u.dc.flat_maps.wstart = NULL; - ctx->b2t.u.dc.hamt_array.pstart = NULL; - ctx->b2t.state = B2TDecode; - - ctx->state = TRANSCODE_DEC_MSG; - case TRANSCODE_DEC_MSG: - if (ctx->b2t.reds <= 0) - ctx->b2t.reds = 1; - decp = dec_term(NULL, NULL, NULL, NULL, &ctx->b2t, 0); - if (ctx->b2t.state < B2TDone) { - return -1; - } - ASSERT(ctx->b2t.state == B2TDone); - ASSERT(decp && decp <= ob->ext_endp); - reds = ctx->b2t.reds / B2T_BYTES_PER_REDUCTION; - b2t_destroy_context(&ctx->b2t); - - ctx->state = TRANSCODE_ENC_CTL; - case TRANSCODE_ENC_CTL: - if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) { - ASSERT(!(dflags & DFLAG_NO_MAGIC)); - ob->extp -= 2; /* VERSION_MAGIC x 2 */ - } - ob->ext_endp = ob->extp; - i = erts_encode_dist_ext(ctx->ctl_term, &ob->ext_endp, dflags, - NULL, NULL, NULL); - ASSERT(i == 0); (void)i; - ASSERT(ob->ext_endp <= ob->alloc_endp); + ASSERT(is_internal_ordinary_ref(tp[2])); + ASSERT(is_internal_pid(tp[3])); + + (void) erts_proc_sig_send_dist_spawn_reply(dep->sysname, + ref, pid, + NULL, am_notsup, + token); + + erts_factory_close(&factory); + erts_free(ERTS_ALC_T_TMP, hp); + if (buf_start != (byte *) iov[2].iov_base) + erts_free(ERTS_ALC_T_TMP, buf_start); + + ob->eiov->vsize = 1; + ob->eiov->size = 0; + + reds -= 4; + + if (reds < 0) + return 0; + return reds; + } + + start_r = r = reds*ERTS_TRANSCODE_REDS_FACT; - if (!have_msg) { - break; + if (~dflags & hopefull_flags) { + + while (hopefull_ix != ERTS_NO_HIX) { + Uint32 new_hopefull_ix; + + if (r <= 0) { /* yield... */ + /* save current hopefull_ix... */ + ep = (byte *) iov[1].iov_base; + ep += 5; + put_int32(hopefull_ix, ep); + return -1; + } + + /* Read next hopefull index */ + ep = (byte *) iov[hopefull_ix].iov_base; + ep -= 4; + new_hopefull_ix = get_int32(ep); + ASSERT(new_hopefull_ix == ERTS_NO_HIX + || (hopefull_ix < new_hopefull_ix + && new_hopefull_ix < eiov->vsize)); + + ep = (byte *) iov[hopefull_ix].iov_base; + switch (*ep) { + + case EXPORT_EXT: { + byte *start_ep, *end_ep; + Eterm module, function; + if (!(hopefull_flags & DFLAG_EXPORT_PTR_TAG)) + break; + /* Read original encoding... */ + ep++; + start_ep = ep; + ep = dec_atom(NULL, ep, &module); + ASSERT(ep && is_atom(module)); + ep = dec_atom(NULL, ep, &function); + ASSERT(ep && is_atom(function)); + end_ep = ep; + ASSERT(*ep == SMALL_INTEGER_EXT + || *ep == INTEGER_EXT + || *ep == SMALL_BIG_EXT + || *ep == LARGE_BIG_EXT); + + /* + * module and function atoms are encoded + * between start_ep and end_ep. Prepend a + * 2-tuple tag before the atoms and + * remove arity at end. + */ + + /* write fallback */ + + ep = start_ep; + ep--; + put_int8(2, ep); + ep--; + *ep = SMALL_TUPLE_EXT; + + iov[hopefull_ix].iov_base = ep; + + /* Update iov sizes... */ + new_len = end_ep - ep; + eiov->size -= iov[hopefull_ix].iov_len; + eiov->size += new_len; + iov[hopefull_ix].iov_len = new_len; + r--; + break; + } + + case BIT_BINARY_EXT: { + Uint bin_sz; + byte bitsize, epilog_byte; + ASSERT(hopefull_ix != ERTS_NO_HIX); + if (!(hopefull_flags & DFLAG_BIT_BINARIES)) { + /* skip to epilog... */ + hopefull_ix = new_hopefull_ix; + ep = (byte *) iov[hopefull_ix].iov_base; + ep -= 4; + new_hopefull_ix = get_int32(ep); + ASSERT(new_hopefull_ix == ERTS_NO_HIX + || (hopefull_ix < new_hopefull_ix + && new_hopefull_ix < eiov->vsize)); + break; + } + + /* read original encoded prolog... */ + ep++; + bin_sz = get_int32(ep); + ep += 4; + bitsize = *ep++; + + /* write fallback prolog... */ + iov[hopefull_ix].iov_base -= 4; + ep = (byte *) iov[hopefull_ix].iov_base; + + *ep++ = SMALL_TUPLE_EXT; + *ep++ = 2; + *ep++ = BINARY_EXT; + put_int32(bin_sz, ep); + ep += 4; + + /* Update iov sizes... */ + new_len = ep - (byte *) iov[hopefull_ix].iov_base; + eiov->size -= iov[hopefull_ix].iov_len; + eiov->size += new_len; + iov[hopefull_ix].iov_len = new_len; + r--; +#ifdef DEBUG + /* + * The binary data between the prolog and the + * epilog should be of size 'bin_sz - 1' and + * exists in the iov elements between prolog + * and epilog... + */ + { + Uint ix, debug_bin_sz = 0; + for (ix = hopefull_ix+1; ix < new_hopefull_ix; ix++) + debug_bin_sz += iov[ix].iov_len; + ASSERT(debug_bin_sz == bin_sz - 1); + } +#endif + /* jump to epilog... */ + hopefull_ix = new_hopefull_ix; + ep = (byte *) iov[hopefull_ix].iov_base; + + /* read original encoded epilog... */ + epilog_byte = *ep; + + ASSERT(1 == iov[hopefull_ix].iov_len); + + iov[hopefull_ix].iov_base -= 4; + ep = (byte *) iov[hopefull_ix].iov_base; + new_hopefull_ix = get_int32(ep); + ASSERT(new_hopefull_ix == ERTS_NO_HIX + || (hopefull_ix < new_hopefull_ix + && new_hopefull_ix < eiov->vsize)); + + /* write fallback epilog... */ + + *ep++ = epilog_byte; + *ep++ = SMALL_INTEGER_EXT; + *ep++ = bitsize; + + /* Update iov sizes... */ + new_len = ep - (byte *) iov[hopefull_ix].iov_base; + eiov->size -= iov[hopefull_ix].iov_len; + eiov->size += new_len; + iov[hopefull_ix].iov_len = new_len; + r--; + break; + } + + default: + ERTS_INTERNAL_ERROR("Unexpected external tag"); + break; + } + + hopefull_ix = new_hopefull_ix; + r--; } - ob->msg_start = ob->ext_endp; - ctx->ttb.wstack.wstart = NULL; - ctx->ttb.flags = dflags; - ctx->ttb.hopefull_flags = 0; - ctx->ttb.level = 0; - - ctx->state = TRANSCODE_ENC_MSG; - case TRANSCODE_ENC_MSG: - reds *= TERM_TO_BINARY_LOOP_FACTOR; - if (erts_encode_dist_ext(ctx->b2t.u.dc.res, &ob->ext_endp, dflags, NULL, - &ctx->ttb, &reds)) { - return -1; + } + + /* + * Replace hopefull data header with actual header... + */ + ep = (byte *) iov[1].iov_base; + eiov->size -= iov[1].iov_len; + + if (dflags & (DFLAG_DIST_HDR_ATOM_CACHE|DFLAG_FRAGMENTS)) { + /* + * Encoding was done without atom caching but receiver expects + * a dist header, so we prepend an empty one. + */ + *ep++ = VERSION_MAGIC; + *ep++ = DIST_HEADER; + *ep++ = 0; /* NumberOfAtomCacheRefs */ + } + else { + hdr += 4; + payload_ix = get_int32(hdr); + + if (payload_ix) { + ASSERT(0 < payload_ix && payload_ix < eiov->vsize); + /* Prepend version magic on payload. */ + iov[payload_ix].iov_base--; + *((byte *) iov[payload_ix].iov_base) = VERSION_MAGIC; + iov[payload_ix].iov_len++; + eiov->size++; + r--; } - reds /= TERM_TO_BINARY_LOOP_FACTOR; - ASSERT(ob->ext_endp <= ob->alloc_endp); - ASSERT(!ctx->ttb.hopefull_flags); + *ep++ = PASS_THROUGH; + *ep++ = VERSION_MAGIC; } - transcode_free_ctx(dep); -done: - if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--(ob->extp) = PASS_THROUGH; + iov[1].iov_len = ep - (byte *) iov[1].iov_base; + eiov->size += iov[1].iov_len; - if (reds < 0) - reds = 0; + r--; + + /* done... */ + reds -= (start_r - r)/ERTS_TRANSCODE_REDS_FACT + 1; + if (reds < 0) + return 0; return reds; } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index e362a6c81f..bc006f83e2 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -60,6 +60,7 @@ #define DIST_HEADER 'D' #define DIST_FRAG_HEADER 'E' #define DIST_FRAG_CONT 'F' +#define HOPEFUL_DATA 'H' #define ATOM_CACHE_REF 'R' #define ATOM_INTERNAL_REF2 'I' #define ATOM_INTERNAL_REF3 'K' @@ -90,6 +91,8 @@ #undef ERL_NODE_TABLES_BASIC_ONLY #include "erl_alloc.h" +#define ERTS_NO_HIX (~((Uint32) 0)) + #define ERTS_ATOM_CACHE_SIZE 2048 typedef struct cache { @@ -153,16 +156,19 @@ typedef struct { /* -------------------------------------------------------------------------- */ +struct TTBSizeContext_; +struct TTBEncodeContext_; void erts_init_atom_cache_map(ErtsAtomCacheMap *); void erts_reset_atom_cache_map(ErtsAtomCacheMap *); void erts_destroy_atom_cache_map(ErtsAtomCacheMap *); -void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); +void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint64); -Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *, Uint); -byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *, Uint, Eterm); +Uint erts_encode_ext_dist_header_size(struct TTBEncodeContext_ *ctx, ErtsAtomCacheMap *, Uint); +byte *erts_encode_ext_dist_header_setup(struct TTBEncodeContext_ *ctx, byte *, + ErtsAtomCacheMap *, Uint, Eterm); byte *erts_encode_ext_dist_header_fragment(byte **, Uint, Eterm); -Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint32 dflags, Sint reds); +Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint64 dflags, Sint reds); struct erts_dsig_send_context; typedef enum { @@ -171,12 +177,13 @@ typedef enum { ERTS_EXT_SZ_SYSTEM_LIMIT } ErtsExtSzRes; -ErtsExtSzRes erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp); -ErtsExtSzRes erts_encode_dist_ext_size_ctx(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp); -struct TTBEncodeContext_; -int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *, - struct TTBEncodeContext_ *, Sint* reds); - +ErtsExtSzRes erts_encode_dist_ext_size(Eterm term, ErtsAtomCacheMap *acmp, + struct TTBSizeContext_ *ctx, + Uint* szp, Sint *redsp, + Sint *vlenp, Uint *fragments); +int erts_encode_dist_ext(Eterm, byte **, Uint64, ErtsAtomCacheMap *, + struct TTBEncodeContext_ *, Uint *, + Sint *); ErtsExtSzRes erts_encode_ext_size(Eterm, Uint *szp); ErtsExtSzRes erts_encode_ext_size_2(Eterm, unsigned, Uint *szp); Uint erts_encode_ext_size_ets(Eterm); @@ -207,7 +214,8 @@ Sint erts_decode_ext_size_ets(byte*, Uint); Eterm erts_decode_ext(ErtsHeapFactory*, byte**, Uint32 flags); Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); -Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); +Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint64 flags); +Eterm erts_debug_term_to_binary(Process *p, Eterm term, Eterm opts); Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint); void erts_binary2term_abort(ErtsBinary2TermState *); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 8a04a4c7f6..b9af01ee57 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -74,9 +74,8 @@ struct enif_resource_type_t struct enif_resource_type_t* next; /* list of all resource types */ struct enif_resource_type_t* prev; struct erl_module_nif* owner; /* that created this type and thus implements the destructor*/ - ErlNifResourceDtor* dtor; /* user destructor function */ - ErlNifResourceStop* stop; - ErlNifResourceDown* down; + ErlNifResourceTypeInit fn; + ErlNifResourceTypeInit fn_real; erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) +1 for active erl_module_nif */ Eterm module; @@ -110,6 +109,7 @@ typedef struct ErtsResource_ extern Eterm erts_bld_resource_ref(Eterm** hp, ErlOffHeap*, ErtsResource*); +extern BeamInstr* erts_call_nif_early(Process* c_p, ErtsCodeInfo* ci); extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); @@ -122,6 +122,10 @@ void erts_nif_demonitored(ErtsResource* resource); extern void erts_add_taint(Eterm mod_atom); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); + +/* Loads the specified NIF. The caller must have code write permission. */ +Eterm erts_load_nif(Process *c_p, BeamInstr *I, Eterm filename, Eterm args); + void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); extern int erts_nif_get_funcs(struct erl_module_nif*, @@ -885,6 +889,8 @@ void erts_bif_info_init(void); /* bif.c */ +void erts_write_bif_wrapper(Export *export, BeamInstr *address); + void erts_queue_monitor_message(Process *, ErtsProcLocks*, Eterm, @@ -926,6 +932,8 @@ void erts_queue_release_literals(Process *c_p, ErtsLiteralArea* literals); #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) +#define ERTS_LITERAL_AREA_SIZE(AP) \ + (ERTS_LITERAL_AREA_ALLOC_SIZE((AP)->end - (AP)->start)) extern erts_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ @@ -1135,6 +1143,12 @@ extern int is_node_name_atom(Eterm a); extern int erts_net_message(Port *, DistEntry *, Uint32 conn_id, byte *, ErlDrvSizeT, Binary *, byte *, ErlDrvSizeT); +int erts_dist_pend_spawn_exit_delete(ErtsMonitor *mon); +int erts_dist_pend_spawn_exit_parent_setup(ErtsMonitor *mon); +int erts_dist_pend_spawn_exit_parent_wait(Process *c_p, + ErtsProcLocks locks, + ErtsMonitor *mon); + extern void init_dist(void); extern int stop_dist(void); @@ -1152,6 +1166,7 @@ void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); +BeamInstr *erts_printable_return_address(Process* p, Eterm *E) ERTS_NOINLINE; /* erl_init.c */ @@ -1291,7 +1306,6 @@ Sint erts_binary_set_loop_limit(Sint limit); /* erl_bif_persistent.c */ void erts_init_bif_persistent_term(void); -Uint erts_persistent_term_count(void); void erts_init_persistent_dumping(void); extern ErtsLiteralArea** erts_persistent_areas; extern Uint erts_num_persistent_areas; diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index 8954dbb06c..177b7cc3d1 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -30,37 +30,19 @@ #include "hash.h" /* -** List of sizes (all are primes) -*/ -static const int h_size_table[] = { - 2, 5, 11, 23, 47, 97, 197, 397, 797, /* double upto here */ - 1201, 1597, - 2411, 3203, - 4813, 6421, - 9643, 12853, - 19289, 25717, - 51437, - 102877, - 205759, - 411527, - 823117, - 1646237, - 3292489, - 6584983, - 13169977, - 26339969, - 52679969, - -1 -}; - -/* ** Get info about hash ** */ +#define MAX_SHIFT (ERTS_SIZEOF_TERM * 8) + +static int hash_get_slots(Hash *h) { + return UWORD_CONSTANT(1) << (MAX_SHIFT - h->shift); +} + void hash_get_info(HashInfo *hi, Hash *h) { - int size = h->size; + int size = hash_get_slots(h); int i; int max_depth = 0; int objects = 0; @@ -84,7 +66,7 @@ void hash_get_info(HashInfo *hi, Hash *h) ASSERT(objects == h->nobjs); hi->name = h->name; - hi->size = h->size; + hi->size = hash_get_slots(h); hi->used = used; hi->objs = h->nobjs; hi->depth = max_depth; @@ -118,15 +100,15 @@ hash_table_sz(Hash *h) int i; for(i=0;h->name[i];i++); i++; - return sizeof(Hash) + h->size*sizeof(HashBucket*) + i; + return sizeof(Hash) + hash_get_slots(h)*sizeof(HashBucket*) + i; } static ERTS_INLINE void set_thresholds(Hash* h) { - h->grow_threshold = (8*h->size)/5; /* grow at 160% load */ - if (h->size_ix > h->min_size_ix) - h->shrink_threshold = h->size / 5; /* shrink at 20% load */ + h->grow_threshold = (8*hash_get_slots(h))/5; /* grow at 160% load */ + if (h->shift < h->max_shift) + h->shrink_threshold = hash_get_slots(h) / 5; /* shrink at 20% load */ else h->shrink_threshold = -1; /* never shrink below inital size */ } @@ -138,29 +120,27 @@ static ERTS_INLINE void set_thresholds(Hash* h) Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun) { int sz; - int ix = 0; + int shift = 1; h->meta_alloc_type = type; - while (h_size_table[ix] != -1 && h_size_table[ix] < size) - ix++; - if (h_size_table[ix] == -1) - return NULL; - - size = h_size_table[ix]; - sz = size*sizeof(HashBucket*); + while ((UWORD_CONSTANT(1) << shift) < size) + shift++; - h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); - - memzero(h->bucket, sz); h->is_allocated = 0; h->name = name; h->fun = fun; - h->size = size; - h->size_ix = ix; - h->min_size_ix = ix; + h->shift = MAX_SHIFT - shift; + h->max_shift = h->shift; h->nobjs = 0; set_thresholds(h); + + sz = hash_get_slots(h) * sizeof(HashBucket*); + h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); + memzero(h->bucket, sz); + + ASSERT(h->shift > 0 && h->shift < 64); + return h; } @@ -183,7 +163,7 @@ Hash* hash_new(int type, char* name, int size, HashFunctions fun) */ void hash_delete(Hash* h) { - int old_size = h->size; + int old_size = hash_get_slots(h); int i; for (i = 0; i < old_size; i++) { @@ -206,22 +186,20 @@ void hash_delete(Hash* h) static void rehash(Hash* h, int grow) { int sz; - int old_size = h->size; + int old_size = hash_get_slots(h); HashBucket** new_bucket; int i; if (grow) { - if ((h_size_table[h->size_ix+1]) == -1) - return; - h->size_ix++; + h->shift--; } else { - if (h->size_ix == 0) + if (h->shift == h->max_shift) return; - h->size_ix--; + h->shift++; } - h->size = h_size_table[h->size_ix]; - sz = h->size*sizeof(HashBucket*); + + sz = hash_get_slots(h)*sizeof(HashBucket*); new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz); memzero(new_bucket, sz); @@ -230,7 +208,7 @@ static void rehash(Hash* h, int grow) HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; - int ix = b->hvalue % h->size; + Uint ix = hash_get_slot(h, b->hvalue); b->next = new_bucket[ix]; new_bucket[ix] = b; b = b_next; @@ -247,16 +225,7 @@ static void rehash(Hash* h, int grow) */ void* hash_get(Hash* h, void* tmpl) { - HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; - HashBucket* b = h->bucket[ix]; - - while(b != (HashBucket*) 0) { - if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) - return (void*) b; - b = b->next; - } - return (void*) 0; + return hash_fetch(h, tmpl, h->fun.hash, h->fun.cmp); } /* @@ -265,7 +234,7 @@ void* hash_get(Hash* h, void* tmpl) void* hash_put(Hash* h, void* tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket* b = h->bucket[ix]; while(b != (HashBucket*) 0) { @@ -291,7 +260,7 @@ void* hash_put(Hash* h, void* tmpl) void* hash_erase(Hash* h, void* tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket* b = h->bucket[ix]; HashBucket* prev = 0; @@ -323,7 +292,7 @@ void * hash_remove(Hash *h, void *tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket *b = h->bucket[ix]; HashBucket *prev = NULL; @@ -343,11 +312,11 @@ hash_remove(Hash *h, void *tmpl) return NULL; } -void hash_foreach(Hash* h, void (*func)(void *, void *), void *func_arg2) +void hash_foreach(Hash* h, HFOREACH_FUN func, void *func_arg2) { int i; - for (i = 0; i < h->size; i++) { + for (i = 0; i < hash_get_slots(h); i++) { HashBucket* b = h->bucket[i]; while(b != (HashBucket*) 0) { (*func)((void *) b, func_arg2); diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index d319aaca83..4e8eb6594b 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -18,16 +18,16 @@ * %CopyrightEnd% */ -/* -** General hash functions -** -*/ +/** + * General hash functions + * + **/ #ifndef __HASH_H__ #define __HASH_H__ #include "sys.h" -typedef unsigned long HashValue; +typedef UWord HashValue; typedef struct hash Hash; typedef int (*HCMP_FUN)(void*, void*); @@ -38,6 +38,7 @@ typedef void (*HFREE_FUN)(void*); typedef void* (*HMALLOC_FUN)(int,size_t); typedef void (*HMFREE_FUN)(int,void*); typedef int (*HMPRINT_FUN)(fmtfn_t,void*,char*, ...); +typedef void (*HFOREACH_FUN)(void *, void *); /* ** This bucket must be placed in top of @@ -75,11 +76,10 @@ struct hash int is_allocated; /* 0 iff hash structure is on stack or is static */ int meta_alloc_type; /* argument to pass to meta_alloc and meta_free */ char* name; /* Table name (static string, for debugging) */ - int size; /* Number of slots */ + int shift; /* How much to shift the hash value */ + int max_shift; /* Never shift more than this value */ int shrink_threshold; int grow_threshold; - int size_ix; /* Size index in size table */ - int min_size_ix; /* Never shrink table smaller than this */ int nobjs; /* Number of objects in table */ HashBucket** bucket; /* Vector of bucket pointers (objects) */ }; @@ -96,6 +96,54 @@ void* hash_get(Hash*, void*); void* hash_put(Hash*, void*); void* hash_erase(Hash*, void*); void* hash_remove(Hash*, void*); -void hash_foreach(Hash*, void (*func)(void *, void *), void *); +void hash_foreach(Hash*, HFOREACH_FUN, void *); + +ERTS_GLB_INLINE Uint hash_get_slot(Hash *h, HashValue hv); +ERTS_GLB_INLINE void* hash_fetch(Hash *, void*, H_FUN, HCMP_FUN); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Uint +hash_get_slot(Hash *h, HashValue hv) +{ + /* This slot mapping function uses fibonacci hashing in order to + * protect itself against a very bad hash function. This is not + * a hash function, so the user of hash.h should still spend time + * to figure out a good hash function for its data. + * + * See https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/ + * for some thoughts and ideas about fibonacci hashing. + */ + + /* This is not strictly part of the fibonacci hashing algorithm + * but it does help to spread the values of the mapping function better. + */ + hv ^= hv >> h->shift; +#ifdef ARCH_64 + /* 2^64 / 1.61803398875 = 11400714819323198485.... */ + return (UWORD_CONSTANT(11400714819323198485) * hv) >> h->shift; +#else + /* 2^32 / 1.61803398875 = 2654435769.... */ + return (UWORD_CONSTANT(2654435769) * hv) >> h->shift; +#endif +} + +ERTS_GLB_INLINE void* hash_fetch(Hash *h, void* tmpl, H_FUN hash, HCMP_FUN cmp) +{ + HashValue hval = hash(tmpl); + Uint ix = hash_get_slot(h, hval); + HashBucket* b = h->bucket[ix]; + ASSERT(h->fun.hash == hash); + ASSERT(h->fun.cmp == cmp); + + while(b != (HashBucket*) 0) { + if ((b->hvalue == hval) && (cmp(tmpl, (void*)b) == 0)) + return (void*) b; + b = b->next; + } + return (void*) 0; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index be1771b037..09d3c24424 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -114,35 +114,26 @@ int index_get(IndexTable* t, void* tmpl) return -1; } -void erts_index_merge(Hash* src, IndexTable* dst) +static void index_merge_foreach(IndexSlot *p, IndexTable *dst) { - int limit = src->size; - HashBucket** bucket = src->bucket; - int i; - - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - IndexSlot* p; - int ix; - - while (b) { - Uint sz; - ix = dst->entries++; - if (ix >= dst->size) { - if (ix >= dst->limit) { - erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", - dst->htable.name, dst->limit); - } - sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); - dst->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(dst->type, sz); - dst->size += INDEX_PAGE_SIZE; - } - p = (IndexSlot*) b; - p->index = ix; - dst->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; - b = b->next; - } + Uint sz; + int ix = dst->entries++; + if (ix >= dst->size) { + if (ix >= dst->limit) { + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", + dst->htable.name, dst->limit); + } + sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); + dst->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(dst->type, sz); + dst->size += INDEX_PAGE_SIZE; } + p->index = ix; + dst->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; +} + +void erts_index_merge(Hash* src, IndexTable* dst) +{ + hash_foreach(src, (HFOREACH_FUN)index_merge_foreach, dst); } void index_erase_latest_from(IndexTable* t, Uint from_ix) diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index 7cffe7fb5c..c28b5f3443 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -19,7 +19,12 @@ // %CopyrightEnd% // -// Stack manipulation instructions +// +// Stack manipulation instructions follow. +// +// See the comment for AH() in macros.tab for information about +// the layout of stack frames. +// allocate(NeedStack, Live) { $AH($NeedStack, 0, $Live); @@ -58,105 +63,170 @@ allocate_heap_zero(NeedStack, NeedHeap, Live) { deallocate(Deallocate) { //| -no_prefetch - SET_CP(c_p, (BeamInstr *) cp_val(*E)); E = ADD_BYTE_OFFSET(E, $Deallocate); } -deallocate_return(Deallocate) { - //| -no_next - int words_to_pop = $Deallocate; - SET_I((BeamInstr *) cp_val(*E)); - E = ADD_BYTE_OFFSET(E, words_to_pop); - CHECK_TERM(x(0)); - DispatchReturn; +// +// Micro-benchmarks showed that the deallocate_return instruction +// became slower when the continuation pointer was moved from +// the process struct to the stack. The reason seems to be read +// dependencies, i.e. that the CPU cannot figure out beforehand +// from which position on the stack the continuation pointer +// should be fetched. +// +// Initializing num_bytes with a constant value seems to restore +// the lost speed, so we've specialized the instruction for the +// most common values. +// + +deallocate_return0 := dealloc_ret.n0.execute; +deallocate_return1 := dealloc_ret.n1.execute; +deallocate_return2 := dealloc_ret.n2.execute; +deallocate_return3 := dealloc_ret.n3.execute; +deallocate_return4 := dealloc_ret.n4.execute; +deallocate_return := dealloc_ret.var.execute; + +dealloc_ret.head() { + Uint num_bytes; } -move_deallocate_return(Src, Deallocate) { - x(0) = $Src; - $deallocate_return($Deallocate); +dealloc_ret.n0() { + num_bytes = (0+1) * sizeof(Eterm); } -// Call instructions +dealloc_ret.n1() { + num_bytes = (1+1) * sizeof(Eterm); +} + +dealloc_ret.n2() { + num_bytes = (2+1) * sizeof(Eterm); +} + +dealloc_ret.n3() { + num_bytes = (3+1) * sizeof(Eterm); +} + +dealloc_ret.n4() { + num_bytes = (4+1) * sizeof(Eterm); +} + +dealloc_ret.var(Deallocate) { + num_bytes = $Deallocate; +} -DISPATCH_REL(CallDest) { +dealloc_ret.execute() { //| -no_next - $SET_I_REL($CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + E = ADD_BYTE_OFFSET(E, num_bytes); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } -DISPATCH_ABS(CallDest) { +move_deallocate_return(Src, Deallocate) { //| -no_next - SET_I((BeamInstr *) $CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + /* + * Explicitly do reads first to mitigate the impact of read + * dependencies. + */ + + Uint bytes_to_pop = $Deallocate; + Eterm src = $Src; + E = ADD_BYTE_OFFSET(E, bytes_to_pop); + x(0) = src; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } +// Call instructions + i_call(CallDest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_REL($CallDest); } move_call(Src, CallDest) { - x(0) = $Src; - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH_REL($CallDest); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_last(CallDest, Deallocate) { + //| -no_next $deallocate($Deallocate); $DISPATCH_REL($CallDest); } move_call_last(Src, CallDest, Deallocate) { - x(0) = $Src; - $i_call_last($CallDest, $Deallocate); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_only(CallDest) { + //| -no_next $DISPATCH_REL($CallDest); } move_call_only(Src, CallDest) { - x(0) = $Src; - $i_call_only($CallDest); -} - -DISPATCHX(Dest) { //| -no_next - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest); - // Dispatchx assumes the Export* is in Arg(0) - I = (&$Dest) - 1; - Dispatchx(); + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_ext(Dest) { - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCHX($Dest); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext(Src, Dest) { - x(0) = $Src; - $i_call_ext($Dest); +i_move_call_ext(Src, CallDest) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_only(Dest) { - $DISPATCHX($Dest); + //| -no_next + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_only(Dest, Src) { - x(0) = $Src; - $i_call_ext_only($Dest); +i_move_call_ext_only(CallDest, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_last(Dest, Deallocate) { + //| -no_next $deallocate($Deallocate); - $DISPATCHX($Dest); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_last(Dest, StackOffset, Src) { - x(0) = $Src; - $i_call_ext_last($Dest, $StackOffset); +i_move_call_ext_last(CallDest, Deallocate, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } APPLY(I, Deallocate, Next) { @@ -167,21 +237,23 @@ APPLY(I, Deallocate, Next) { } HANDLE_APPLY_ERROR() { - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[BIF_apply_3].info.mfa); goto post_error_handling; } i_apply() { + //| -no_next BeamInstr *next; $APPLY(NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } i_apply_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY(I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -192,6 +264,7 @@ i_apply_last(Deallocate) { } i_apply_only() { + //| -no_next BeamInstr *next; $APPLY(I, 0, next); if (ERTS_LIKELY(next != NULL)) { @@ -208,16 +281,18 @@ FIXED_APPLY(Arity, I, Deallocate, Next) { } apply(Arity) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } apply_last(Arity, Deallocate) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -237,23 +312,19 @@ HANDLE_APPLY_FUN_ERROR() { goto find_func_info; } -DISPATCH_FUN(I) { - //| -no_next - SET_I($I); - Dispatchfun(); -} - i_apply_fun() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_apply_fun_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -264,6 +335,7 @@ i_apply_fun_last(Deallocate) { } i_apply_fun_only() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -280,16 +352,18 @@ CALL_FUN(Fun, Next) { } i_call_fun(Fun) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_call_fun_last(Fun, Deallocate) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { @@ -301,18 +375,12 @@ i_call_fun_last(Fun, Deallocate) { return() { //| -no_next - SET_I(c_p->cp); - DTRACE_RETURN_FROM_PC(c_p); - - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - DispatchReturn; + + $DISPATCH_RETURN(); } get_list(Src, Hd, Tl) { @@ -478,16 +546,21 @@ i_make_fun(FunP, NumFree) { } move_trim(Src, Dst, Words) { - Uint cp = E[0]; $Dst = $Src; - E += $Words; - E[0] = cp; + $i_trim($Words); } i_trim(Words) { - Uint cp = E[0]; E += $Words; - E[0] = cp; + + /* + * Clear the reserved location for the continuation pointer at + * E[0]. This is not strictly necessary for correctness, but if a + * GC is triggered before E[0] is overwritten by another + * continuation pointer the now dead term at E[0] would be + * retained by the GC. + */ + E[0] = NIL; } move(Src, Dst) { @@ -599,9 +672,9 @@ move_window5(S1, S2, S3, S4, S5, D) { move_return(Src) { //| -no_next x(0) = $Src; - SET_I(c_p->cp); - c_p->cp = 0; - DispatchReturn; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + $DISPATCH_RETURN(); } move_x1(Src) { @@ -683,10 +756,11 @@ swap(R1, R2) { $R2 = V; } -swap_temp(R1, R2, Tmp) { - Eterm V = $R1; - $R1 = $R2; - $R2 = $Tmp = V; +swap2(R1, R2, R3) { + Eterm V = $R2; + $R2 = $R1; + $R1 = $R3; + $R3 = V; } test_heap(Nh, Live) { @@ -952,6 +1026,14 @@ is_ge(Fail, X, Y) { CMP_GE_ACTION($X, $Y, $FAIL($Fail)); } +is_lt_literal(Fail, X, Y) { + CMP_LT_LITERAL_ACTION($X, $Y, $FAIL($Fail)); +} + +is_ge_literal(Fail, X, Y) { + CMP_GE_LITERAL_ACTION($X, $Y, $FAIL($Fail)); +} + badarg(Fail) { $BADARG($Fail); //| -no_next; @@ -991,6 +1073,7 @@ catch_end(Y) { $try_end($Y); if (is_non_value(r(0))) { c_p->fvalue = NIL; + c_p->ftrace = NIL; if (x(1) == am_throw) { r(0) = x(2); } else { @@ -1024,6 +1107,7 @@ try_case(Y) { $try_end($Y); ASSERT(is_non_value(r(0))); c_p->fvalue = NIL; + c_p->ftrace = NIL; r(0) = x(1); x(1) = x(2); x(2) = x(3); diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab index 1b5e5f66b0..802c8aec9a 100644 --- a/erts/emulator/beam/macros.tab +++ b/erts/emulator/beam/macros.tab @@ -104,14 +104,136 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) { // Make sure that there are NeedStack + NeedHeap + 1 words available -// on the combined heap/stack segment, then allocates NeedHeap + 1 -// words on the stack and saves CP. +// on the combined heap/stack segment, then decrement the stack +// pointer by (NeedStack + 1) words. Finally clear the word reserved +// for the continuation pointer at the top of the stack. +// +// Stack frame layout: +// +// +-----------+ +// y(N) | Term | +// +-----------+ +// . +// . +// . +// +-----------+ +// y(0) | Term | +// +-----------+ +// E ==> | NIL or CP | +// +-----------+ +// +// When the function owning the stack frame is the currently executing +// function, the word at the top of the stack is NIL. When calling +// another function, the continuation pointer will be stored in the +// word at the top of the stack. When returning to the function +// owning the stack frame, the word at the stack top will again be set +// to NIL. + AH(NeedStack, NeedHeap, Live) { unsigned needed = $NeedStack + 1; $GC_TEST(needed, $NeedHeap, $Live); E -= needed; - *E = make_cp(c_p->cp); - c_p->cp = 0; + *E = NIL; +} + + +// +// Helpers for call instructions +// + +DISPATCH() { + BeamInstr dis_next; + + dis_next = *I; + CHECK_ARGS(I); + + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(dis_next); + } else { + goto context_switch; + } +} + +DISPATCH_ABS(CallDest) { + SET_I((BeamInstr *) $CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + + $DISPATCH(); +} + +DISPATCH_EXPORT(Export) { + BeamInstr dis_next; + Export *ep; + + ep = (Export*)($Export); + + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, ep); + + SET_I(ep->addressv[erts_active_code_ix()]); + CHECK_ARGS(I); + dis_next = *I; + + if (ERTS_UNLIKELY(FCALLS <= 0)) { + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) && FCALLS > neg_o_reds) { + save_calls(c_p, ep); + } else { + goto context_switch; + } + } + + FCALLS--; + Goto(dis_next); +} + +DISPATCH_FUN(I) { + BeamInstr dis_next; + + SET_I($I); + + dis_next = *I; + CHECK_ARGS(I); + + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(dis_next); + } else { + goto context_switch_fun; + } +} + +DISPATCH_REL(CallDest) { + $SET_I_REL($CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + + $DISPATCH(); +} + +DISPATCH_RETURN() { + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(*I); + } else { + c_p->current = NULL; + c_p->arity = 1; + goto context_switch3; + } +} + +// Save the continuation pointer in the reserved slot at the +// top of the stack as preparation for doing a function call. + +SAVE_CONTINUATION_POINTER(IP) { + ASSERT(VALID_INSTR(*($IP))); + *E = (BeamInstr) ($IP); +} + +// Return to the function whose continuation pointer is stored +// at the top of the stack and set that word to NIL. + +RETURN() { + SET_I(cp_val(*E)); + *E = NIL; } NEXT0() { @@ -167,7 +289,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) { } reg[0] = $Op1; SWAPOUT; - I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[$BIF].info.mfa); goto post_error_handling; } @@ -179,6 +301,6 @@ BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) { reg[0] = $Op1; reg[1] = $Op2; SWAPOUT; - I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[$BIF].info.mfa); goto post_error_handling; } diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index b08466c830..3c989b6e60 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -210,7 +210,8 @@ remove_message() { ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); + ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)) + || is_atom(SEQ_TRACE_TOKEN_SENDER(c_p))); c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index b9d4f6afcc..8c9361c99b 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -74,22 +74,29 @@ trace_jump W return -# To ensure that a "move Src x(0)" instruction can be combined with -# the following call instruction, we need to make sure that there is -# no line/1 instruction between the move and the call. # -# A tail-recursive call to an external function (BIF or non-BIF) will -# never be saved on the stack, so there is no reason to keep the line -# instruction. +# A tail call will not refer to the current function on error unless it's a +# BIF, so we can omit the line instruction for non-BIFs. +# -move S X0=x==0 | line Loc | call_ext Ar Func => \ - line Loc | move S X0 | call_ext Ar Func -move S X0=x==0 | line Loc | call_ext_last Ar Func D => \ +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ move S X0 | call_ext_last Ar Func D -move S X0=x==0 | line Loc | call_ext_only Ar Func => \ +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ move S X0 | call_ext_only Ar Func -move S X0=x==0 | line Loc | call Ar Func => \ - line Loc | move S X0 | call Ar Func + +move S X0=x==0 | line Loc | call_last Ar Func D => \ + move S X0 | call_last Ar Func D +move S X0=x==0 | line Loc | call_only Ar Func => \ + move S X0 | call_only Ar Func + +# To ensure that a "move Src x(0)" instruction can be combined with +# the following call instruction, we need to make sure that there is +# no line/1 instruction between the move and the call. (We don't +# need to match the call instruction, because reordering the move +# and line instructions would be harmless even if no call instruction +# follows.) + +move S X0=x==0 | line Loc => line Loc | move S X0 line Loc | func_info M F A => func_info M F A | line Loc @@ -273,26 +280,19 @@ move_jump f c r # Movement to and from the stack is common. # Try to pack as much as we can into one instruction. -# Window move -move_window/5 -move_window/6 - # x -> y -move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ - move_window X1 X2 X3 Y1 Y3 - -move X1=x Y1=y | move X2=x Y2=y | succ(Y1,Y2) => \ +move X1=x Y1=y | move X2=x Y2=y | succ(Y1, Y2) => \ move_window2 X1 X2 Y1 -move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \ - move_window X1 X2 X3 X4 Y1 Y4 +move_window2 X1 X2 Y1 | move X3=x Y3=y | offset(Y1, Y3, 2) => \ + move_window3 X1 X2 X3 Y1 -move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ - move_window5 X1 X2 X3 X4 X5 Y1 +move_window3 X1 X2 X3 Y1 | move X4=x Y4=y | offset(Y1, Y4, 3) => \ + move_window4 X1 X2 X3 X4 Y1 -move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 -move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 +move_window4 X1 X2 X3 X4 Y1=y | move X5=x Y5=y | offset(Y1, Y5, 4) => \ + move_window5 X1 X2 X3 X4 X5 Y1 move_window2 x x y move_window3 x x x y @@ -324,76 +324,15 @@ move_src_window2 y x x move_src_window3 y x x x move_src_window4 y x x x x -# Swap registers. -move R1=xy Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp - -# The compiler uses x(1022) when swapping registers. It will definitely -# not be used again. -swap_temp R1 R2 Tmp=x==1022 => swap R1 R2 - -swap_temp R1 R2 Tmp | move Src Tmp => swap R1 R2 | move Src Tmp - -swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \ - swap R1 R2 | line Loc | apply Live -swap_temp R1 R2 Tmp | line Loc | apply_last Live D | is_killed_apply(Tmp, Live) => \ - swap R1 R2 | line Loc | apply_last Live D - -swap_temp R1 R2 Tmp | line Loc | call_fun Live | is_killed_by_call_fun(Tmp, Live) => \ - swap R1 R2 | line Loc | call_fun Live -swap_temp R1 R2 Tmp | make_fun2 OldIndex=u | is_killed_by_make_fun(Tmp, OldIndex) => \ - swap R1 R2 | make_fun2 OldIndex - -swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | line Loc | call Live Addr -swap_temp R1 R2 Tmp | call_only Live Addr | \ - is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr -swap_temp R1 R2 Tmp | call_last Live Addr D | \ - is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D - -swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | line Loc | call_ext Live Addr -swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ - is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr -swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ - is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D - -swap_temp R1 R2 Tmp | call_ext Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext Live Addr -swap_temp R1 R2 Tmp | call_ext_only Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext_only Live Addr -swap_temp R1 R2 Tmp | call_ext_last Live Addr D | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext_last Live Addr D - -swap_temp R1 R2 Tmp | move Src Any | line Loc | call Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_ext Live Addr -swap_temp R1 R2 Tmp | move Src Any | call_only Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | call_only Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext_only Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_ext_only Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_fun Live | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_fun Live - -swap_temp R1 R2 Tmp | line Loc | send | is_killed_by_send(Tmp) => \ - swap R1 R2 | line Loc | send - -# swap_temp/3 with Y register operands are rare. -swap_temp R1 R2=y Tmp => swap R1 R2 | move R2 Tmp -swap_temp R1=y R2 Tmp => swap R1 R2 | move R2 Tmp - swap R1=x R2=y => swap R2 R1 -swap_temp x x x - swap xy x swap y y +swap R1=x R2=x | swap R3=x R1 => swap2 R1 R2 R3 + +swap2 x x x + # move_shift move SD=x D=x | move Src=cxy SD=x | distinct(D, Src) => move_shift Src SD D @@ -450,16 +389,6 @@ move2_par X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 move3 y x y x y x move3 x x x x x x -# move_x1, move_x2 - -move C=aiq X=x==1 => move_x1 C -move C=aiq X=x==2 => move_x2 C - -move n D=y => init D - -move_x1 c -move_x2 c - move xy xy move c xy move n x @@ -542,16 +471,27 @@ is_eq_exact f? y y is_ne_exact f? S S +# When either operand for is_lt or is_ge is a literal, +# that literal is almost always an integer and almost never +# an atom. Therefore we use a specialized instruction when +# one of the operands is a literal. + +is_lt Fail Src=x Lit=c => is_lt_literal Fail Src Lit +is_lt Fail Lit=c Src=x => is_lt_literal Fail Lit Src + is_lt f? x x -is_lt f? x c -is_lt f? c x +is_lt_literal f? x c +is_lt_literal f? c x %cold is_lt f? s s %hot +is_ge Fail Src=x Lit=c => is_ge_literal Fail Src Lit +is_ge Fail Lit=c Src=x => is_ge_literal Fail Lit Src + is_ge f? x x -is_ge f? x c -is_ge f? c x +is_ge_literal f? x c +is_ge_literal f? c x %cold is_ge f? s s %hot @@ -635,8 +575,9 @@ put_list s s d %cold normal_exit continue_exit -apply_bif -call_nif +call_bif W +call_nif W W W +call_nif_early call_error_handler error_action_code return_trace @@ -657,8 +598,20 @@ move S x==0 | deallocate D | return => move_deallocate_return S D move_deallocate_return xycn Q +deallocate u==0 | return => deallocate_return0 +deallocate u==1 | return => deallocate_return1 +deallocate u==2 | return => deallocate_return2 +deallocate u==3 | return => deallocate_return3 +deallocate u==4 | return => deallocate_return4 + deallocate D | return => deallocate_return D +deallocate_return0 +deallocate_return1 +deallocate_return2 +deallocate_return3 +deallocate_return4 + deallocate_return Q test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y @@ -836,62 +789,22 @@ allocate_init t t? y # External function and bif calls. ################################################################# -# -# The BIFs erts_internal:check_process_code/1 must be called like a function, -# to ensure that c_p->i (program counter) is set correctly (an ordinary -# BIF call doesn't set it). -# - -call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif +# Expands into call_light_bif(_only)/2 +call_light_bif/1 +call_light_bif_only/1 +call_light_bif_last/2 # -# The BIFs erts_internal:garbage_collect/1 must be called like a function, -# to allow them to invoke the garbage collector. (The stack pointer must -# be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.) +# The load_nif/2 BIF is an instruction. # -call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif -# -# put/2 and erase/1 must be able to do garbage collection, so we must call -# them like functions. -# - -call_ext u==2 Bif=u$bif:erlang:put/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:put/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erlang:put/2 => i_call_ext_only Bif - -call_ext u==1 Bif=u$bif:erlang:erase/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erlang:erase/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erlang:erase/1 => i_call_ext_only Bif - -# -# The process_info/1,2 BIF should be called like a function, to force -# the emulator to set c_p->current before calling it (a BIF call doesn't -# set it). -# -# In addition, we force the use of a non-tail-recursive call. This will ensure -# that c_p->cp points into the function making the call. -# - -call_ext u==1 Bif=u$bif:erlang:process_info/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erlang:process_info/1 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==1 Bif=u$bif:erlang:process_info/1 => allocate u Ar | i_call_ext Bif | deallocate_return u - -call_ext u==2 Bif=u$bif:erlang:process_info/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:process_info/2 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==2 Bif=u$bif:erlang:process_info/2 => allocate u Ar | i_call_ext Bif | deallocate_return u - -# -# load_nif/2 also needs to know calling function like process_info -# -call_ext u==2 Bif=u$bif:erlang:load_nif/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:load_nif/2 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==2 Bif=u$bif:erlang:load_nif/2 => allocate u Ar | i_call_ext Bif | deallocate_return u +call_ext u==2 u$func:erlang:load_nif/2 => i_load_nif +call_ext_last u==2 u$func:erlang:load_nif/2 D => i_load_nif | deallocate_return D +call_ext_only u==2 u$func:erlang:load_nif/2 => i_load_nif | return +%cold +i_load_nif +%hot # # apply/2 is an instruction, not a BIF. @@ -910,33 +823,6 @@ call_ext_last u==3 u$bif:erlang:apply/3 D => i_apply_last D call_ext_only u==3 u$bif:erlang:apply/3 => i_apply_only # -# The exit/1 and throw/1 BIFs never execute the instruction following them; -# thus there is no need to generate any return instruction. -# - -call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif -call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif - -call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif -call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif - -# -# The error/1 and error/2 BIFs never execute the instruction following them; -# thus there is no need to generate any return instruction. -# However, they generate stack backtraces, so if the call instruction -# is call_ext_only/2 instruction, we explicitly do an allocate/2 to store -# the continuation pointer on the stack. -# - -call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif -call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif - -call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \ - allocate u Ar | call_bif Bif -call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \ - allocate u Ar | call_bif Bif - -# # The yield/0 BIF is an instruction # @@ -1048,17 +934,24 @@ call_ext_only u==0 u$func:os:perf_counter/0 => \ i_perf_counter | return # -# The general case for BIFs that have no special instructions. -# A BIF used in the tail must be followed by a return instruction. +# BIFs like process_info/1,2 require up-to-date information about the current +# emulator state, which the ordinary call_light_bif instruction doesn't save. # -# To make trapping and stack backtraces work correctly, we make sure that -# the continuation pointer is always stored on the stack. -call_ext u Bif=u$is_bif => call_bif Bif +call_ext u Bif=u$is_bif | is_heavy_bif(Bif) => \ + i_call_ext Bif +call_ext_last u Bif=u$is_bif D | is_heavy_bif(Bif) => \ + i_call_ext Bif | deallocate_return D +call_ext_only Ar=u Bif=u$is_bif | is_heavy_bif(Bif) => \ + allocate u Ar | i_call_ext Bif | deallocate_return u -call_ext_last u Bif=u$is_bif D => deallocate D | call_bif_only Bif +# +# The general case for BIFs that have no special requirements. +# -call_ext_only Ar=u Bif=u$is_bif => call_bif_only Bif +call_ext u Bif=u$is_bif => call_light_bif Bif +call_ext_last u Bif=u$is_bif D => call_light_bif_last Bif D +call_ext_only Ar=u Bif=u$is_bif => call_light_bif_only Bif # # Any remaining calls are calls to Erlang functions, not BIFs. @@ -1083,14 +976,32 @@ i_apply_fun i_apply_fun_last Q i_apply_fun_only +# +# When a BIF is traced, these instructions make a body call through the export +# entry instead of calling the BIF directly (setting up a temporary stack frame +# if needed). We therefore retain the stack frame in call_light_bif_last, and +# add a deallocate_return after call_light_bif_only to remove the temporary +# stack frame before returning. +# + +call_light_bif Bif=u$is_bif => \ + call_light_bif Bif Bif + +call_light_bif_last Bif=u$is_bif D => \ + call_light_bif Bif Bif | deallocate_return D + +call_light_bif_only Bif=u$is_bif => \ + call_light_bif_only Bif Bif | deallocate_return u + +call_light_bif b e +call_light_bif_only b e + %cold -i_hibernate +i_hibernate i_perf_counter -%hot -call_bif e -call_bif_only e +%hot # # Calls to non-building and guard BIFs. @@ -1252,7 +1163,7 @@ i_bs_get_integer_imm Ms Bits Live Fail Flags Y=y => \ i_bs_get_integer_small_imm xy W f? t x i_bs_get_integer_imm xy W t f? t x -i_bs_get_integer xy f? t t s d +i_bs_get_integer xy f? t t S d i_bs_get_integer_8 xy f? d i_bs_get_integer_16 xy f? d @@ -1265,7 +1176,7 @@ bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) i_bs_get_binary_imm2 xy f? t W t d -i_bs_get_binary2 xy f t? s t d +i_bs_get_binary2 xy f t? S t d i_bs_get_binary_all2 xy f? t t d # Fetching float from binaries. @@ -1302,16 +1213,32 @@ bs_get_tail xy d t # "slots" in the context itself, which lets us continue matching even after # we've passed it off to another function. +bs_start_match4 a==am_no_fail Live=u Src=xy Ctx=d => \ + bs_start_match3 p Src Live Ctx +bs_start_match4 Fail=f Live=u Src=xy Ctx=d => \ + bs_start_match3 Fail Src Live Ctx + %if ARCH_64 + +# This instruction nops on 64-bit platforms +bs_start_match4 a==am_resume Live Same Same => +bs_start_match4 a==am_resume Live Ctx Dst => move Ctx Dst + bs_start_match3 Fail Bin Live Ctx | bs_get_position Ctx Pos=x Ignored => \ i_bs_start_match3_gp Bin Live Fail Ctx Pos -i_bs_start_match3_gp xy t f d x +i_bs_start_match3_gp xy t j d x + +%else + +bs_start_match4 a==am_resume Live Ctx Dst => \ + bs_start_match4 a=am_no_fail Live Ctx Dst + %endif -bs_start_match3 Fail=f ica Live Dst => jump Fail +bs_start_match3 Fail=j ica Live Dst => jump Fail bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst -i_bs_start_match3 xy t f d +i_bs_start_match3 xy t j d # Match context position instructions. 64-bit assumes that all positions can # fit into an unsigned small. @@ -1793,3 +1720,21 @@ i_recv_set build_stacktrace raw_raise + +# +# Specialized move instructions. Since they don't require a second +# instruction, we have intentionally placed them after any other +# transformation rules that starts with a move instruction in order to +# produce better code for the transformation engine. +# + +# move_x1, move_x2 + +move C=aiq X=x==1 => move_x1 C +move C=aiq X=x==2 => move_x2 C + +move n D=y => init D + +move_x1 c +move_x2 c + diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 4b526887b5..8f24725326 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -816,9 +816,6 @@ int packet_parse_http(const char* buf, int len, int* statep, ptr++; if (--n == 0) return -1; } - while (n && SP(ptr)) { /* Skip white space before ':' */ - ptr++; n--; - } if (*ptr != ':') { return -1; } @@ -837,7 +834,9 @@ int packet_parse_http(const char* buf, int len, int* statep, while (n && SP(ptr)) { ptr++; n--; } - return pcb->http_header(arg, name, name_ptr, name_len, + return pcb->http_header(arg, name, + name_ptr, name_len, + buf, name_len, ptr, n); } return -1; diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index 358d650804..b05623efec 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -77,8 +77,10 @@ typedef int HttpResponseMessageFn(void* arg, int major, int minor, int status, typedef int HttpRequestMessageFn(void* arg, const http_atom_t* meth, const char* meth_ptr, int meth_len, const PacketHttpURI*, int major, int minor); typedef int HttpEohMessageFn(void *arg); -typedef int HttpHeaderMessageFn(void* arg, const http_atom_t* name, const char* name_ptr, - int name_len, const char* value_ptr, int value_len); +typedef int HttpHeaderMessageFn(void* arg, const http_atom_t* name, + const char* name_ptr, int name_len, + const char* oname_ptr, int oname_len, + const char* value_ptr, int value_len); typedef int HttpErrorMessageFn(void* arg, const char* buf, int len); typedef int SslTlsFn(void* arg, unsigned type, unsigned major, unsigned minor, const char* data, int len, const char* prefix, int plen); diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index c7e02c6d48..8e44b527a2 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -265,10 +265,8 @@ Eterm erts_whereis_name_to_id(Process *c_p, Eterm name) { Eterm res = am_undefined; - HashValue hval; - int ix; - HashBucket* b; ErtsProcLocks c_p_locks = 0; + RegProc *rp, tmpl; if (c_p) { c_p_locks = ERTS_PROC_LOCK_MAIN; ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); @@ -278,29 +276,14 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) if (c_p && !c_p_locks) erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); - hval = REG_HASH(name); - ix = hval % process_reg.size; - b = process_reg.bucket[ix]; + tmpl.name = name; + rp = hash_fetch(&process_reg, &tmpl, (H_FUN)reg_hash, (HCMP_FUN)reg_cmp); - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b) { - RegProc* rp = (RegProc *) b; - if (rp->name == name) { - /* - * SMP NOTE: No need to lock registered entity since it cannot - * be removed without acquiring write reg lock and id on entity - * is read only. - */ - if (rp->p) - res = rp->p->common.id; - else if (rp->pt) - res = rp->pt->common.id; - break; - } - b = b->next; + if (rp) { + if (rp->p) + res = rp->p->common.id; + else if (rp->pt) + res = rp->pt->common.id; } reg_read_unlock(); @@ -321,10 +304,7 @@ erts_whereis_name(Process *c_p, Port** port, int lock_port) { - RegProc* rp = NULL; - HashValue hval; - int ix; - HashBucket* b; + RegProc* rp = NULL, tmpl; ErtsProcLocks current_c_p_locks; Port *pending_port = NULL; @@ -342,21 +322,8 @@ erts_whereis_name(Process *c_p, * - current_c_p_locks (either c_p_locks or 0) on c_p */ - hval = REG_HASH(name); - ix = hval % process_reg.size; - b = process_reg.bucket[ix]; - - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b) { - if (((RegProc *) b)->name == name) { - rp = (RegProc *) b; - break; - } - b = b->next; - } + tmpl.name = name; + rp = hash_fetch(&process_reg, &tmpl, (H_FUN)reg_hash, (HCMP_FUN)reg_cmp); if (proc) { if (!rp) @@ -564,18 +531,6 @@ int erts_unregister_name(Process *c_p, return res; } -int process_reg_size(void) -{ - int size; - int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) - reg_read_lock(); - size = process_reg.size; - if (lock) - reg_read_unlock(); - return size; -} - int process_reg_sz(void) { int sz; @@ -592,15 +547,24 @@ int process_reg_sz(void) #include "bif.h" +struct registered_foreach_arg { + Eterm res; + Eterm *hp; +}; + +static void +registered_foreach(RegProc *reg, struct registered_foreach_arg *arg) +{ + arg->res = CONS(arg->hp, reg->name, arg->res); + arg->hp += 2; +} + /* return a list of the registered processes */ BIF_RETTYPE registered_0(BIF_ALIST_0) { - int i; - Eterm res; + struct registered_foreach_arg arg; Uint need; - Eterm* hp; - HashBucket **bucket; ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P); @@ -608,41 +572,21 @@ BIF_RETTYPE registered_0(BIF_ALIST_0) if (!proc_locks) erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - bucket = process_reg.bucket; - - /* work out how much heap we need & maybe garb, by scanning through - the registered process table */ - need = 0; - for (i = 0; i < process_reg.size; i++) { - HashBucket *b = bucket[i]; - while (b != NULL) { - need += 2; - b = b->next; - } - } + /* work out how much heap we need */ + need = process_reg.nobjs * 2; if (need == 0) { reg_read_unlock(); BIF_RET(NIL); } - hp = HAlloc(BIF_P, need); - - /* scan through again and make the list */ - res = NIL; + /* scan through again and make the list */ + arg.hp = HAlloc(BIF_P, need); + arg.res = NIL; - for (i = 0; i < process_reg.size; i++) { - HashBucket *b = bucket[i]; - while (b != NULL) { - RegProc *reg = (RegProc *) b; - - res = CONS(hp, reg->name, res); - hp += 2; - b = b->next; - } - } + hash_foreach(&process_reg, (HFOREACH_FUN)registered_foreach, &arg); reg_read_unlock(); - BIF_RET(res); + BIF_RET(arg.res); } diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 27a314ca78..c77bd03653 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -41,7 +41,6 @@ typedef struct reg_proc Eterm name; /* Atom name */ } RegProc; -int process_reg_size(void); int process_reg_sz(void); void init_register_table(void); void register_info(fmtfn_t, void *); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 0d85211be8..3316d9dfde 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -92,6 +92,12 @@ # define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 #endif +#ifdef __GNUC__ +# define ERTS_NOINLINE __attribute__((__noinline__)) +#else +# define ERTS_NOINLINE +#endif + #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS #endif @@ -172,7 +178,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_UNLIKELY(BOOL) (BOOL) #endif -#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +/* AIX doesn't like this and claims section conflicts */ +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) && !defined(_AIX) #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) # define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") )) #else @@ -211,7 +218,7 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif /* In VC++, noreturn is a declspec that has to be before the types, - * but in GNUC it is an att ribute to be placed between return type + * but in GNUC it is an attribute to be placed between return type * and function name, hence __decl_noreturn <types> __noreturn <function name> * * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h @@ -254,6 +261,39 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #endif /* + * ERTS_GCC_DIAG_ON and ERTS_GCC_DIAG_OFF can be used to temporarly + * disable a gcc or clang warning in a file. + * + * Example: + * GCC_DIAG_OFF(unused-function) + * static int test(){ return 0;} + * GCC_DIAG_ON(unused-function) + * + * These macros were orginally authored by Jonathan Wakely and has + * been modified by Patrick Horgan. + * + * Source: http://dbp-consulting.com/tutorials/SuppressingGCCWarnings.html + * + */ +#if ((__GNUC__ * 100) + __GNUC_MINOR__) >= 402 +#define ERTS_GCC_DIAG_STR(s) #s +#define ERTS_GCC_DIAG_JOINSTR(x,y) ERTS_GCC_DIAG_STR(x ## y) +# define ERTS_GCC_DIAG_DO_PRAGMA(x) _Pragma (#x) +# define ERTS_GCC_DIAG_PRAGMA(x) ERTS_GCC_DIAG_DO_PRAGMA(GCC diagnostic x) +# if ((__GNUC__ * 100) + __GNUC_MINOR__) >= 406 +# define ERTS_GCC_DIAG_OFF(x) ERTS_GCC_DIAG_PRAGMA(push) \ + ERTS_GCC_DIAG_PRAGMA(ignored ERTS_GCC_DIAG_JOINSTR(-W,x)) +# define ERTS_GCC_DIAG_ON(x) ERTS_GCC_DIAG_PRAGMA(pop) +# else +# define ERTS_GCC_DIAG_OFF(x) ERTS_GCC_DIAG_PRAGMA(ignored ERTS_GCC_DIAG_JOINSTR(-W,x)) +# define ERTS_GCC_DIAG_ON(x) ERTS_GCC_DIAG_PRAGMA(warning ERTS_GCC_DIAG_JOINSTR(-W,x)) +# endif +#else +# define ERTS_GCC_DIAG_OFF(x) +# define ERTS_GCC_DIAG_ON(x) +#endif + +/* * Compile time assert * (the actual compiler error msg can be a bit confusing) */ @@ -666,7 +706,16 @@ typedef struct preload { */ typedef Eterm ErtsTracer; -#include "erl_osenv.h" + +/* + * This structure contains the rb tree for the erlang osenv copy + * see erl_osenv.h for more details. + */ +typedef struct __erts_osenv_t { + struct __env_rbtnode_t *tree; + int variable_count; + int content_size; +} erts_osenv_t; extern char *erts_default_arg0; diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab index 6bcd12e22e..a6cdbcbc20 100644 --- a/erts/emulator/beam/trace_instrs.tab +++ b/erts/emulator/beam/trace_instrs.tab @@ -20,16 +20,15 @@ // return_trace() { - ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[1]); SWAPOUT; /* Needed for shared heap */ ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+2)/* tracer */); ERTS_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[2])); E += 3; + $RETURN(); Goto(*I); //| -no_next } @@ -45,14 +44,12 @@ i_generic_breakpoint() { } i_return_time_trace() { - ErtsCodeInfo *cinfo = (!is_CP(E[0]) ? NULL - : erts_code_to_codeinfo((BeamInstr*)E[0])); + ErtsCodeInfo *cinfo = (!is_CP(E[1]) ? NULL : erts_code_to_codeinfo((BeamInstr*)E[1])); SWAPOUT; erts_trace_time_return(c_p, cinfo); SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[1])); E += 2; + $RETURN(); Goto(*I); //| -no_next } @@ -60,8 +57,10 @@ i_return_time_trace() { i_return_to_trace() { if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { Uint *cpp = (Uint*) E; + while (is_not_CP(*cpp)) { + cpp++; + } for(;;) { - ASSERT(is_CP(*cpp)); if (IsOpCode(*cp_val(*cpp), return_trace)) { do ++cpp; @@ -81,9 +80,8 @@ i_return_to_trace() { ERTS_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; } - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[0])); E += 1; + $RETURN(); Goto(*I); //| -no_next } @@ -109,7 +107,7 @@ i_hibernate() { goto do_schedule; } else { HEAVY_SWAPIN; - I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); + I = handle_error(c_p, I, reg, &bif_trap_export[BIF_hibernate_3].info.mfa); goto post_error_handling; } //| -no_next diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 9fb37f77fc..f06a4a3d60 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -66,7 +66,7 @@ #undef M_MMAP_THRESHOLD #undef M_MMAP_MAX -#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) +#if (defined(__GLIBC__) || defined(_AIX)) && defined(HAVE_MALLOC_H) #include <malloc.h> #endif @@ -907,7 +907,7 @@ tail_recur: hash = hash * FUNNY_NUMBER10 + num_free; hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; + hash = hash*FUNNY_NUMBER2 + funp->fe->index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { @@ -1069,54 +1069,237 @@ do { \ #define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */ -static Uint32 -block_hash(byte *k, Uint length, Uint32 initval) +typedef struct { + Uint32 a,b,c; +} ErtsBlockHashHelperCtx; + +#define BLOCK_HASH_BYTES_PER_ITER 12 + +/* The three functions below are separated into different functions even + though they are always used together to make trapping and handling + of unaligned binaries easier. Examples of how they are used can be + found in block_hash and make_hash2_helper.*/ +static ERTS_INLINE +void block_hash_setup(Uint32 initval, + ErtsBlockHashHelperCtx* ctx /* out parameter */) +{ + ctx->a = ctx->b = HCONST; + ctx->c = initval; /* the previous hash value */ +} + +static ERTS_INLINE +void block_hash_buffer(byte *buf, + Uint buf_length, + ErtsBlockHashHelperCtx* ctx /* out parameter */) { - Uint32 a,b,c; - Uint len; - - /* Set up the internal state */ - len = length; - a = b = HCONST; - c = initval; /* the previous hash value */ - - while (len >= 12) - { - a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); - b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); - c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); - MIX(a,b,c); - k += 12; len -= 12; - } - - c += length; - switch(len) /* all the case statements fall through */ - { - case 11: c+=((Uint32)k[10]<<24); - case 10: c+=((Uint32)k[9]<<16); - case 9 : c+=((Uint32)k[8]<<8); - /* the first byte of c is reserved for the length */ - case 8 : b+=((Uint32)k[7]<<24); - case 7 : b+=((Uint32)k[6]<<16); - case 6 : b+=((Uint32)k[5]<<8); - case 5 : b+=k[4]; - case 4 : a+=((Uint32)k[3]<<24); - case 3 : a+=((Uint32)k[2]<<16); - case 2 : a+=((Uint32)k[1]<<8); - case 1 : a+=k[0]; - /* case 0: nothing left to add */ - } - MIX(a,b,c); - return c; + Uint len = buf_length; + byte *k = buf; + ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0); + while (len >= BLOCK_HASH_BYTES_PER_ITER) { + ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); + ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); + ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); + MIX(ctx->a,ctx->b,ctx->c); + k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER; + } } +static ERTS_INLINE +Uint32 block_hash_final_bytes(byte *buf, + Uint buf_length, + Uint full_length, + ErtsBlockHashHelperCtx* ctx) +{ + Uint len = buf_length; + byte *k = buf; + ctx->c += full_length; + switch(len) + { /* all the case statements fall through */ + case 11: ctx->c+=((Uint32)k[10]<<24); + case 10: ctx->c+=((Uint32)k[9]<<16); + case 9 : ctx->c+=((Uint32)k[8]<<8); + /* the first byte of c is reserved for the length */ + case 8 : ctx->b+=((Uint32)k[7]<<24); + case 7 : ctx->b+=((Uint32)k[6]<<16); + case 6 : ctx->b+=((Uint32)k[5]<<8); + case 5 : ctx->b+=k[4]; + case 4 : ctx->a+=((Uint32)k[3]<<24); + case 3 : ctx->a+=((Uint32)k[2]<<16); + case 2 : ctx->a+=((Uint32)k[1]<<8); + case 1 : ctx->a+=k[0]; + /* case 0: nothing left to add */ + } + MIX(ctx->a,ctx->b,ctx->c); + return ctx->c; +} + +static Uint32 -make_hash2(Eterm term) +block_hash(byte *block, Uint block_length, Uint32 initval) { + ErtsBlockHashHelperCtx ctx; + Uint no_bytes_not_in_loop = + (block_length % BLOCK_HASH_BYTES_PER_ITER); + Uint no_bytes_to_process_in_loop = + block_length - no_bytes_not_in_loop; + byte *final_bytes = block + no_bytes_to_process_in_loop; + block_hash_setup(initval, &ctx); + block_hash_buffer(block, + no_bytes_to_process_in_loop, + &ctx); + return block_hash_final_bytes(final_bytes, + no_bytes_not_in_loop, + block_length, + &ctx); +} + +typedef enum { + tag_primary_list, + arityval_subtag, + hamt_subtag_head_flatmap, + map_subtag, + fun_subtag, + neg_big_subtag, + sub_binary_subtag_1, + sub_binary_subtag_2, + hash2_common_1, + hash2_common_2, + hash2_common_3, +} ErtsMakeHash2TrapLocation; + +typedef struct { + int c; + Uint32 sh; + Eterm* ptr; +} ErtsMakeHash2Context_TAG_PRIMARY_LIST; + +typedef struct { + int i; + int arity; + Eterm* elem; +} ErtsMakeHash2Context_ARITYVAL_SUBTAG; + +typedef struct { + Eterm *ks; + Eterm *vs; + int i; + Uint size; +} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP; + +typedef struct { + Eterm* ptr; + int i; +} ErtsMakeHash2Context_MAP_SUBTAG; + +typedef struct { + Uint num_free; + Eterm* bptr; +} ErtsMakeHash2Context_FUN_SUBTAG; + +typedef struct { + Eterm* ptr; + Uint i; + Uint n; + Uint32 con; +} ErtsMakeHash2Context_NEG_BIG_SUBTAG; + +typedef struct { + byte* bptr; + Uint sz; + Uint bitsize; + Uint bitoffs; + Uint no_bytes_processed; + ErtsBlockHashHelperCtx block_hash_ctx; + /* The following fields are only used when bitoffs != 0 */ + byte* buf; + int done; + +} ErtsMakeHash2Context_SUB_BINARY_SUBTAG; + +typedef struct { + int dummy__; /* Empty structs are not supported on all platforms */ +} ErtsMakeHash2Context_EMPTY; + +typedef struct { + ErtsMakeHash2TrapLocation trap_location; + /* specific to the trap location: */ + union { + ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list; + ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag; + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap; + ErtsMakeHash2Context_MAP_SUBTAG map_subtag; + ErtsMakeHash2Context_FUN_SUBTAG fun_subtag; + ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2; + ErtsMakeHash2Context_EMPTY hash2_common_1; + ErtsMakeHash2Context_EMPTY hash2_common_2; + ErtsMakeHash2Context_EMPTY hash2_common_3; + } trap_location_state; + /* same for all trap locations: */ + Eterm term; Uint32 hash; Uint32 hash_xor_pairs; - DeclareTmpHeapNoproc(tmp_big,2); + ErtsEStack stack; +} ErtsMakeHash2Context; + +static int make_hash2_ctx_bin_dtor(Binary *context_bin) { + ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin); + DESTROY_SAVED_ESTACK(&context->stack); + if (context->trap_location == sub_binary_subtag_2 && + context->trap_location_state.sub_binary_subtag_2.buf != NULL) { + erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf); + } + return 1; +} +/* hash2_save_trap_state is called seldom so we want to avoid inlining */ +static ERTS_NOINLINE +Eterm hash2_save_trap_state(Eterm state_mref, + Uint32 hash_xor_pairs, + Uint32 hash, + Process* p, + Eterm term, + Eterm* ESTK_DEF_STACK(s), + ErtsEStack s, + ErtsMakeHash2TrapLocation trap_location, + void* trap_location_state_ptr, + size_t trap_location_state_size) { + Binary* state_bin; + ErtsMakeHash2Context* context; + if (state_mref == THE_NON_VALUE) { + Eterm* hp; + state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context), + make_hash2_ctx_bin_dtor); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin); + } else { + state_bin = erts_magic_ref2bin(state_mref); + } + context = ERTS_MAGIC_BIN_DATA(state_bin); + context->term = term; + context->hash = hash; + context->hash_xor_pairs = hash_xor_pairs; + ESTACK_SAVE(s, &context->stack); + context->trap_location = trap_location; + sys_memcpy(&context->trap_location_state, + trap_location_state_ptr, + trap_location_state_size); + erts_set_gc_state(p, 0); + BUMP_ALL_REDS(p); + return state_mref; +} +#undef NOINLINE_HASH2_SAVE_TRAP_STATE + +/* Writes back a magic reference to *state_mref_write_back when the + function traps */ +static ERTS_INLINE Uint32 +make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p) +{ + static const Uint ITERATIONS_PER_RED = 64; + Uint32 hash; + Uint32 hash_xor_pairs; + Eterm term = term_param; ERTS_UNDEF(hash_xor_pairs, 0); /* (HCONST * {2, ..., 22}) mod 2^32 */ @@ -1168,12 +1351,63 @@ make_hash2(Eterm term) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define NOT_SSMALL28_HASH(SMALL) \ + do { \ + Uint64 t; \ + Uint32 x, y; \ + Uint32 con; \ + if (SMALL < 0) { \ + con = HCONST_10; \ + t = (Uint64)(SMALL * (-1)); \ + } else { \ + con = HCONST_11; \ + t = SMALL; \ + } \ + x = t & 0xffffffff; \ + y = t >> 32; \ + UINT32_HASH_2(x, y, con); \ + } while(0) + #ifdef ARCH_64 # define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) #else # define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) #endif +#define TRAP_LOCATION_NO_RED(location_name) \ + do { \ + if(can_trap && iterations_until_trap <= 0) { \ + *state_mref_write_back = \ + hash2_save_trap_state(state_mref, \ + hash_xor_pairs, \ + hash, \ + p, \ + term, \ + ESTK_DEF_STACK(s), \ + s, \ + location_name, \ + &ctx, \ + sizeof(ctx)); \ + return 0; \ + L_##location_name: \ + ctx = context->trap_location_state. location_name; \ + } \ + } while(0) + +#define TRAP_LOCATION(location_name) \ + do { \ + if (can_trap) { \ + iterations_until_trap--; \ + TRAP_LOCATION_NO_RED(location_name); \ + } \ + } while(0) + +#define TRAP_LOCATION_NO_CTX(location_name) \ + do { \ + ErtsMakeHash2Context_EMPTY ctx; \ + TRAP_LOCATION(location_name); \ + } while(0) + /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1186,51 +1420,94 @@ make_hash2(Eterm term) break; case _TAG_IMMED1_SMALL: { - Sint x = signed_val(term); - - if (SMALL_BITS > 28 && !IS_SSMALL28(x)) { - term = small_to_big(x, tmp_big); - break; + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + hash = 0; + NOT_SSMALL28_HASH(small); + return hash; } hash = 0; - SINT32_HASH(x, HCONST); + SINT32_HASH(small, HCONST); return hash; } } }; { Eterm tmp; + long max_iterations = 0; + long iterations_until_trap = 0; + Eterm state_mref = THE_NON_VALUE; + ErtsMakeHash2Context* context = NULL; DECLARE_ESTACK(s); - - UseTmpHeapNoproc(2); + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + if(can_trap){ +#ifdef DEBUG + (void)ITERATIONS_PER_RED; + iterations_until_trap = max_iterations = + (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227; +#else + iterations_until_trap = max_iterations = + ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); +#endif + } + if (can_trap && is_internal_magic_ref(term)) { + Binary* state_bin; + state_mref = term; + state_bin = erts_magic_ref2bin(state_mref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) { + /* Restore state after a trap */ + context = ERTS_MAGIC_BIN_DATA(state_bin); + term = context->term; + hash = context->hash; + hash_xor_pairs = context->hash_xor_pairs; + ESTACK_RESTORE(s, &context->stack); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + switch (context->trap_location) { + case hash2_common_3: goto L_hash2_common_3; + case tag_primary_list: goto L_tag_primary_list; + case arityval_subtag: goto L_arityval_subtag; + case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap; + case map_subtag: goto L_map_subtag; + case fun_subtag: goto L_fun_subtag; + case neg_big_subtag: goto L_neg_big_subtag; + case sub_binary_subtag_1: goto L_sub_binary_subtag_1; + case sub_binary_subtag_2: goto L_sub_binary_subtag_2; + case hash2_common_1: goto L_hash2_common_1; + case hash2_common_2: goto L_hash2_common_2; + } + } + } hash = 0; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: { - int c = 0; - Uint32 sh = 0; - Eterm* ptr = list_val(term); - while (is_byte(*ptr)) { + ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = { + .c = 0, + .sh = 0, + .ptr = list_val(term)}; + while (is_byte(*ctx.ptr)) { /* Optimization for strings. */ - sh = (sh << 8) + unsigned_val(*ptr); - if (c == 3) { - UINT32_HASH(sh, HCONST_4); - c = sh = 0; + ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr); + if (ctx.c == 3) { + UINT32_HASH(ctx.sh, HCONST_4); + ctx.c = ctx.sh = 0; } else { - c++; + ctx.c++; } - term = CDR(ptr); + term = CDR(ctx.ptr); if (is_not_list(term)) break; - ptr = list_val(term); + ctx.ptr = list_val(term); + TRAP_LOCATION(tag_primary_list); } - if (c > 0) - UINT32_HASH(sh, HCONST_4); + if (ctx.c > 0) + UINT32_HASH(ctx.sh, HCONST_4); if (is_list(term)) { - tmp = CDR(ptr); + tmp = CDR(ctx.ptr); ESTACK_PUSH(s, tmp); - term = CAR(ptr); + term = CAR(ctx.ptr); } } break; @@ -1241,34 +1518,39 @@ make_hash2(Eterm term) switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - int i; - int arity = header_arity(hdr); - Eterm* elem = tuple_val(term); - UINT32_HASH(arity, HCONST_9); - if (arity == 0) /* Empty tuple */ + ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = { + .i = 0, + .arity = header_arity(hdr), + .elem = tuple_val(term)}; + UINT32_HASH(ctx.arity, HCONST_9); + if (ctx.arity == 0) /* Empty tuple */ goto hash2_common; - for (i = arity; ; i--) { - term = elem[i]; - if (i == 1) + for (ctx.i = ctx.arity; ; ctx.i--) { + term = ctx.elem[ctx.i]; + if (ctx.i == 1) break; ESTACK_PUSH(s, term); + TRAP_LOCATION(arityval_subtag); } } break; case MAP_SUBTAG: { - Eterm* ptr = boxed_val(term) + 1; Uint size; - int i; + ErtsMakeHash2Context_MAP_SUBTAG ctx = { + .ptr = boxed_val(term) + 1, + .i = 0}; switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_FLATMAP: { flatmap_t *mp = (flatmap_t *)flatmap_val(term); - Eterm *ks = flatmap_get_keys(mp); - Eterm *vs = flatmap_get_values(mp); - size = flatmap_get_size(mp); - UINT32_HASH(size, HCONST_16); - if (size == 0) + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = { + .ks = flatmap_get_keys(mp), + .vs = flatmap_get_values(mp), + .i = 0, + .size = flatmap_get_size(mp)}; + UINT32_HASH(ctx.size, HCONST_16); + if (ctx.size == 0) goto hash2_common; /* We want a portable hash function that is *independent* of @@ -1281,17 +1563,18 @@ make_hash2(Eterm term) ESTACK_PUSH(s, HASH_MAP_TAIL); hash = 0; hash_xor_pairs = 0; - for (i = size - 1; i >= 0; i--) { + for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) { ESTACK_PUSH(s, HASH_MAP_PAIR); - ESTACK_PUSH(s, vs[i]); - ESTACK_PUSH(s, ks[i]); + ESTACK_PUSH(s, ctx.vs[ctx.i]); + ESTACK_PUSH(s, ctx.ks[ctx.i]); + TRAP_LOCATION(hamt_subtag_head_flatmap); } goto hash2_common; } case HAMT_SUBTAG_HEAD_ARRAY: case HAMT_SUBTAG_HEAD_BITMAP: - size = *ptr++; + size = *ctx.ptr++; UINT32_HASH(size, HCONST_16); if (size == 0) goto hash2_common; @@ -1303,27 +1586,28 @@ make_hash2(Eterm term) } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: - i = 16; + ctx.i = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: case HAMT_SUBTAG_NODE_BITMAP: - i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: erts_exit(ERTS_ERROR_EXIT, "bad header"); } - while (i) { - if (is_list(*ptr)) { - Eterm* cons = list_val(*ptr); + while (ctx.i) { + if (is_list(*ctx.ptr)) { + Eterm* cons = list_val(*ctx.ptr); ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); } else { - ASSERT(is_boxed(*ptr)); - ESTACK_PUSH(s, *ptr); + ASSERT(is_boxed(*ctx.ptr)); + ESTACK_PUSH(s, *ctx.ptr); } - i--; ptr++; + ctx.i--; ctx.ptr++; + TRAP_LOCATION(map_subtag); } goto hash2_common; } @@ -1344,22 +1628,25 @@ make_hash2(Eterm term) case FUN_SUBTAG: { ErlFunThing* funp = (ErlFunThing *) fun_val(term); - Uint num_free = funp->num_free; + ErtsMakeHash2Context_FUN_SUBTAG ctx = { + .num_free = funp->num_free, + .bptr = NULL}; UINT32_HASH_2 - (num_free, + (ctx.num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, HCONST); UINT32_HASH_2 - (funp->fe->old_index, funp->fe->old_uniq, HCONST); - if (num_free == 0) { + (funp->fe->index, funp->fe->old_uniq, HCONST); + if (ctx.num_free == 0) { goto hash2_common; } else { - Eterm* bptr = funp->env + num_free - 1; - while (num_free-- > 1) { - term = *bptr--; + ctx.bptr = funp->env + ctx.num_free - 1; + while (ctx.num_free-- > 1) { + term = *ctx.bptr--; ESTACK_PUSH(s, term); + TRAP_LOCATION(fun_subtag); } - term = *bptr; + term = *ctx.bptr; } } break; @@ -1367,70 +1654,190 @@ make_hash2(Eterm term) case HEAP_BINARY_SUBTAG: case SUB_BINARY_SUBTAG: { - byte* bptr; - unsigned sz = binary_size(term); +#define BYTE_BITS 8 + ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = { + .bptr = 0, + /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + * + * The size is truncated to 32 bits on the line + * below so that the code is compatible with old + * versions of the code. This means that hash + * values for binaries with a size greater than + * 4GB do not take all bytes in consideration. + * + * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + */ + .sz = (0xFFFFFFFF & binary_size(term)), + .bitsize = 0, + .bitoffs = 0, + .no_bytes_processed = 0 + }; Uint32 con = HCONST_13 + hash; - Uint bitoffs; - Uint bitsize; - - ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); - if (sz == 0 && bitsize == 0) { + Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER); + ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize); + if (ctx.sz == 0 && ctx.bitsize == 0) { hash = con; - } else { - if (bitoffs == 0) { - hash = block_hash(bptr, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), - HCONST_15); - } - } else { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, - sz + (bitsize != 0)); - erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); - hash = block_hash(buf, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), - HCONST_15); - } - erts_free(ERTS_ALC_T_TMP, (void *) buf); - } + } else if (ctx.bitoffs == 0 && + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + if (can_trap) iterations_until_trap -= iters_for_bin; + hash = block_hash(ctx.bptr, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (ctx.bitoffs == 0) { + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + block_hash_setup(con, block_hash_ctx); + do { + Uint max_bytes_to_process = + iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER : + iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER; + Uint bytes_left = ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(max_bytes_to_process, even_bytes_left); + block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed], + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_1); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while ((ctx.sz - ctx.no_bytes_processed) >= + BLOCK_HASH_BYTES_PER_ITER); + hash = block_hash_final_bytes(ctx.bptr + + ctx.no_bytes_processed, + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (/* ctx.bitoffs != 0 && */ + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes); + Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize; + if (can_trap) iterations_until_trap -= iters_for_bin; + erts_copy_bits(ctx.bptr, + ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy); + hash = block_hash(buf, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, buf); + } else /* ctx.bitoffs != 0 && */ { +#ifdef DEBUG +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3) +#else +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256) +#endif +#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS) + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0); + ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP, + MIN(nr_of_bytes, BINARY_BUF_SIZE)); + block_hash_setup(con, block_hash_ctx); + do { + Uint bytes_left = + ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(BINARY_BUF_SIZE, even_bytes_left); + Uint nr_of_bits_left = + (ctx.sz*BYTE_BITS+ctx.bitsize) - + ctx.no_bytes_processed*BYTE_BITS; + Uint nr_of_bits_to_copy = + MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS); + ctx.done = nr_of_bits_left == nr_of_bits_to_copy; + erts_copy_bits(ctx.bptr + ctx.no_bytes_processed, + ctx.bitoffs, 1, ctx.buf, 0, 1, + nr_of_bits_to_copy); + block_hash_buffer(ctx.buf, + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_2); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while (!ctx.done); + nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + hash = block_hash_final_bytes(ctx.buf + + (ctx.no_bytes_processed - + ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE), + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + Uint last_byte_index = + nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1; + UINT32_HASH_2(ctx.bitsize, + (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf); + context->trap_location_state.sub_binary_subtag_2.buf = NULL; } goto hash2_common; +#undef BYTE_BITS +#undef BINARY_BUF_SIZE +#undef BINARY_BUF_SIZE_BITS } break; case POS_BIG_SUBTAG: case NEG_BIG_SUBTAG: { - Eterm* ptr = big_val(term); - Uint i = 0; - Uint n = BIG_SIZE(ptr); - Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; + Eterm* big_val_ptr = big_val(term); + ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = { + .ptr = big_val_ptr, + .i = 0, + .n = BIG_SIZE(big_val_ptr), + .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11}; #if D_EXP == 16 do { Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - UINT32_HASH_2(x, y, con); - } while (i < n); + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #elif D_EXP == 32 do { Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - UINT32_HASH_2(x, y, con); - } while (i < n); + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #elif D_EXP == 64 do { Uint t; Uint32 x, y; - ASSERT(i < n); - t = BIG_DIGIT(ptr, i++); + ASSERT(ctx.i < ctx.n); + t = BIG_DIGIT(ctx.ptr, ctx.i++); x = t & 0xffffffff; y = t >> 32; - UINT32_HASH_2(x, y, con); - } while (i < n); + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #else #error "unsupported D_EXP size" #endif @@ -1508,13 +1915,13 @@ make_hash2(Eterm term) } case _TAG_IMMED1_SMALL: { - Sint x = signed_val(term); + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + NOT_SSMALL28_HASH(small); + } else { + SINT32_HASH(small, HCONST); + } - if (SMALL_BITS > 28 && !IS_SSMALL28(x)) { - term = small_to_big(x, tmp_big); - break; - } - SINT32_HASH(x, HCONST); goto hash2_common; } } @@ -1529,7 +1936,10 @@ make_hash2(Eterm term) if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); - UnUseTmpHeapNoproc(2); + if (can_trap) { + BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); + ASSERT(!(p->flags & F_DISABLE_GC)); + } return hash; } @@ -1540,18 +1950,37 @@ make_hash2(Eterm term) hash = (Uint32) ESTACK_POP(s); UINT32_HASH(hash_xor_pairs, HCONST_19); hash_xor_pairs = (Uint32) ESTACK_POP(s); + TRAP_LOCATION_NO_CTX(hash2_common_1); goto hash2_common; } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; hash = 0; + TRAP_LOCATION_NO_CTX(hash2_common_2); goto hash2_common; default: break; } + } + TRAP_LOCATION_NO_CTX(hash2_common_3); } } +#undef TRAP_LOCATION_NO_RED +#undef TRAP_LOCATION +#undef TRAP_LOCATION_NO_CTX +} + +Uint32 +make_hash2(Eterm term) +{ + return make_hash2_helper(term, 0, NULL, NULL); +} + +Uint32 +trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p) +{ + return make_hash2_helper(term, 1, state_mref_write_back, p); } /* Term hash function for internal use. @@ -1731,7 +2160,7 @@ make_internal_hash(Eterm term, Uint32 salt) ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; UINT32_HASH_2(num_free, funp->fe->module, HCONST_20); - UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21); + UINT32_HASH_2(funp->fe->index, funp->fe->old_uniq, HCONST_21); if (num_free == 0) { goto pop_next; } else { @@ -2381,7 +2810,7 @@ tailrecur_ne: f1 = (ErlFunThing *) fun_val(a); f2 = (ErlFunThing *) fun_val(b); if (f1->fe->module != f2->fe->module || - f1->fe->old_index != f2->fe->old_index || + f1->fe->index != f2->fe->index || f1->fe->old_uniq != f2->fe->old_uniq || f1->num_free != f2->num_free) { goto not_equal; @@ -2544,10 +2973,14 @@ tailrecur_ne: bb = hashmap_val(b) + 1; switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: + if (aa[0] != bb[0]) + goto not_equal; aa++; bb++; sz = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: + if (aa[0] != bb[0]) + goto not_equal; aa++; bb++; case HAMT_SUBTAG_NODE_BITMAP: sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); @@ -2701,8 +3134,7 @@ Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) if((AN)->sysname != (BN)->sysname) \ RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname)); \ ASSERT((AN)->creation != (BN)->creation); \ - if ((AN)->creation != 0 && (BN)->creation != 0) \ - RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ + RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ } \ } while (0) @@ -2976,7 +3408,7 @@ tailrecur_ne: if (diff != 0) { RETURN_NEQ(diff); } - diff = f1->fe->old_index - f2->fe->old_index; + diff = f1->fe->index - f2->fe->index; if (diff != 0) { RETURN_NEQ(diff); } @@ -4454,201 +4886,6 @@ erts_get_emu_args(Process *c_p) return res; } - -Eterm -erts_get_ethread_info(Process *c_p) -{ - Uint sz, *szp; - Eterm res, *hp, **hpp, *end_hp = NULL; - - sz = 0; - szp = &sz; - hpp = NULL; - - while (1) { - Eterm tup, list, name; -#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ - || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ - || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) - char buf[1024]; - int i; - char **str; -#endif - - res = NIL; - -#ifdef ETHR_X86_MEMBAR_H__ - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, "sse2"), -#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - erts_bld_string(hpp, szp, - (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - ? "yes" : "no")) -#else - erts_bld_string(hpp, szp, "yes") -#endif - ); - res = erts_bld_cons(hpp, szp, tup, res); - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, - "x86" -#ifdef ARCH_64 - "_64" -#endif - " OOO"), - erts_bld_string(hpp, szp, -#ifdef ETHR_X86_OUT_OF_ORDER - "yes" -#else - "no" -#endif - )); - - res = erts_bld_cons(hpp, szp, tup, res); -#endif - -#ifdef ETHR_SPARC_V9_MEMBAR_H__ - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, "Sparc V9"), - erts_bld_string(hpp, szp, -#if defined(ETHR_SPARC_TSO) - "TSO" -#elif defined(ETHR_SPARC_PSO) - "PSO" -#elif defined(ETHR_SPARC_RMO) - "RMO" -#else - "undefined" -#endif - )); - - res = erts_bld_cons(hpp, szp, tup, res); - -#endif - -#ifdef ETHR_PPC_MEMBAR_H__ - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, "lwsync"), - erts_bld_string(hpp, szp, -#if defined(ETHR_PPC_HAVE_LWSYNC) - "yes" -#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) - "no" -#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) - ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" -#else - "undefined" -#endif - )); - - res = erts_bld_cons(hpp, szp, tup, res); - -#endif - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, "Native rw-spinlocks"), -#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL - erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) -#else - erts_bld_string(hpp, szp, "no") -#endif - ); - res = erts_bld_cons(hpp, szp, tup, res); - - tup = erts_bld_tuple(hpp, szp, 2, - erts_bld_string(hpp, szp, "Native spinlocks"), -#ifdef ETHR_NATIVE_SPINLOCK_IMPL - erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) -#else - erts_bld_string(hpp, szp, "no") -#endif - ); - res = erts_bld_cons(hpp, szp, tup, res); - - - list = NIL; -#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL - if (ethr_have_native_dw_atomic()) { - name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); - str = ethr_native_dw_atomic_ops(); - for (i = 0; str[i]; i++) { - erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); - list = erts_bld_cons(hpp, szp, - erts_bld_string(hpp, szp, buf), - list); - } - str = ethr_native_su_dw_atomic_ops(); - for (i = 0; str[i]; i++) { - erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); - list = erts_bld_cons(hpp, szp, - erts_bld_string(hpp, szp, buf), - list); - } - } - else -#endif - name = erts_bld_string(hpp, szp, "no"); - - tup = erts_bld_tuple(hpp, szp, 3, - erts_bld_string(hpp, szp, "Double word native atomics"), - name, - list); - res = erts_bld_cons(hpp, szp, tup, res); - - list = NIL; -#ifdef ETHR_NATIVE_ATOMIC64_IMPL - name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); - str = ethr_native_atomic64_ops(); - for (i = 0; str[i]; i++) { - erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); - list = erts_bld_cons(hpp, szp, - erts_bld_string(hpp, szp, buf), - list); - } -#else - name = erts_bld_string(hpp, szp, "no"); -#endif - tup = erts_bld_tuple(hpp, szp, 3, - erts_bld_string(hpp, szp, "64-bit native atomics"), - name, - list); - res = erts_bld_cons(hpp, szp, tup, res); - - list = NIL; -#ifdef ETHR_NATIVE_ATOMIC32_IMPL - name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); - str = ethr_native_atomic32_ops(); - for (i = 0; str[i]; i++) { - erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); - list = erts_bld_cons(hpp, szp, - erts_bld_string(hpp, szp, buf), - list); - } -#else - name = erts_bld_string(hpp, szp, "no"); -#endif - tup = erts_bld_tuple(hpp, szp, 3, - erts_bld_string(hpp, szp, "32-bit native atomics"), - name, - list); - res = erts_bld_cons(hpp, szp, tup, res); - - if (hpp) { - HRelease(c_p, end_hp, *hpp) - return res; - } - - hp = HAlloc(c_p, sz); - end_hp = hp + sz; - hpp = &hp; - szp = NULL; - } -} - /* * To be used to silence unused result warnings, but do not abuse it. */ |