diff options
125 files changed, 4832 insertions, 2370 deletions
diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex 9e81b610a5..80b2e770fa 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 325fd5c7f6..9b7a502d2d 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -364,19 +364,10 @@ dnl dnl Try to find perl version 5. If found set PERL to the absolute path dnl of the program, if not found set PERL to false. dnl -dnl On some systems /usr/bin/perl is perl 4 and e.g. -dnl /usr/local/bin/perl is perl 5. We try to handle this case by -dnl putting a couple of -dnl Tries to handle the case that there are two programs called perl -dnl in the path and one of them is perl 5 and the other isn't. -dnl AC_DEFUN(LM_PROG_PERL5, [AC_PATH_PROGS(PERL, perl5 perl, false, /usr/local/bin:/opt/local/bin:/usr/local/gnu/bin:${PATH}) -changequote(, )dnl -dnl[ That bracket is needed to balance the right bracket below -if test "$PERL" = "false" || $PERL -e 'exit ($] >= 5)'; then -changequote([, ])dnl +if test "$PERL" = "false"; then ac_cv_path_PERL=false PERL=false dnl AC_MSG_WARN(perl version 5 not found) diff --git a/erts/configure.in b/erts/configure.in index 4890e0a71d..418a111922 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2866,9 +2866,11 @@ AC_CHECK_PROG(M4, m4, m4) dnl Test if JIT can be enabled +JIT_ARCH= if test ${enable_jit} != no; then case "$ARCH" in amd64) + JIT_ARCH=x86 ;; *) if test ${enable_jit} = yes; then @@ -3009,6 +3011,7 @@ else fi AC_SUBST(JIT_ENABLED) +AC_SUBST(JIT_ARCH) AC_SUBST(PRIMARY_FLAVOR) AC_SUBST(FLAVORS) diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index 5359a41d37..1d98dce19d 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -195,11 +195,14 @@ The word <c>LOCATION</c> represents a location, and denotes the number of the last line, and possibly the number of the last column on that line, in the source file. See <seeerl - marker="stdlib:erl_anno">erl_anno(3)</seeerl> for + marker="stdlib:erl_anno"><c>erl_anno(3)</c></seeerl> for details. </p> </item> </list> + <p>See <seetype marker="stdlib:erl_parse#form_info"> + <c>the form_info/0</c></seetype> type in + <c>erl_parse(3)</c> for more details about these values.</p> </section> </section> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 2bba258c07..f69bc9c1d8 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -5596,7 +5596,18 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="3" + <name name="process_flag" arity="2" clause_i="3" since="OTP @OTP-17285@"/> + <fsummary>Set process flag fullsweep_after for the calling process. + </fsummary> + <desc> + <p>Changes the maximum number of generational collections + before forcing a fullsweep for the calling process.</p> + <p>Returns the old value of the flag.</p> + </desc> + </func> + + <func> + <name name="process_flag" arity="2" clause_i="4" anchor="process_flag_min_heap_size" since=""/> <fsummary>Set process flag min_heap_size for the calling process. </fsummary> @@ -5607,7 +5618,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="4" since="OTP R13B04"/> + <name name="process_flag" arity="2" clause_i="5" since="OTP R13B04"/> <fsummary>Set process flag min_bin_vheap_size for the calling process. </fsummary> <desc> @@ -5618,7 +5629,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="5" + <name name="process_flag" arity="2" clause_i="6" anchor="process_flag_max_heap_size" since="OTP 19.0"/> <fsummary>Set process flag max_heap_size for the calling process. </fsummary> @@ -5692,7 +5703,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="6" + <name name="process_flag" arity="2" clause_i="7" anchor="process_flag_message_queue_data" since="OTP 19.0"/> <fsummary>Set process flag message_queue_data for the calling process. </fsummary> @@ -5734,7 +5745,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="7" + <name name="process_flag" arity="2" clause_i="8" anchor="process_flag_priority" since=""/> <fsummary>Set process flag priority for the calling process.</fsummary> <type name="priority_level"/> @@ -5807,7 +5818,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="8" since=""/> + <name name="process_flag" arity="2" clause_i="9" since=""/> <fsummary>Set process flag save_calls for the calling process.</fsummary> <desc> <p><c><anno>N</anno></c> must be an integer in the interval 0..10000. @@ -5838,7 +5849,7 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="9" since=""/> + <name name="process_flag" arity="2" clause_i="10" since=""/> <fsummary>Set process flag sensitive for the calling process.</fsummary> <desc> <p>Sets or clears flag <c>sensitive</c> for the current process. @@ -11608,13 +11619,6 @@ improper_end</pre> <p>When <c>PidPort</c> gets unlinked from a process <c>Pid2</c>.</p> </item> <tag> - <marker id="trace_3_trace_messages_exit"></marker> - <c>{trace, Pid, exit, Reason}</c> - </tag> - <item> - <p>When <c>Pid</c> exits with reason <c>Reason</c>.</p> - </item> - <tag> <marker id="trace_3_trace_messages_open"></marker> <c>{trace, Port, open, Pid, Driver}</c> </tag> @@ -11722,6 +11726,12 @@ improper_end</pre> <item>The total size of binaries allowed in the virtual old heap in the process before doing a garbage collection.</item> + <tag><c>wordsize</c></tag> + <item>For the <c>gc_minor_start</c> event it is the size + of the need that triggered the GC. For the corresponding + <c>gc_minor_end</c> event it is the size of reclaimed + memory = start <c>heap_size</c> - end + <c>heap_size</c>.</item> </taglist> <p>All sizes are in words.</p> </item> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 14a3c2b58d..de11e50ec2 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -1369,6 +1369,50 @@ </section> +<section><title>Erts 10.7.2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in the timer implementation which could cause + timers that were set to more than 37.25 hours in the + future to be delayed. This could occur if there were + multiple timers scheduled to be triggered very close in + time, but still at different times, and the scheduler + thread handling the timers was not able to handle them + quickly enough. Delayed timers were in this case + triggered when another unrelated timer was triggered.</p> + <p> + Own Id: OTP-17253</p> + </item> + <item> + <p> + Fix bug in call_time tracing (used by eprof) that could + cause VM crash. Bug exists since OTP-22.2 (but not in + OTP-23).</p> + <p> + Own Id: OTP-17290 Aux Id: GH-4635 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix a file descriptor leak when using sendfile and the + remote side closes the connection. This bug has been + present since OTP-21.0.</p> + <p> + Own Id: OTP-17244</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.7.2.8</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3389,6 +3433,42 @@ </section> +<section><title>Erts 10.3.5.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in the timer implementation which could cause + timers that were set to more than 37.25 hours in the + future to be delayed. This could occur if there were + multiple timers scheduled to be triggered very close in + time, but still at different times, and the scheduler + thread handling the timers was not able to handle them + quickly enough. Delayed timers were in this case + triggered when another unrelated timer was triggered.</p> + <p> + Own Id: OTP-17253</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix a file descriptor leak when using sendfile and the + remote side closes the connection. This bug has been + present since OTP-21.0.</p> + <p> + Own Id: OTP-17244</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.3.5.16</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 5320de8bef..c3449c5f73 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -25,6 +25,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk ENABLE_ALLOC_TYPE_VARS = @ENABLE_ALLOC_TYPE_VARS@ JIT_ENABLED=@JIT_ENABLED@ +JIT_ARCH=@JIT_ARCH@ DTRACE_ENABLED=@DTRACE_ENABLED@ DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@ USE_VM_PROBES=@USE_VM_PROBES@ @@ -57,9 +58,9 @@ endif ifeq ($(FLAVOR),jit) OMIT_OMIT_FP=no OPCODE_TABLES += \ - beam/jit/ops.tab \ - beam/jit/predicates.tab \ - beam/jit/generators.tab + beam/jit/$(JIT_ARCH)/ops.tab \ + beam/jit/$(JIT_ARCH)/predicates.tab \ + beam/jit/$(JIT_ARCH)/generators.tab else OMIT_OMIT_FP=no OPCODE_TABLES += \ @@ -414,8 +415,8 @@ CREATE_DIRS += $(OBJDIR) \ $(ZLIB_OBJDIR) ifeq ($(FLAVOR),jit) -CREATE_DIRS+=$(OBJDIR)/asmjit/ $(OBJDIR)/asmjit/core $(OBJDIR)/asmjit/x86 -CREATE_DIRS+=$(TTF_DIR)/asmjit $(TTF_DIR)/asmjit/core $(TTF_DIR)/asmjit/x86 +CREATE_DIRS+=$(OBJDIR)/asmjit/ $(OBJDIR)/asmjit/core $(OBJDIR)/asmjit/$(JIT_ARCH) +CREATE_DIRS+=$(TTF_DIR)/asmjit $(TTF_DIR)/asmjit/core $(TTF_DIR)/asmjit/$(JIT_ARCH) endif BINDIR = $(ERL_TOP)/bin/$(TARGET) @@ -690,8 +691,8 @@ $(UTILS_YCF_OUTPUT): beam/utils.c $(ERTS_LIB) GENERATE += $(DB_INSERT_LIST_OUTPUT) $(MAPS_YCF_OUTPUT) $(UTILS_YCF_OUTPUT) ## ASMJIT source files and headers -ASMJIT_CPP = $(wildcard asmjit/core/*.cpp) $(wildcard asmjit/x86/*.cpp) -ASMJIT_H = $(wildcard asmjit/*.h) $(wildcard asmjit/core/*.h) $(wildcard asmjit/x86/*.h) +ASMJIT_CPP = $(wildcard asmjit/core/*.cpp) $(wildcard asmjit/$(JIT_ARCH)/*.cpp) +ASMJIT_H = $(wildcard asmjit/*.h) $(wildcard asmjit/core/*.h) $(wildcard asmjit/$(JIT_ARCH)/*.h) ASMJIT_TTF_H = $(foreach HEADER,$(ASMJIT_H),$(TTF_DIR)/$(HEADER)) ifeq ($(FLAVOR),jit) @@ -776,6 +777,10 @@ COMMON_INCLUDES += -Ipcre COMMON_INCLUDES += -I../include -I../include/$(TARGET) COMMON_INCLUDES += -I../include/internal -I../include/internal/$(TARGET) +ifeq ($(FLAVOR),jit) +COMMON_INCLUDES += -Ibeam/jit -Ibeam/jit/$(JIT_ARCH) +endif + INCLUDES = -I$(TTF_DIR) $(COMMON_INCLUDES) ifeq ($(TARGET),win32) @@ -901,7 +906,7 @@ ASMJIT_FLAGS=-DASMJIT_EMBED=1 -DASMJIT_NO_BUILDER=1 -DASMJIT_NO_DEPRECATED=1 -DA ASMJIT_PCH_OBJ=$(TTF_DIR)/asmjit/asmjit.hpp.gch ASMJIT_PCH_SRC=$(TTF_DIR)/asmjit/asmjit.hpp -$(OBJDIR)/%.o: beam/jit/%.cpp beam/jit/beam_asm.hpp $(ASMJIT_PCH_OBJ) +$(OBJDIR)/%.o: beam/jit/$(JIT_ARCH)/%.cpp beam/jit/$(JIT_ARCH)/beam_asm.hpp $(ASMJIT_PCH_OBJ) $(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES) \ $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS)) \ -include $(ASMJIT_PCH_SRC) -c $< -o $@ @@ -987,7 +992,11 @@ COMMON_OBJS = \ ASMJIT_OBJS = $(patsubst %.cpp,$(OBJDIR)/%.o,$(ASMJIT_CPP)) JIT_OBJS = \ - $(OBJDIR)/asm_load.o \ + $(OBJDIR)/asm_load.o \ + $(OBJDIR)/beam_jit_common.o + +ifeq ($(JIT_ARCH), x86) +JIT_OBJS += \ $(OBJDIR)/beam_asm.o \ $(OBJDIR)/beam_asm_global.o \ $(OBJDIR)/beam_asm_module.o \ @@ -1002,9 +1011,10 @@ JIT_OBJS = \ $(OBJDIR)/instr_map.o \ $(OBJDIR)/instr_msg.o \ $(OBJDIR)/instr_select.o \ - $(OBJDIR)/instr_trace.o \ - $(ASMJIT_OBJS) + $(OBJDIR)/instr_trace.o +endif +JIT_OBJS += $(ASMJIT_OBJS) EMU_OBJS = \ $(OBJDIR)/emu_load.o @@ -1262,7 +1272,7 @@ BEAM_SRC=$(wildcard beam/*.c) ifeq ($(FLAVOR),emu) BEAM_SRC+=$(wildcard beam/emu/*.c) else -BEAM_CPP_SRC=$(wildcard beam/*.cpp) $(wildcard beam/jit/*.cpp) +BEAM_CPP_SRC=$(wildcard beam/*.cpp) $(wildcard beam/jit/*.cpp) $(wildcard beam/jit/$(JIT_ARCH)/*.cpp) endif DRV_COMMON_SRC=$(wildcard drivers/common/*.c) DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c) @@ -1315,7 +1325,7 @@ DEP_INCLUDES=$(INCLUDES) \ -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \ -Inifs/common -Inifs/$(ERLANG_OSTYPE) DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(DEP_INCLUDES) -DEP_CXXFLAGS=-MM $(MG_FLAG) $(CXXFLAGS) $(DEP_INCLUDES) +DEP_CXXFLAGS=-MM $(MG_FLAG) $(CXXFLAGS) $(DEP_INCLUDES) $(ASMJIT_FLAGS) SYS_SRC=$(ALL_SYS_SRC) endif @@ -1378,9 +1388,12 @@ compdb: cat $(TTF_DIR)/*.json > $(ERL_TOP)/compile_commands.json format-check: - clang-format --Werror --dry-run -i beam/jit/*.cpp beam/jit/*.hpp beam/jit/*.c beam/jit/*.h + clang-format --Werror --dry-run -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \ + beam/jit/*/*.cpp beam/jit/*/*.hpp + format: - clang-format -i beam/jit/*.cpp beam/jit/*.hpp beam/jit/*.c beam/jit/*.h + clang-format -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \ + beam/jit/*/*.cpp beam/jit/*/*.hpp ifneq ($(ERTS_SKIP_DEPEND),true) ifneq ($(MAKECMDGOALS),clean) diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index af3437c919..d834939cab 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1854,6 +1854,19 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) else BIF_RET(old_value); } + else if (BIF_ARG_1 == am_fullsweep_after) { + Sint i; + if (!is_small(BIF_ARG_2)) { + goto error; + } + i = signed_val(BIF_ARG_2); + if (i < 0) { + goto error; + } + old_value = make_small(BIF_P->max_gen_gcs); + BIF_P->max_gen_gcs = i; + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_min_heap_size) { Sint i; if (!is_small(BIF_ARG_2)) { diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c index b925c1e339..e45708e077 100644 --- a/erts/emulator/beam/erl_bif_persistent.c +++ b/erts/emulator/beam/erl_bif_persistent.c @@ -439,7 +439,7 @@ BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2) BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) { - HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + HashTable* hash_table; TrapData* trap_data; Eterm res = NIL; Eterm magic_ref; @@ -450,6 +450,8 @@ BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0) ERTS_BIF_YIELD0(BIF_TRAP_EXPORT(BIF_persistent_term_get_0), BIF_P); } + hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + magic_ref = alloc_trap_data(BIF_P); mbp = erts_magic_ref2bin(magic_ref); trap_data = ERTS_MAGIC_BIN_DATA(mbp); @@ -673,7 +675,7 @@ BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0) BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) { - HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + HashTable* hash_table; TrapData* trap_data; Eterm res = NIL; Eterm magic_ref; @@ -684,6 +686,8 @@ BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0) ERTS_BIF_YIELD0(BIF_TRAP_EXPORT(BIF_persistent_term_info_0), BIF_P); } + hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table); + magic_ref = alloc_trap_data(BIF_P); mbp = erts_magic_ref2bin(magic_ref); trap_data = ERTS_MAGIC_BIN_DATA(mbp); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 7cdb532f75..28b0e80e02 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -3477,19 +3477,21 @@ reached_max_heap_size(Process *p, Uint total_heap_size, Eterm *o_hp, *hp, args = NIL; /* Build the format message */ - erts_dsprintf(dsbufp, " Process: ~p "); + erts_dsprintf(dsbufp, " Process: ~p "); if (alive) erts_dsprintf(dsbufp, "on node ~p"); - erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); - erts_dsprintf(dsbufp, " Max Heap Size: ~p~n"); - erts_dsprintf(dsbufp, " Total Heap Size: ~p~n"); - erts_dsprintf(dsbufp, " Kill: ~p~n"); - erts_dsprintf(dsbufp, " Error Logger: ~p~n"); - erts_dsprintf(dsbufp, " GC Info: ~p~n"); + erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); + erts_dsprintf(dsbufp, " Max Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Total Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Kill: ~p~n"); + erts_dsprintf(dsbufp, " Error Logger: ~p~n"); + erts_dsprintf(dsbufp, " Message Queue Len: ~p~n"); + erts_dsprintf(dsbufp, " GC Info: ~p~n"); /* Build the args in reverse order */ - o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm)); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 8 : 7) * sizeof(Eterm)); args = CONS(hp, msg, args); hp += 2; + args = CONS(hp, make_small((p)->sig_inq.len), args); hp += 2; args = CONS(hp, am_true, args); hp += 2; args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2; args = CONS(hp, make_small(total_heap_size), args); hp += 2; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index c61094c95b..6ea1759cf8 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -205,7 +205,7 @@ int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ int erts_no_line_info = 0; /* -L: Don't load line information */ #ifdef BEAMASM -int erts_asm_dump = 0; /* -asmdump: Dump assembly code */ +int erts_jit_asm_dump = 0; /* -JDdump: Dump assembly code */ #endif /* @@ -1629,6 +1629,20 @@ erl_start(int argc, char **argv) switch (sub_param[0]) { + case 'D': + sub_param++; + if (has_prefix("dump", sub_param)) { + arg = get_arg(sub_param+4, argv[i + 1], &i); + if (sys_strcmp(arg, "true") == 0) { + erts_jit_asm_dump = 1; + } else if (sys_strcmp(arg, "false") == 0) { + erts_jit_asm_dump = 0; + } else { + erts_fprintf(stderr, "bad +JDdump flag %s\n", arg); + erts_usage(); + } + } + break; case 'P': sub_param++; @@ -1655,7 +1669,7 @@ erl_start(int argc, char **argv) } break; default: - erts_fprintf(stderr, "invalid JIT option %s\n", arg); + erts_fprintf(stderr, "invalid JIT option %s\n", argv[i]); erts_usage(); break; } @@ -2116,12 +2130,6 @@ erl_start(int argc, char **argv) break; case 'a': -#ifdef BEAMASM - if (strcmp(argv[i]+2, "smdump") == 0) { - erts_asm_dump = 1; - break; - } -#endif /* suggested stack size (Kilo Words) for threads in thread pool */ arg = get_arg(argv[i]+2, argv[i+1], &i); erts_async_thread_suggested_stack_size = atoi(arg); diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 9b1762e299..4f9d9ce287 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -2370,39 +2370,53 @@ Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Uint size, upsz; Eterm *hp, res = THE_NON_VALUE; DECLARE_ESTACK(stack); - if (erts_hashmap_insert_down(hx, key, map, &size, &upsz, &stack, is_update)) { - hp = HAlloc(p, size); - res = erts_hashmap_insert_up(hp, key, value, &upsz, &stack); + if (erts_hashmap_insert_down(hx, key, value, map, &size, &upsz, &stack, + is_update)) { + if (size) { + /* We are putting a new value (under a new or existing key) */ + hp = HAlloc(p, size); + res = erts_hashmap_insert_up(hp, key, value, upsz, &stack); + } + else { + /* We are putting the same key-value */ + res = map; + } + } + else { + /* We are updating and the key does not exist */ + ASSERT(is_update); } - DESTROY_ESTACK(stack); + DESTROY_ESTACK(stack); return res; } -int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm value, Eterm node, Uint *sz, Uint *update_size, ErtsEStack *sp, int is_update) { Eterm *ptr; Eterm hdr, ckey; Uint32 ix, cix, bp, hval, chx; Uint slot, lvl = 0, clvl; Uint size = 0, n = 0; - DeclareTmpHeapNoproc(th,2); + Eterm th[2]; *update_size = 1; - UseTmpHeapNoproc(2); for (;;) { switch(primary_tag(node)) { case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */ ptr = list_val(node); ckey = CAR(ptr); if (EQ(ckey, key)) { + if (CDR(ptr) == value) { + *sz = 0; /* same value, same map, no heap needed */ + return 1; + } *update_size = 0; goto unroll; } if (is_update) { - UnUseTmpHeapNoproc(2); return 0; } goto insert_subnodes; @@ -2435,7 +2449,6 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, if (!(bp & hval)) { /* not occupied */ if (is_update) { - UnUseTmpHeapNoproc(2); return 0; } size += HAMT_NODE_BITMAP_SZ(n+1); @@ -2467,7 +2480,6 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, } /* not occupied */ if (is_update) { - UnUseTmpHeapNoproc(2); return 0; } size += HAMT_HEAD_BITMAP_SZ(n+1); @@ -2501,12 +2513,11 @@ insert_subnodes: unroll: *sz = size + /* res cons */ 2; - UnUseTmpHeapNoproc(2); return 1; } Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, - Uint *update_size, ErtsEStack *sp) { + Uint update_size, ErtsEStack *sp) { Eterm node, *ptr, hdr; Eterm res; Eterm *nhp = NULL; @@ -2551,7 +2562,7 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, nhp = hp; n = HAMT_HEAD_ARRAY_SZ - 2; *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; - *hp++ = (*ptr++) + *update_size; + *hp++ = (*ptr++) + update_size; while(n--) { *hp++ = *ptr++; } nhp[slot+2] = res; res = make_hashmap(nhp); @@ -2579,7 +2590,7 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, hval = MAP_HEADER_VAL(hdr); nhp = hp; *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++; - *hp++ = (*ptr++) + *update_size; + *hp++ = (*ptr++) + update_size; n -= slot; while(slot--) { *hp++ = *ptr++; } diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 718d400e22..980cf246c4 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -85,10 +85,10 @@ int erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update); -int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm value, Eterm node, Uint *sz, Uint *upsz, struct ErtsEStack_ *sp, int is_update); Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, - Uint *upsz, struct ErtsEStack_ *sp); + Uint upsz, struct ErtsEStack_ *sp); int erts_validate_and_sort_flatmap(flatmap_t* map); void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 40ba7f1128..3181958498 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -5092,16 +5092,27 @@ static void dbg_assert_in_env(ErlNifEnv* env, Eterm term, real_htop = env->hp; } if (!erts_dbg_within_proc(ptr_val(term), env->proc, real_htop)) { - fprintf(stderr, "\r\nFAILED ASSERTION in %s:\r\n", func); - if (nr) { - fprintf(stderr, "Term #%d of the %s is not from same ErlNifEnv.", - nr, type); + int ok = 0; + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + Process* real_proc = env->proc->next; + ASSERT(real_proc); + erts_proc_lock(real_proc, ERTS_PROC_LOCK_MAIN); + ok = (ERTS_PROC_IS_EXITING(real_proc) + || erts_dbg_within_proc(ptr_val(term), real_proc, NULL)); + erts_proc_unlock(real_proc, ERTS_PROC_LOCK_MAIN); } - else { - fprintf(stderr, "The %s is not from the same ErlNifEnv.", type); + if (!ok) { + fprintf(stderr, "\r\nFAILED ASSERTION in %s:\r\n", func); + if (nr) { + fprintf(stderr, "Term #%d of the %s is not from same ErlNifEnv.", + nr, type); + } + else { + fprintf(stderr, "The %s is not from the same ErlNifEnv.", type); + } + fprintf(stderr, "\r\nABORTING\r\n"); + abort(); } - fprintf(stderr, "\r\nABORTING\r\n"); - abort(); } if (env->heap_frag) { env->heap_frag->used_size = saved_used_size; diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 8b8b3fdbe3..ac1ce1473f 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -3973,6 +3973,10 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing, ASSERT(tracing); if (*next_nm_sig != &c_p->sig_qs.cont) { + if (ERTS_SIG_IS_RECV_MARKER(c_p->sig_qs.cont)) { + ErtsRecvMarker *markp = (ErtsRecvMarker *) c_p->sig_qs.cont; + markp->prev_next = c_p->sig_qs.last; + } if (*next_nm_sig == tracing->messages.next) tracing->messages.next = &c_p->sig_qs.cont; *c_p->sig_qs.last = c_p->sig_qs.cont; @@ -6270,7 +6274,7 @@ handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing, */ next_sig = &sig->next; sig = *next_sig; - ASSERT(!ERTS_SIG_IS_RECV_MARKER(sig) + ASSERT(!sig || !ERTS_SIG_IS_RECV_MARKER(sig) || !((ErtsRecvMarker *) sig)->in_msgq); } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9c4346c5ec..dfd3549391 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -661,7 +661,7 @@ typedef struct ErtsSchedulerRegisters_ { ErtsCodePtr start_time_i; UWord start_time; -#if !defined(NATIVE_ERLANG_STACK) && defined(HARD_DEBUG) +#if !defined(NATIVE_ERLANG_STACK) && defined(JIT_HARD_DEBUG) /* Holds the initial thread stack pointer. Used to ensure that everything * that is pushed to the stack is also popped. */ UWord *initial_sp; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2051377398..956d1dc468 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1263,7 +1263,7 @@ extern int erts_initialized; extern int erts_compat_rel; #ifdef BEAMASM -extern int erts_asm_dump; +extern int erts_jit_asm_dump; #endif void erl_start(int, char**); diff --git a/erts/emulator/beam/jit/beam_jit_common.c b/erts/emulator/beam/jit/beam_jit_common.c new file mode 100644 index 0000000000..9b56fb1ac1 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.c @@ -0,0 +1,756 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "bif.h" +#include "code_ix.h" +#include "erl_proc_sig_queue.h" +#include "erl_binary.h" +#include "erl_bits.h" +#include "erl_map.h" +#include "beam_common.h" +#ifdef USE_VM_PROBES +# include "dtrace-wrapper.h" +#endif + +#include "beam_jit_common.h" + +#if defined(DEBUG) && defined(JIT_HARD_DEBUG) +void beam_jit_validate_term(Eterm term) { + if (is_boxed(term)) { + Eterm header = *boxed_val(term); + + if (header_is_bin_matchstate(header)) { + return; + } + } + + size_object_x(term, NULL); +} +#endif + +Eterm beam_jit_call_bif(Process *c_p, + Eterm *reg, + ErtsCodePtr I, + ErtsBifFunc vbf, + Uint arity) { + ErlHeapFragment *live_hf_end; + Eterm result; + + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + { + live_hf_end = c_p->mbuf; + + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + result = vbf(c_p, reg, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_CHK_MBUF_SZ(c_p); + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_HOLE_CHECK(c_p); + } + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + + if (ERTS_IS_GC_DESIRED(c_p)) { + result = erts_gc_after_bif_call_lhf(c_p, + live_hf_end, + result, + reg, + arity); + } + + return result; +} + +Eterm beam_jit_call_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg, + BeamJitNifF *fp, + struct erl_module_nif *NifMod) { + Eterm nif_bif_result; + Eterm bif_nif_arity; + ErlHeapFragment *live_hf_end; + const ErtsCodeMFA *codemfa; + + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + bif_nif_arity = codemfa->arity; + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + + { + struct enif_environment_t env; + ASSERT(c_p->scheduler_data); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + erts_pre_nif(&env, c_p, NifMod, NULL); + + ASSERT((c_p->scheduler_data)->current_nif == NULL); + (c_p->scheduler_data)->current_nif = &env; + + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; + + ASSERT((c_p->scheduler_data)->current_nif == &env); + (c_p->scheduler_data)->current_nif = NULL; + + erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ASSERT(!env.exiting); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + ERTS_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, + live_hf_end, + nif_bif_result, + reg, + bif_nif_arity); + } + + return nif_bif_result; +} + +enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg) { + if (erts_try_seize_code_write_permission(c_p)) { + Eterm result; + + PROCESS_MAIN_CHK_LOCKS((c_p)); + ERTS_UNREQ_PROC_MAIN_LOCK((c_p)); + result = erts_load_nif(c_p, I, reg[0], reg[1]); + erts_release_code_write_permission(); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + + if (ERTS_LIKELY(is_value(result))) { + reg[0] = result; + return RET_NIF_success; + } else { + c_p->freason = BADARG; + return RET_NIF_error; + } + } else { + /* Yield and try again. */ + c_p->current = NULL; + c_p->arity = 2; + return RET_NIF_yield; + } +} + +Uint beam_jit_get_map_elements(Eterm map, + Eterm *reg, + Eterm *E, + Uint n, + Eterm *fs) { + Uint sz; + + /* This instruction assumes Arg1 is a map, i.e. that it follows a test + * is_map if needed. */ + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; + + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); + + if (sz == 0) { + return 0; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while (sz) { + if (EQ(fs[0], *ks)) { + PUT_TERM_REG(*vs, fs[1]); + + n--; + fs += 3; + + /* no more values to fetch, we are done */ + if (n == 0) { + return 1; + } + } + + ks++, sz--, vs++; + } + return 0; + } else { + ASSERT(is_hashmap(map)); + + while (n--) { + const Eterm *v; + Uint32 hx; + + hx = fs[2]; + ASSERT(hx == hashmap_make_hash(fs[0])); + + if ((v = erts_hashmap_get(hx, fs[0], map)) == NULL) { + return 0; + } + + PUT_TERM_REG(*v, fs[1]); + fs += 3; + } + + return 1; + } +} + +static void test_bin_vheap(Process *c_p, + Eterm *reg, + Uint VNh, + Uint Nh, + Uint Live) { + int need = Nh; + + if (c_p->stop - c_p->htop < (need + S_RESERVED) || + MSO(c_p).overhead + VNh >= BIN_VHEAP_SZ(c_p)) { + c_p->fcalls -= + erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); + } +} + +static void gc_test(Process *c_p, Eterm *reg, Uint Ns, Uint Nh, Uint Live) { + int need = Nh + Ns; + + if (ERTS_UNLIKELY(c_p->stop - c_p->htop < (need + S_RESERVED))) { + c_p->fcalls -= + erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); + } +} + +void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size) { + if (((is_small(size) && signed_val(size) >= 0) || + (is_big(size) && !big_sign(size)))) { + /* If the argument is a positive integer, we must've had a system_limit + * error. */ + c_p->freason = SYSTEM_LIMIT; + } else { + c_p->freason = BADARG; + } +} + +/* Set the exception code for bs_add argument errors after the fact, which is + * much easier and more compact than discriminating within module code. */ +void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B) { + if (((is_small(A) && signed_val(A) >= 0) || (is_big(A) && !big_sign(A))) && + ((is_small(B) && signed_val(B) >= 0) || (is_big(B) && !big_sign(B)))) { + /* If all arguments are positive integers, we must've had a system_limit + * error. */ + c_p->freason = SYSTEM_LIMIT; + } else { + c_p->freason = BADARG; + } +} + +static Eterm i_bs_start_match2_gc_test_preserve(Process *c_p, + Eterm *reg, + Uint need, + Uint live, + Eterm preserve) { + Uint words_left = (Uint)(STACK_TOP(c_p) - HEAP_TOP(c_p)); + + if (ERTS_UNLIKELY(words_left < need + S_RESERVED)) { + reg[live] = preserve; + PROCESS_MAIN_CHK_LOCKS(c_p); + c_p->fcalls -= erts_garbage_collect_nobump(c_p, + need, + reg, + live + 1, + c_p->fcalls); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + preserve = reg[live]; + } + + return preserve; +} + +Eterm beam_jit_bs_start_match2(Eterm context, + Uint live, + Uint slots, + Process *c_p, + Eterm *reg) { + Eterm header; + if (!is_boxed(context)) { + return THE_NON_VALUE; + } + header = *boxed_val(context); + + slots++; + + if (header_is_bin_matchstate(header)) { + ErlBinMatchState *ms = (ErlBinMatchState *)boxed_val(context); + Uint actual_slots = HEADER_NUM_SLOTS(header); + + /* We're not compatible with contexts created by bs_start_match3. */ + ASSERT(actual_slots >= 1); + + ms->save_offset[0] = ms->mb.offset; + if (ERTS_UNLIKELY(actual_slots < slots)) { + ErlBinMatchState *expanded; + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + context = i_bs_start_match2_gc_test_preserve(c_p, + reg, + wordsneeded, + live, + context); + ms = (ErlBinMatchState *)boxed_val(context); + expanded = (ErlBinMatchState *)HEAP_TOP(c_p); + *expanded = *ms; + *HEAP_TOP(c_p) = HEADER_BIN_MATCHSTATE(slots); + HEAP_TOP(c_p) += wordsneeded; + context = make_matchstate(expanded); + } + return context; + } else if (is_binary_header(header)) { + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + context = i_bs_start_match2_gc_test_preserve(c_p, + reg, + wordsneeded, + live, + context); + return erts_bs_start_match_2(c_p, context, slots); + } else { + return THE_NON_VALUE; + } +} + +Eterm beam_jit_bs_init(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Eterm BsOp1, + Eterm BsOp2, + unsigned Live) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin *hb; + Uint bin_need; + + bin_need = heap_bin_size(BsOp1); + erts_bin_offset = 0; + erts_writable_bin = 0; + gc_test(c_p, reg, 0, bin_need + BsOp2 + ERL_SUB_BIN_SIZE, Live); + hb = (ErlHeapBin *)c_p->htop; + c_p->htop += bin_need; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; + erts_current_bin = (byte *)hb->data; + return make_binary(hb); + } else { + Binary *bptr; + ProcBin *pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + test_bin_vheap(c_p, + reg, + BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, + Live); + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(BsOp1); + erts_current_bin = (byte *)bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *)c_p->htop; + c_p->htop += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = BsOp1; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header *)pb; + pb->val = bptr; + pb->bytes = (byte *)bptr->orig_bytes; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); + + return make_binary(pb); + } +} + +Eterm beam_jit_bs_init_bits(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Uint num_bits, + Uint alloc, + unsigned Live) { + Eterm new_binary; + Uint num_bytes = ((Uint64)num_bits + (Uint64)7) >> 3; + + if (num_bits & 7) { + alloc += ERL_SUB_BIN_SIZE; + } + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + alloc += heap_bin_size(num_bytes); + } else { + alloc += PROC_BIN_SIZE; + } + gc_test(c_p, reg, 0, alloc, Live); + + /* num_bits = Number of bits to build + * num_bytes = Number of bytes to allocate in the binary + * alloc = Total number of words to allocate on heap + * Operands: NotUsed NotUsed Dst + */ + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin *hb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + hb = (ErlHeapBin *)c_p->htop; + c_p->htop += heap_bin_size(num_bytes); + hb->thing_word = header_heap_bin(num_bytes); + hb->size = num_bytes; + erts_current_bin = (byte *)hb->data; + new_binary = make_binary(hb); + + do_bits_sub_bin: + if (num_bits & 7) { + ErlSubBin *sb; + + sb = (ErlSubBin *)c_p->htop; + c_p->htop += ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = num_bytes - 1; + sb->bitsize = num_bits & 7; + sb->offs = 0; + sb->bitoffs = 0; + sb->is_writable = 0; + sb->orig = new_binary; + new_binary = make_binary(sb); + } + /* HEAP_SPACE_VERIFIED(0); */ + return new_binary; + } else { + Binary *bptr; + ProcBin *pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(num_bytes); + erts_current_bin = (byte *)bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *)c_p->htop; + c_p->htop += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = num_bytes; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header *)pb; + pb->val = bptr; + pb->bytes = (byte *)bptr->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); + new_binary = make_binary(pb); + goto do_bits_sub_bin; + } +} + +Eterm beam_jit_bs_get_integer(Process *c_p, + Eterm *reg, + Eterm context, + Uint flags, + Uint size, + Uint Live) { + ErlBinMatchBuffer *mb; + + if (size >= SMALL_BITS) { + Uint wordsneeded; + + /* Check bits size before potential gc. + * We do not want a gc and then realize we don't need + * the allocated space (i.e. if the op fails). + * + * Remember to re-acquire the matchbuffer after gc. + */ + mb = ms_matchbuffer(context); + if (mb->size - mb->offset < size) { + return THE_NON_VALUE; + } + + wordsneeded = 1 + WSIZE(NBYTES((Uint)size)); + reg[Live] = context; + gc_test(c_p, reg, 0, wordsneeded, Live + 1); + context = reg[Live]; + } + + mb = ms_matchbuffer(context); + return erts_bs_get_integer_2(c_p, size, flags, mb); +} + +void beam_jit_bs_context_to_binary(Eterm context) { + if (is_boxed(context) && header_is_bin_matchstate(*boxed_val(context))) { + Uint orig, size, offs, hole_size; + ErlBinMatchBuffer *mb; + ErlBinMatchState *ms; + ErlSubBin *sb; + ms = (ErlBinMatchState *)boxed_val(context); + mb = &ms->mb; + offs = ms->save_offset[0]; + size = mb->size - offs; + orig = mb->orig; + sb = (ErlSubBin *)boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); + hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = BYTE_OFFSET(size); + sb->bitsize = BIT_OFFSET(size); + sb->offs = BYTE_OFFSET(offs); + sb->bitoffs = BIT_OFFSET(offs); + sb->is_writable = 0; + sb->orig = orig; + if (hole_size) { + sb[1].thing_word = make_pos_bignum_header(hole_size - 1); + } + } +} + +ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp) { + if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + + /* TODO: Add DTrace probe for this bad message situation? */ + erts_msgq_unlink_msg(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + + return NULL; + } + + return msgp; +} + +/* Remove a (matched) message from the message queue. */ +Sint beam_jit_remove_message(Process *c_p, + Sint FCALLS, + Eterm *HTOP, + Eterm *E, + Uint32 active_code_ix) { + ErtsMessage *msgp; + + ERTS_CHK_MBUF_SZ(c_p); + + if (active_code_ix == ERTS_SAVE_CALLS_CODE_IX) { + save_calls(c_p, &exp_receive); + } + + msgp = erts_msgq_peek_msg(c_p); + + if (ERL_MESSAGE_TOKEN(msgp) == NIL) { +#ifdef USE_VM_PROBES + if (DT_UTAG(c_p) != NIL) { + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { + SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; + } else { + DT_UTAG(c_p) = NIL; + SEQ_TRACE_TOKEN(c_p) = NIL; + } + } else { +#endif + SEQ_TRACE_TOKEN(c_p) = NIL; +#ifdef USE_VM_PROBES + } + DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; +#endif + } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { + Eterm msg; + SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); +#ifdef USE_VM_PROBES + if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { + if (DT_UTAG(c_p) == NIL) { + DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); + } + DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; + } else { +#endif + ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); + ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); + ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); + ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)) || + is_atom(SEQ_TRACE_TOKEN_SENDER(c_p))); + c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + if (c_p->seq_trace_clock < + unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { + c_p->seq_trace_clock = + unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + } + msg = ERL_MESSAGE_TERM(msgp); + seq_trace_output(SEQ_TRACE_TOKEN(c_p), + msg, + SEQ_TRACE_RECEIVE, + c_p->common.id, + c_p); +#ifdef USE_VM_PROBES + } +#endif + } +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_receive)) { + Eterm token2 = NIL; + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Sint len = erts_proc_sig_privqs_len(c_p); + + dtrace_proc_str(c_p, receiver_name); + token2 = SEQ_TRACE_TOKEN(c_p); + if (have_seqtrace(token2)) { + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); + } + DTRACE6(message_receive, + receiver_name, + size_object(ERL_MESSAGE_TERM(msgp)), + len, /* This is NOT message queue len, but its something... */ + tok_label, + tok_lastcnt, + tok_serial); + } +#endif + erts_msgq_unlink_msg(c_p, msgp); + erts_msgq_set_save_first(c_p); + CANCEL_TIMER(c_p); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } + + ERTS_CHK_MBUF_SZ(c_p); + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + return FCALLS; +} + +void beam_jit_take_receive_lock(Process *c_p) { + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); +} + +void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp) { + c_p->arity = 0; + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) { + erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->current = NULL; + c_p->i = cp; +} + +void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp) { + beam_jit_take_receive_lock(c_p); + beam_jit_wait_locked(c_p, cp); +} + +enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p, + Eterm timeout_value, + ErtsCodePtr next) { + /* + * If we have already set the timer, we must NOT set it again. Therefore, + * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. + */ + if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { + if (timeout_value == make_small(0)) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + return RET_next; + } else if (timeout_value == am_infinity) { + c_p->flags |= F_TIMO; + } else { + int tres = erts_set_proc_timer_term(c_p, timeout_value); + if (tres == 0) { + /* + * The timer routiner will set c_p->i to the value in + * c_p->def_arg_reg[0]. Note that it is safe to use this + * location because there are no living x registers in + * a receive statement. + */ + c_p->def_arg_reg[0] = (Eterm)next; + } else { /* Wrong time */ + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->freason = EXC_TIMEOUT_VALUE; + return RET_badarg; + } + } + } + return RET_wait; +} + +void beam_jit_timeout(Process *c_p) { + if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { + trace_receive(c_p, am_clock_service, am_timeout, NULL); + } + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + save_calls(c_p, &exp_timeout); + } + c_p->flags &= ~F_TIMO; + erts_msgq_set_save_first(c_p); +} + +void beam_jit_timeout_locked(Process *c_p) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + beam_jit_timeout(c_p); +} diff --git a/erts/emulator/beam/jit/beam_jit_common.h b/erts/emulator/beam/jit/beam_jit_common.h new file mode 100644 index 0000000000..b2da85d0b8 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.h @@ -0,0 +1,101 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" + +#if defined(DEBUG) && defined(JIT_HARD_DEBUG) +void beam_jit_validate_term(Eterm term); +#endif + +typedef Eterm BeamJitNifF(struct enif_environment_t *, int argc, Eterm argv[]); +enum beam_jit_nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield }; + +Eterm beam_jit_call_bif(Process *c_p, + Eterm *reg, + ErtsCodePtr I, + ErtsBifFunc vbf, + Uint arity); +Eterm beam_jit_call_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg, + BeamJitNifF *fp, + struct erl_module_nif *NifMod); +enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg); + +Uint beam_jit_get_map_elements(Eterm map, + Eterm *reg, + Eterm *E, + Uint n, + Eterm *fs); + +void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size); +void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B); +Eterm beam_jit_bs_start_match2(Eterm context, + Uint live, + Uint slots, + Process *c_p, + Eterm *reg); +Eterm beam_jit_bs_init(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Eterm BsOp1, + Eterm BsOp2, + unsigned Live); +Eterm beam_jit_bs_init_bits(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Uint num_bits, + Uint alloc, + unsigned Live); +Eterm beam_jit_bs_get_integer(Process *c_p, + Eterm *reg, + Eterm context, + Uint flags, + Uint size, + Uint Live); + +ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp); +Sint beam_jit_remove_message(Process *c_p, + Sint FCALLS, + Eterm *HTOP, + Eterm *E, + Uint32 active_code_ix); +void beam_jit_bs_context_to_binary(Eterm context); + +void beam_jit_take_receive_lock(Process *c_p); +void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp); +void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp); + +enum beam_jit_tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 }; + +enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p, + Eterm timeout_value, + ErtsCodePtr next); + +void beam_jit_timeout(Process *c_p); +void beam_jit_timeout_locked(Process *c_p); diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp new file mode 100644 index 0000000000..f8f894c676 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.hpp @@ -0,0 +1,137 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include <string> +#include <vector> +#include <unordered_map> +#include <map> + +extern "C" +{ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" + +#include "beam_jit_common.h" +} + +class ArgVal { + BeamOpArg gen_op; + +public: + enum TYPE { + u = TAG_u, + i = TAG_i, + x = TAG_x, + y = TAG_y, + f = TAG_f, + q = TAG_q, + e = TAG_r, + l = TAG_l /* float register */ + }; + + ArgVal(const BeamOpArg &arg) { + gen_op = arg; + } + + ArgVal(enum TYPE t, BeamInstr val) { + gen_op.type = t; + gen_op.val = val; + } + + ArgVal(unsigned t, BeamInstr val) { +#ifdef DEBUG + switch (t) { + case TAG_u: + break; + case TAG_i: + break; + case TAG_x: + break; + case TAG_y: + break; + case TAG_f: + break; + case TAG_q: + break; + case TAG_r: + break; + case TAG_l: + break; + default: + ASSERT(0); + } +#endif + + gen_op.type = t; + gen_op.val = val; + } + + constexpr enum TYPE getType() const { + return (enum TYPE)gen_op.type; + } + + constexpr uint64_t getValue() const { + return gen_op.val; + } + + constexpr bool isMem() const { + return gen_op.type == x || gen_op.type == y; + } + + constexpr bool isLiteral() const { + return gen_op.type == q; + } + + constexpr bool isImmed() const { + return gen_op.type == i; + } + + template<typename T> + ArgVal operator+(T val) const { + return ArgVal(gen_op.type, val + gen_op.val); + } + + template<typename T> + ArgVal operator*(T val) const { + return ArgVal(gen_op.type, val * gen_op.val); + } + + enum Relation { none, consecutive, reverse_consecutive }; + + static Relation register_relation(const ArgVal &arg1, const ArgVal &arg2) { + TYPE type = arg1.getType(); + bool same_reg_types = + type == arg2.getType() && (type == TYPE::x || type == TYPE::y); + if (!same_reg_types) { + return none; + } else if (arg1.getValue() + 1 == arg2.getValue()) { + return consecutive; + } else if (arg1.getValue() == arg2.getValue() + 1) { + return reverse_consecutive; + } else { + return none; + } + }; +}; diff --git a/erts/emulator/beam/jit/beam_asm.cpp b/erts/emulator/beam/jit/x86/beam_asm.cpp index c4a7ff0c61..4368cc9479 100644 --- a/erts/emulator/beam/jit/beam_asm.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm.cpp @@ -355,12 +355,9 @@ void BeamGlobalAssembler::emit_process_main() { x86::qword_ptr(x86::rsp, offsetof(ErtsSchedulerRegisters, x_reg_array.d))); - load_erl_bits_state(ARG1); - runtime_call<1>(erts_bits_init_state); - #if defined(DEBUG) && defined(NATIVE_ERLANG_STACK) /* Save stack bounds so they can be tested without clobbering anything. */ - runtime_call<0>(erts_get_stacklimit); + a.call(erts_get_stacklimit); a.mov(getSchedulerRegRef( offsetof(ErtsSchedulerRegisters, runtime_stack_end)), @@ -371,7 +368,7 @@ void BeamGlobalAssembler::emit_process_main() { #elif !defined(NATIVE_ERLANG_STACK) /* Save the initial SP of the thread so that we can verify that it * doesn't grow. */ -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG a.mov(getInitialSPRef(), x86::rsp); # endif @@ -386,6 +383,9 @@ void BeamGlobalAssembler::emit_process_main() { a.and_(x86::rsp, imm(-16)); #endif + load_erl_bits_state(ARG1); + runtime_call<1>(erts_bits_init_state); + a.mov(start_time_i, imm(0)); a.mov(start_time, imm(0)); diff --git a/erts/emulator/beam/jit/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp index f97aaee840..9e1254fec1 100644 --- a/erts/emulator/beam/jit/beam_asm.hpp +++ b/erts/emulator/beam/jit/x86/beam_asm.hpp @@ -41,105 +41,7 @@ extern "C" #include "beam_asm.h" } -class ArgVal { - BeamOpArg gen_op; - -public: - enum TYPE { - u = TAG_u, - i = TAG_i, - x = TAG_x, - y = TAG_y, - f = TAG_f, - q = TAG_q, - e = TAG_r, - l = TAG_l /* float register */ - }; - - ArgVal(const BeamOpArg &arg) { - gen_op = arg; - } - - ArgVal(enum TYPE t, BeamInstr val) { - gen_op.type = t; - gen_op.val = val; - } - - ArgVal(unsigned t, BeamInstr val) { -#ifdef DEBUG - switch (t) { - case TAG_u: - break; - case TAG_i: - break; - case TAG_x: - break; - case TAG_y: - break; - case TAG_f: - break; - case TAG_q: - break; - case TAG_r: - break; - case TAG_l: - break; - default: - ASSERT(0); - } -#endif - - gen_op.type = t; - gen_op.val = val; - } - - constexpr enum TYPE getType() const { - return (enum TYPE)gen_op.type; - } - - constexpr uint64_t getValue() const { - return gen_op.val; - } - - constexpr bool isMem() const { - return gen_op.type == x || gen_op.type == y; - } - - constexpr bool isLiteral() const { - return gen_op.type == q; - } - - constexpr bool isImmed() const { - return gen_op.type == i; - } - - template<typename T> - ArgVal operator+(T val) const { - return ArgVal(gen_op.type, val + gen_op.val); - } - - template<typename T> - ArgVal operator*(T val) const { - return ArgVal(gen_op.type, val * gen_op.val); - } - - enum Relation { none, consecutive, reverse_consecutive }; - - static Relation register_relation(const ArgVal &arg1, const ArgVal &arg2) { - TYPE type = arg1.getType(); - bool same_reg_types = - type == arg2.getType() && (type == TYPE::x || type == TYPE::y); - if (!same_reg_types) { - return none; - } else if (arg1.getValue() + 1 == arg2.getValue()) { - return consecutive; - } else if (arg1.getValue() == arg2.getValue() + 1) { - return reverse_consecutive; - } else { - return none; - } - }; -}; +#include "beam_jit_common.hpp" using namespace asmjit; @@ -285,7 +187,7 @@ public: } BeamAssembler(const std::string &log) : BeamAssembler() { - if (erts_asm_dump) { + if (erts_jit_asm_dump) { setLogger(log + ".asm"); } } @@ -366,9 +268,9 @@ protected: } #if !defined(NATIVE_ERLANG_STACK) -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG constexpr x86::Mem getInitialSPRef() const { - int base = offsetof(ErtsSchedulerRegisters, aux_regs.d.initial_sp); + int base = offsetof(ErtsSchedulerRegisters, initial_sp); return getSchedulerRegRef(base); } @@ -446,21 +348,27 @@ protected: a.lea(reg, getSchedulerRegRef(offset)); } + /* Ensure that the Erlang stack is used and the redzone is unused. + * We combine those test to minimize the number of instructions. + */ void emit_assert_redzone_unused() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG const int REDZONE_BYTES = S_REDZONE * sizeof(Eterm); - Label next = a.newLabel(); + Label ok = a.newLabel(), crash = a.newLabel(); /* We modify the stack pointer to avoid spilling into a register, * TMP_MEM, or using the stack. */ a.sub(E, imm(REDZONE_BYTES)); a.cmp(HTOP, E); - a.add(E, imm(REDZONE_BYTES)); + a.short_().ja(crash); + a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, hend))); + a.short_().jle(ok); - a.jbe(next); + a.bind(crash); a.ud2(); - a.bind(next); + a.bind(ok); + a.add(E, imm(REDZONE_BYTES)); #endif } @@ -472,7 +380,6 @@ protected: #ifdef NATIVE_ERLANG_STACK /* We use the Erlang stack as the native stack. We can use a * native `call` instruction. */ - emit_assert_erlang_stack(); emit_assert_redzone_unused(); aligned_call(Target); #else @@ -498,10 +405,9 @@ protected: */ template<typename Any> void fragment_call(Any Target) { - emit_assert_erlang_stack(); emit_assert_redzone_unused(); -#if defined(HARD_DEBUG) && !defined(NATIVE_ERLANG_STACK) +#if defined(JIT_HARD_DEBUG) && !defined(NATIVE_ERLANG_STACK) /* Verify that the stack has not grown. */ Label next = a.newLabel(); a.cmp(x86::rsp, getInitialSPRef()); @@ -515,7 +421,7 @@ protected: /* * Calls the given function pointer. In a debug build with - * HARD_DEBUG defined, it will be enforced that the redzone is + * JIT_HARD_DEBUG defined, it will be enforced that the redzone is * unused. * * The return will NOT be aligned, and thus will not form a valid @@ -525,7 +431,6 @@ protected: * switch. */ void safe_fragment_call(void (*Target)()) { - emit_assert_erlang_stack(); emit_assert_redzone_unused(); a.call(imm(Target)); } @@ -574,7 +479,7 @@ protected: a.embed(nops[nop_count - 1], nop_count); } -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG /* TODO: When frame pointers are in place, assert (at runtime) that the * destination has a `push rbp; mov rbp, rsp` sequence. */ #endif @@ -697,13 +602,9 @@ protected: } void emit_assert_runtime_stack() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG Label crash = a.newLabel(), next = a.newLabel(); - /* Are we 16-byte aligned? */ - a.test(E, (16 - 1)); - a.jne(crash); - # ifdef NATIVE_ERLANG_STACK /* Ensure that we are using the runtime stack. */ int end_offs, start_offs; @@ -712,32 +613,37 @@ protected: start_offs = offsetof(ErtsSchedulerRegisters, runtime_stack_start); a.cmp(E, getSchedulerRegRef(end_offs)); - a.short_().jl(crash); + a.short_().jbe(crash); a.cmp(E, getSchedulerRegRef(start_offs)); - a.short_().jle(next); - + a.short_().ja(crash); # endif + + /* Are we 16-byte aligned? */ + a.test(x86::rsp, (16 - 1)); + a.short_().je(next); + a.bind(crash); a.ud2(); + a.bind(next); #endif } void emit_assert_erlang_stack() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG Label crash = a.newLabel(), next = a.newLabel(); /* Are we term-aligned? */ a.test(E, imm(sizeof(Eterm) - 1)); - a.jne(crash); + a.short_().jne(crash); a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, heap))); - a.jl(crash); + a.short_().jl(crash); a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, hend))); - a.jle(next); + a.short_().jle(next); a.bind(crash); - a.hlt(); + a.ud2(); a.bind(next); #endif } diff --git a/erts/emulator/beam/jit/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp index 1c6479186a..1c6479186a 100644 --- a/erts/emulator/beam/jit/beam_asm_global.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp diff --git a/erts/emulator/beam/jit/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index 0da710d878..0da710d878 100644 --- a/erts/emulator/beam/jit/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp diff --git a/erts/emulator/beam/jit/beam_asm_perf.cpp b/erts/emulator/beam/jit/x86/beam_asm_perf.cpp index 79c0977339..5f7c274195 100644 --- a/erts/emulator/beam/jit/beam_asm_perf.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_perf.cpp @@ -71,24 +71,6 @@ class JitPerfDump { Uint64 timestamp; }; - struct JitCodeLoadRecord { - RecordHeader header; - Uint32 pid; - Uint32 tid; - Uint64 vma; - Uint64 code_addr; - Uint64 code_size; - Uint64 code_index; - /* Null terminated M:F/A */ - /* Native code */ - - JitCodeLoadRecord() { - header.id = JIT_CODE_LOAD; - pid = getpid(); - tid = erts_thr_self(); - } - }; - public: bool init() { char name[MAXPATHLEN]; @@ -137,10 +119,25 @@ public: void update_perf_info(std::string modulename, std::vector<BeamAssembler::AsmRange> &ranges) { + struct JitCodeLoadRecord { + RecordHeader header; + Uint32 pid; + Uint32 tid; + Uint64 vma; + Uint64 code_addr; + Uint64 code_size; + Uint64 code_index; + /* Null terminated M:F/A */ + /* Native code */ + }; JitCodeLoadRecord record; + record.header.id = JIT_CODE_LOAD; + record.pid = getpid(); + record.tid = erts_thr_self(); for (BeamAssembler::AsmRange &r : ranges) { size_t nameLen = r.name.size(); ptrdiff_t codeSize = (char *)r.stop - (char *)r.start; + ASSERT(codeSize > 0); record.header.total_size = sizeof(record) + nameLen + 1 + codeSize; record.vma = (Uint64)r.start; record.code_addr = (Uint64)r.start; @@ -184,6 +181,7 @@ public: ptrdiff_t size = stop - start; fprintf(file, "%p %tx $%s\n", start, size, r.name.c_str()); } + fflush(file); } }; diff --git a/erts/emulator/beam/jit/generators.tab b/erts/emulator/beam/jit/x86/generators.tab index ab755fde4c..ab755fde4c 100644 --- a/erts/emulator/beam/jit/generators.tab +++ b/erts/emulator/beam/jit/x86/generators.tab diff --git a/erts/emulator/beam/jit/instr_arith.cpp b/erts/emulator/beam/jit/x86/instr_arith.cpp index 5615230551..5615230551 100644 --- a/erts/emulator/beam/jit/instr_arith.cpp +++ b/erts/emulator/beam/jit/x86/instr_arith.cpp diff --git a/erts/emulator/beam/jit/instr_bif.cpp b/erts/emulator/beam/jit/x86/instr_bif.cpp index 456b837099..ae329e3f0c 100644 --- a/erts/emulator/beam/jit/instr_bif.cpp +++ b/erts/emulator/beam/jit/x86/instr_bif.cpp @@ -439,6 +439,33 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { runtime_call(RET, 3); #endif +#ifdef ERTS_MSACC_EXTENDED_STATES + { + Label skip_msacc = a.newLabel(); + + a.cmp(erts_msacc_cache, imm(0)); + a.short_().je(skip_msacc); + + /* update cache if it was changed in the bif. + TMP_MEM1q is already taken to save ARG1 above */ + a.mov(TMP_MEM2q, RET); + a.lea(ARG1, erts_msacc_cache); + runtime_call<1>(erts_msacc_update_cache); + a.mov(RET, TMP_MEM2q); + + /* set state to emulator if msacc has been enabled */ + a.cmp(erts_msacc_cache, imm(0)); + a.short_().je(skip_msacc); + a.mov(ARG1, erts_msacc_cache); + a.mov(ARG2, imm(ERTS_MSACC_STATE_EMULATOR)); + a.mov(ARG3, imm(1)); + runtime_call<3>(erts_msacc_set_state_m__); + a.mov(RET, TMP_MEM2q); + + a.bind(skip_msacc); + } +#endif + /* ERTS_IS_GC_DESIRED_INTERNAL */ { a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, stop))); @@ -474,7 +501,8 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { a.bind(check_bif_return); emit_test_the_non_value(ARG3); - a.short_().je(trap); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.je(trap); a.mov(HTOP, ARG5); #ifdef NATIVE_ERLANG_STACK @@ -639,99 +667,6 @@ void BeamModuleAssembler::emit_send() { fragment_call(ga->get_call_light_bif_shared()); } -static Eterm call_bif(Process *c_p, - Eterm *reg, - ErtsCodePtr I, - ErtsBifFunc vbf, - Uint arity) { - ErlHeapFragment *live_hf_end; - Eterm result; - - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - { - live_hf_end = c_p->mbuf; - - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = vbf(c_p, reg, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_HOLE_CHECK(c_p); - } - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - - if (ERTS_IS_GC_DESIRED(c_p)) { - result = erts_gc_after_bif_call_lhf(c_p, - live_hf_end, - result, - reg, - arity); - } - - return result; -} - -typedef Eterm NifF(struct enif_environment_t *, int argc, Eterm argv[]); - -static Eterm call_nif(Process *c_p, - ErtsCodePtr I, - Eterm *reg, - NifF *fp, - struct erl_module_nif *NifMod) { - Eterm nif_bif_result; - Eterm bif_nif_arity; - ErlHeapFragment *live_hf_end; - const ErtsCodeMFA *codemfa; - - codemfa = erts_code_to_codemfa(I); - - c_p->current = codemfa; /* current and vbf set to please handle_error */ - - bif_nif_arity = codemfa->arity; - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - - { - struct enif_environment_t env; - ASSERT(c_p->scheduler_data); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, NifMod, NULL); - - ASSERT((c_p->scheduler_data)->current_nif == NULL); - (c_p->scheduler_data)->current_nif = &env; - - nif_bif_result = (*fp)(&env, bif_nif_arity, reg); - if (env.exception_thrown) - nif_bif_result = THE_NON_VALUE; - - ASSERT((c_p->scheduler_data)->current_nif == &env); - (c_p->scheduler_data)->current_nif = NULL; - - erts_post_nif(&env); - ERTS_CHK_MBUF_SZ(c_p); - - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ASSERT(!env.exiting); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } - ERTS_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - - if (ERTS_IS_GC_DESIRED(c_p)) { - nif_bif_result = erts_gc_after_bif_call_lhf(c_p, - live_hf_end, - nif_bif_result, - reg, - bif_nif_arity); - } - - return nif_bif_result; -} - void BeamGlobalAssembler::emit_bif_nif_epilogue(void) { Label check_trap = a.newLabel(), trap = a.newLabel(), error = a.newLabel(); @@ -862,7 +797,7 @@ void BeamGlobalAssembler::emit_call_bif_shared(void) { a.mov(ARG1, c_p); load_x_reg_array(ARG2); /* ARG3 (I), ARG4 (func), and ARG5 (arity) have already been provided. */ - runtime_call<5>(call_bif); + runtime_call<5>(beam_jit_call_bif); #ifdef ERTS_MSACC_EXTENDED_STATES a.mov(TMP_MEM1q, RET); @@ -986,7 +921,7 @@ void BeamGlobalAssembler::emit_call_nif_shared(void) { a.mov(ARG4, x86::qword_ptr(ARG2, 8 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); a.mov(ARG5, x86::qword_ptr(ARG2, 16 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); a.mov(ARG6, x86::qword_ptr(ARG2, 24 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); - runtime_call<5>(call_nif); + runtime_call<5>(beam_jit_call_nif); emit_bif_nif_epilogue(); } @@ -1042,33 +977,6 @@ void BeamModuleAssembler::emit_call_nif(const ArgVal &Func, } } -enum nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield }; - -static enum nif_load_ret load_nif(Process *c_p, ErtsCodePtr I, Eterm *reg) { - if (erts_try_seize_code_write_permission(c_p)) { - Eterm result; - - PROCESS_MAIN_CHK_LOCKS((c_p)); - ERTS_UNREQ_PROC_MAIN_LOCK((c_p)); - result = erts_load_nif(c_p, I, reg[0], reg[1]); - erts_release_code_write_permission(); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - - if (ERTS_LIKELY(is_value(result))) { - reg[0] = result; - return RET_NIF_success; - } else { - c_p->freason = BADARG; - return RET_NIF_error; - } - } else { - /* Yield and try again. */ - c_p->current = NULL; - c_p->arity = 2; - return RET_NIF_yield; - } -} - /* ARG2 = entry address. */ void BeamGlobalAssembler::emit_i_load_nif_shared() { static ErtsCodeMFA bif_mfa = {am_erlang, am_load_nif, 2}; @@ -1082,7 +990,7 @@ void BeamGlobalAssembler::emit_i_load_nif_shared() { a.mov(ARG1, c_p); /* ARG2 has already been set by caller */ load_x_reg_array(ARG3); - runtime_call<3>(load_nif); + runtime_call<3>(beam_jit_load_nif); emit_leave_runtime<Update::eStack | Update::eHeap>(); @@ -1142,7 +1050,7 @@ void BeamModuleAssembler::emit_i_load_nif() { a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(currLabel)); load_x_reg_array(ARG3); - runtime_call<3>(load_nif); + runtime_call<3>(beam_jit_load_nif); emit_leave_runtime<Update::eStack | Update::eHeap>(); diff --git a/erts/emulator/beam/jit/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index 939b6c6b9f..7df0b84cae 100644 --- a/erts/emulator/beam/jit/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -95,82 +95,6 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgVal &Size, } } -void TEST_BIN_VHEAP(Process *c_p, Eterm *reg, Uint VNh, Uint Nh, Uint Live) { - int need = Nh; - - if (c_p->stop - c_p->htop < (need + S_RESERVED) || - MSO(c_p).overhead + VNh >= BIN_VHEAP_SZ(c_p)) { - c_p->fcalls -= - erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); - } -} - -void GC_TEST(Process *c_p, Eterm *reg, Uint Ns, Uint Nh, Uint Live) { - int need = Nh + Ns; - - if (ERTS_UNLIKELY(c_p->stop - c_p->htop < (need + S_RESERVED))) { - c_p->fcalls -= - erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); - } -} - -Eterm i_bs_init(Process *c_p, - Eterm *reg, - ERL_BITS_DECLARE_STATEP, - Eterm BsOp1, - Eterm BsOp2, - unsigned Live) { - if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin *hb; - Uint bin_need; - - bin_need = heap_bin_size(BsOp1); - erts_bin_offset = 0; - erts_writable_bin = 0; - GC_TEST(c_p, reg, 0, bin_need + BsOp2 + ERL_SUB_BIN_SIZE, Live); - hb = (ErlHeapBin *)c_p->htop; - c_p->htop += bin_need; - hb->thing_word = header_heap_bin(BsOp1); - hb->size = BsOp1; - erts_current_bin = (byte *)hb->data; - return make_binary(hb); - } else { - Binary *bptr; - ProcBin *pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - TEST_BIN_VHEAP(c_p, - reg, - BsOp1 / sizeof(Eterm), - BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, - Live); - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(BsOp1); - erts_current_bin = (byte *)bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *)c_p->htop; - c_p->htop += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = BsOp1; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header *)pb; - pb->val = bptr; - pb->bytes = (byte *)bptr->orig_bytes; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); - - return make_binary(pb); - } -} - void BeamModuleAssembler::emit_i_bs_init_heap(const ArgVal &Size, const ArgVal &Heap, const ArgVal &Live, @@ -185,29 +109,18 @@ void BeamModuleAssembler::emit_i_bs_init_heap(const ArgVal &Size, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init); + runtime_call<6>(beam_jit_bs_init); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); mov_arg(Dst, RET); } -static void bs_field_size_argument_error(Process *c_p, Eterm size) { - if (((is_small(size) && signed_val(size) >= 0) || - (is_big(size) && !big_sign(size)))) { - /* If the argument is a positive integer, we must've had a system_limit - * error. */ - c_p->freason = SYSTEM_LIMIT; - } else { - c_p->freason = BADARG; - } -} - /* Set the error reason when a size check has failed. */ void BeamGlobalAssembler::emit_bs_size_check_shared() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<2>(bs_field_size_argument_error); + runtime_call<2>(beam_jit_bs_field_size_argument_error); emit_leave_runtime(); a.ret(); } @@ -236,7 +149,7 @@ void BeamModuleAssembler::emit_i_bs_init_fail_heap(const ArgVal &Size, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init); + runtime_call<6>(beam_jit_bs_init); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -276,90 +189,6 @@ void BeamModuleAssembler::emit_i_bs_init_fail(const ArgVal &Size, emit_i_bs_init_fail_heap(Size, Heap, Fail, Live, Dst); } -static Eterm i_bs_init_bits(Process *c_p, - Eterm *reg, - ERL_BITS_DECLARE_STATEP, - Uint num_bits, - Uint alloc, - unsigned Live) { - Eterm new_binary; - Uint num_bytes = ((Uint64)num_bits + (Uint64)7) >> 3; - - if (num_bits & 7) { - alloc += ERL_SUB_BIN_SIZE; - } - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - alloc += heap_bin_size(num_bytes); - } else { - alloc += PROC_BIN_SIZE; - } - GC_TEST(c_p, reg, 0, alloc, Live); - - /* num_bits = Number of bits to build - * num_bytes = Number of bytes to allocate in the binary - * alloc = Total number of words to allocate on heap - * Operands: NotUsed NotUsed Dst - */ - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin *hb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - hb = (ErlHeapBin *)c_p->htop; - c_p->htop += heap_bin_size(num_bytes); - hb->thing_word = header_heap_bin(num_bytes); - hb->size = num_bytes; - erts_current_bin = (byte *)hb->data; - new_binary = make_binary(hb); - - do_bits_sub_bin: - if (num_bits & 7) { - ErlSubBin *sb; - - sb = (ErlSubBin *)c_p->htop; - c_p->htop += ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = num_bytes - 1; - sb->bitsize = num_bits & 7; - sb->offs = 0; - sb->bitoffs = 0; - sb->is_writable = 0; - sb->orig = new_binary; - new_binary = make_binary(sb); - } - /* HEAP_SPACE_VERIFIED(0); */ - return new_binary; - } else { - Binary *bptr; - ProcBin *pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(num_bytes); - erts_current_bin = (byte *)bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *)c_p->htop; - c_p->htop += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = num_bytes; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header *)pb; - pb->val = bptr; - pb->bytes = (byte *)bptr->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); - new_binary = make_binary(pb); - goto do_bits_sub_bin; - } -} - void BeamModuleAssembler::emit_i_bs_init_bits(const ArgVal &NumBits, const ArgVal &Live, const ArgVal &Dst) { @@ -381,7 +210,7 @@ void BeamModuleAssembler::emit_i_bs_init_bits_heap(const ArgVal &NumBits, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init_bits); + runtime_call<6>(beam_jit_bs_init_bits); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -422,7 +251,7 @@ void BeamModuleAssembler::emit_i_bs_init_bits_fail_heap(const ArgVal &NumBits, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init_bits); + runtime_call<6>(beam_jit_bs_init_bits); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -728,7 +557,11 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src, a.and_(RETb, imm(_HEADER_SUBTAG_MASK)); a.cmp(RETb, imm(BIN_MATCHSTATE_SUBTAG)); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif if (Fail.getValue() != 0) { comment("is_binary_header"); @@ -818,38 +651,6 @@ void BeamModuleAssembler::emit_i_bs_get_position(const ArgVal &Ctx, mov_arg(Dst, ARG1); } -static Eterm i_bs_get_integer(Process *c_p, - Eterm *reg, - Eterm context, - Uint flags, - Uint size, - Uint Live) { - ErlBinMatchBuffer *mb; - - if (size >= SMALL_BITS) { - Uint wordsneeded; - - /* Check bits size before potential gc. - * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails). - * - * Remember to re-acquire the matchbuffer after gc. - */ - mb = ms_matchbuffer(context); - if (mb->size - mb->offset < size) { - return THE_NON_VALUE; - } - - wordsneeded = 1 + WSIZE(NBYTES((Uint)size)); - reg[Live] = context; - GC_TEST(c_p, reg, 0, wordsneeded, Live + 1); - context = reg[Live]; - } - - mb = ms_matchbuffer(context); - return erts_bs_get_integer_2(c_p, size, flags, mb); -} - /* ARG3 = flags | (size << 3), * ARG4 = tagged match context */ void BeamGlobalAssembler::emit_bs_fixed_integer_shared() { @@ -890,7 +691,11 @@ x86::Mem BeamModuleAssembler::emit_bs_get_integer_prologue(Label next, /* The above call can't fail since we work on small numbers and * bounds-tested above. */ +#ifdef JIT_HARD_DEBUG + a.jmp(next); +#else a.short_().jmp(next); +#endif a.bind(aligned); { @@ -1108,7 +913,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer(const ArgVal &Ctx, a.mov(ARG1, c_p); load_x_reg_array(ARG2); - runtime_call<6>(i_bs_get_integer); + runtime_call<6>(beam_jit_bs_get_integer); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -1569,24 +1374,11 @@ void BeamModuleAssembler::emit_bs_test_unit(const ArgVal &Fail, a.jnz(labels[Fail.getValue()]); } -/* Set the exception code for bs_add argument errors after the fact, which is - * much easier and more compact than discriminating within module code. */ -static void bs_add_argument_error(Process *c_p, Eterm A, Eterm B) { - if (((is_small(A) && signed_val(A) >= 0) || (is_big(A) && !big_sign(A))) && - ((is_small(B) && signed_val(B) >= 0) || (is_big(B) && !big_sign(B)))) { - /* If all arguments are positive integers, we must've had a system_limit - * error. */ - c_p->freason = SYSTEM_LIMIT; - } else { - c_p->freason = BADARG; - } -} - /* Set the error reason when bs_add has failed. */ void BeamGlobalAssembler::emit_bs_add_shared() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<3>(bs_add_argument_error); + runtime_call<3>(beam_jit_bs_add_argument_error); emit_leave_runtime(); a.ret(); } @@ -1762,79 +1554,6 @@ void BeamModuleAssembler::emit_bs_init_writable() { emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); } -static Eterm i_bs_start_match2_gc_test_preserve(Process *c_p, - Eterm *reg, - Uint need, - Uint live, - Eterm preserve) { - Uint words_left = (Uint)(STACK_TOP(c_p) - HEAP_TOP(c_p)); - - if (ERTS_UNLIKELY(words_left < need + S_RESERVED)) { - reg[live] = preserve; - PROCESS_MAIN_CHK_LOCKS(c_p); - c_p->fcalls -= erts_garbage_collect_nobump(c_p, - need, - reg, - live + 1, - c_p->fcalls); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - preserve = reg[live]; - } - - return preserve; -} - -static Eterm i_bs_start_match2(Eterm context, - Uint live, - Uint slots, - Process *c_p, - Eterm *reg) { - Eterm header; - if (!is_boxed(context)) { - return THE_NON_VALUE; - } - header = *boxed_val(context); - - slots++; - - if (header_is_bin_matchstate(header)) { - ErlBinMatchState *ms = (ErlBinMatchState *)boxed_val(context); - Uint actual_slots = HEADER_NUM_SLOTS(header); - - /* We're not compatible with contexts created by bs_start_match3. */ - ASSERT(actual_slots >= 1); - - ms->save_offset[0] = ms->mb.offset; - if (ERTS_UNLIKELY(actual_slots < slots)) { - ErlBinMatchState *expanded; - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - context = i_bs_start_match2_gc_test_preserve(c_p, - reg, - wordsneeded, - live, - context); - ms = (ErlBinMatchState *)boxed_val(context); - expanded = (ErlBinMatchState *)HEAP_TOP(c_p); - *expanded = *ms; - *HEAP_TOP(c_p) = HEADER_BIN_MATCHSTATE(slots); - HEAP_TOP(c_p) += wordsneeded; - context = make_matchstate(expanded); - } - return context; - } else if (is_binary_header(header)) { - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - context = i_bs_start_match2_gc_test_preserve(c_p, - reg, - wordsneeded, - live, - context); - return erts_bs_start_match_2(c_p, context, slots); - } else { - return THE_NON_VALUE; - } -} - /* Old compatibility instructions for <= OTP-21. Kept in order to be able to * load old code. While technically we could remove these in OTP-24, we've * decided to keep them until at least OTP-25 to make things easier for @@ -1852,7 +1571,7 @@ void BeamModuleAssembler::emit_i_bs_start_match2(const ArgVal &Src, a.mov(ARG4, c_p); load_x_reg_array(ARG5); - runtime_call<5>(i_bs_start_match2); + runtime_call<5>(beam_jit_bs_start_match2); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -1883,41 +1602,12 @@ void BeamModuleAssembler::emit_i_bs_restore2(const ArgVal &Ctx, a.mov(emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.offset)), ARG2); } -static void bs_context_to_binary(Eterm context) { - if (is_boxed(context) && header_is_bin_matchstate(*boxed_val(context))) { - Uint orig, size, offs, hole_size; - ErlBinMatchBuffer *mb; - ErlBinMatchState *ms; - ErlSubBin *sb; - ms = (ErlBinMatchState *)boxed_val(context); - mb = &ms->mb; - offs = ms->save_offset[0]; - size = mb->size - offs; - orig = mb->orig; - sb = (ErlSubBin *)boxed_val(context); - /* Since we're going to overwrite the match state with the result, an - * ErlBinMatchState must be at least as large as an ErlSubBin. */ - ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); - hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(size); - sb->bitsize = BIT_OFFSET(size); - sb->offs = BYTE_OFFSET(offs); - sb->bitoffs = BIT_OFFSET(offs); - sb->is_writable = 0; - sb->orig = orig; - if (hole_size) { - sb[1].thing_word = make_pos_bignum_header(hole_size - 1); - } - } -} - void BeamModuleAssembler::emit_bs_context_to_binary(const ArgVal &Src) { mov_arg(ARG1, Src); emit_enter_runtime(); - runtime_call<1>(bs_context_to_binary); + runtime_call<1>(beam_jit_bs_context_to_binary); emit_leave_runtime(); } diff --git a/erts/emulator/beam/jit/instr_call.cpp b/erts/emulator/beam/jit/x86/instr_call.cpp index 51b09edaf9..b79ef5567c 100644 --- a/erts/emulator/beam/jit/instr_call.cpp +++ b/erts/emulator/beam/jit/x86/instr_call.cpp @@ -41,7 +41,7 @@ void BeamGlobalAssembler::emit_dispatch_return() { void BeamModuleAssembler::emit_return() { Label dispatch_return = a.newLabel(); -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG /* Validate return address and {x,0} */ emit_validate(ArgVal(ArgVal::u, 1)); #endif @@ -236,6 +236,8 @@ x86::Gp BeamModuleAssembler::emit_apply_fun() { a.lea(ARG5, TMP_MEM1q); runtime_call<5>(apply_fun); + emit_leave_runtime<Update::eStack | Update::eHeap>(); + a.mov(ARG2, RET); a.test(RET, RET); @@ -244,8 +246,6 @@ x86::Gp BeamModuleAssembler::emit_apply_fun() { * `emit_setup_export_call` for details. */ a.mov(RET, TMP_MEM1q); - emit_leave_runtime<Update::eStack | Update::eHeap>(); - a.short_().jne(dispatch); emit_handle_error(); a.bind(dispatch); @@ -284,6 +284,8 @@ x86::Gp BeamModuleAssembler::emit_call_fun(const ArgVal &Fun) { a.lea(ARG5, TMP_MEM1q); runtime_call<5>(call_fun); + emit_leave_runtime<Update::eStack | Update::eHeap>(); + a.mov(ARG2, RET); a.test(RET, RET); @@ -292,8 +294,6 @@ x86::Gp BeamModuleAssembler::emit_call_fun(const ArgVal &Fun) { * `emit_setup_export_call` for details. */ a.mov(RET, TMP_MEM1q); - emit_leave_runtime<Update::eStack | Update::eHeap>(); - a.short_().jne(dispatch); emit_handle_error(); a.bind(dispatch); diff --git a/erts/emulator/beam/jit/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp index 4c278d7a5a..4ca7a10629 100644 --- a/erts/emulator/beam/jit/instr_common.cpp +++ b/erts/emulator/beam/jit/x86/instr_common.cpp @@ -133,20 +133,6 @@ void BeamModuleAssembler::emit_gc_test(const ArgVal &Ns, a.bind(after_gc_check); } -#if defined(DEBUG) && defined(HARD_DEBUG) -static void validate_term(Eterm term) { - if (is_boxed(term)) { - Eterm header = *boxed_val(term); - - if (header_is_bin_matchstate(header)) { - return; - } - } - - size_object_x(term, nullptr); -} -#endif - void BeamModuleAssembler::emit_validate(const ArgVal &arity) { #ifdef DEBUG Label next = a.newLabel(), crash = a.newLabel(); @@ -169,16 +155,17 @@ void BeamModuleAssembler::emit_validate(const ArgVal &arity) { a.hlt(); a.bind(next); -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG emit_enter_runtime(); for (unsigned i = 0; i < arity.getValue(); i++) { a.mov(ARG1, getXRef(i)); - runtime_call<1>(validate_term); + runtime_call<1>(beam_jit_validate_term); } emit_leave_runtime(); # endif + #endif } @@ -1220,7 +1207,11 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, mov_arg(ARG1, X); a.cmp(ARG1, ARG2); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif /* Fancy way of checking if both are immediates. */ a.mov(RETd, ARG1d); @@ -1279,7 +1270,11 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, a.and_(RETd, ARG2d); a.and_(RETb, imm(_TAG_PRIMARY_MASK)); a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif emit_enter_runtime(); @@ -1437,7 +1432,9 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { a.sub(ARG3d, imm(_TAG_HEADER_FLOAT)); a.sub(ARG5d, imm(_TAG_HEADER_FLOAT)); a.or_(ARG3d, ARG5d); - a.short_().jne(generic_compare); + + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(generic_compare); boxed_ptr = emit_ptr_val(ARG1, ARG1); a.movsd(x86::xmm0, emit_boxed_val(boxed_ptr, sizeof(Eterm))); @@ -1660,7 +1657,8 @@ void BeamGlobalAssembler::emit_catch_end_shared() { a.bind(not_throw); { a.cmp(getXRef(1), imm(am_error)); - a.short_().jne(not_error); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(not_error); /* This is an error, attach a stacktrace to the reason. */ emit_enter_runtime<Update::eStack | Update::eHeap>(); diff --git a/erts/emulator/beam/jit/instr_float.cpp b/erts/emulator/beam/jit/x86/instr_float.cpp index 43acf485ff..43acf485ff 100644 --- a/erts/emulator/beam/jit/instr_float.cpp +++ b/erts/emulator/beam/jit/x86/instr_float.cpp diff --git a/erts/emulator/beam/jit/instr_guard_bifs.cpp b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp index 1084a1c0e9..1084a1c0e9 100644 --- a/erts/emulator/beam/jit/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp diff --git a/erts/emulator/beam/jit/instr_map.cpp b/erts/emulator/beam/jit/x86/instr_map.cpp index c3444c9755..a1fb7394d9 100644 --- a/erts/emulator/beam/jit/instr_map.cpp +++ b/erts/emulator/beam/jit/x86/instr_map.cpp @@ -134,71 +134,6 @@ void BeamModuleAssembler::emit_i_get_map_element(const ArgVal &Fail, } } -static Uint i_get_map_elements(Eterm map, - Eterm *reg, - Eterm *E, - Uint n, - Eterm *fs) { - Uint sz; - - /* This instruction assumes Arg1 is a map, i.e. that it follows a test - * is_map if needed. */ - - if (is_flatmap(map)) { - flatmap_t *mp; - Eterm *ks; - Eterm *vs; - - mp = (flatmap_t *)flatmap_val(map); - sz = flatmap_get_size(mp); - - if (sz == 0) { - return 0; - } - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - - while (sz) { - if (EQ(fs[0], *ks)) { - PUT_TERM_REG(*vs, fs[1]); - - n--; - fs += 3; - - /* no more values to fetch, we are done */ - if (n == 0) { - return 1; - } - } - - ks++, sz--, vs++; - } - return 0; - } else { - ASSERT(is_hashmap(map)); - - while (n--) { - const Eterm *v; - Uint32 hx; - - hx = fs[2]; - ASSERT(hx == hashmap_make_hash(fs[0])); - - if ((v = erts_hashmap_get(hx, fs[0], map)) == NULL) { - return 0; - } - - PUT_TERM_REG(*v, fs[1]); - fs += 3; - } - - return 1; - } -} - -#undef PUT_TERM_REG - void BeamModuleAssembler::emit_i_get_map_elements( const ArgVal &Fail, const ArgVal &Src, @@ -216,7 +151,7 @@ void BeamModuleAssembler::emit_i_get_map_elements( mov_imm(ARG4, args.size() / 3); a.lea(ARG5, x86::qword_ptr(data)); load_x_reg_array(ARG2); - runtime_call<5>(i_get_map_elements); + runtime_call<5>(beam_jit_get_map_elements); emit_leave_runtime(); diff --git a/erts/emulator/beam/jit/instr_msg.cpp b/erts/emulator/beam/jit/x86/instr_msg.cpp index a929fec93f..e9110e174b 100644 --- a/erts/emulator/beam/jit/instr_msg.cpp +++ b/erts/emulator/beam/jit/x86/instr_msg.cpp @@ -30,24 +30,6 @@ extern "C" #endif } -static ErtsMessage *decode_dist(Process *c_p, ErtsMessage *msgp) { - if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - - /* TODO: Add DTrace probe for this bad message situation? */ - erts_msgq_unlink_msg(c_p, msgp); - msgp->next = NULL; - erts_cleanup_messages(msgp); - - return nullptr; - } - - return msgp; -} - #ifdef ERTS_SUPPORT_OLD_RECV_MARK_INSTRS static void recv_mark(Process *p) { @@ -219,7 +201,8 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { /* Need to spill message_ptr to ARG1 as check_is_distributed uses it */ a.mov(ARG1, message_ptr); a.test(ARG1, ARG1); - a.short_().jne(check_is_distributed); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(check_is_distributed); /* Did we receive a signal or run out of reds? */ a.cmp(get_out, imm(0)); @@ -252,7 +235,8 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { { a.cmp(x86::qword_ptr(ARG1, offsetof(ErtsSignal, common.tag)), imm(THE_NON_VALUE)); - a.short_().jne(done); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(done); a.sub(FCALLS, imm(10)); @@ -260,7 +244,7 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { a.mov(ARG2, ARG1); a.mov(ARG1, c_p); - runtime_call<2>(decode_dist); + runtime_call<2>(beam_jit_decode_dist); emit_leave_runtime(); @@ -291,119 +275,6 @@ void BeamModuleAssembler::emit_i_loop_rec(const ArgVal &Wait) { fragment_call(ga->get_i_loop_rec_shared()); } -/* Remove a (matched) message from the message queue. */ -static Sint remove_message(Process *c_p, - Sint FCALLS, - Eterm *HTOP, - Eterm *E, - Uint32 active_code_ix) { - ErtsMessage *msgp; - - ERTS_CHK_MBUF_SZ(c_p); - - if (active_code_ix == ERTS_SAVE_CALLS_CODE_IX) { - save_calls(c_p, &exp_receive); - } - - msgp = erts_msgq_peek_msg(c_p); - - if (ERL_MESSAGE_TOKEN(msgp) == NIL) { -#ifdef USE_VM_PROBES - if (DT_UTAG(c_p) != NIL) { - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { - SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; - } else { - DT_UTAG(c_p) = NIL; - SEQ_TRACE_TOKEN(c_p) = NIL; - } - } else { -#endif - SEQ_TRACE_TOKEN(c_p) = NIL; -#ifdef USE_VM_PROBES - } - DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; -#endif - } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { - Eterm msg; - SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); -#ifdef USE_VM_PROBES - if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { - if (DT_UTAG(c_p) == NIL) { - DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); - } - DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; - } else { -#endif - ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)) || - is_atom(SEQ_TRACE_TOKEN_SENDER(c_p))); - c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - if (c_p->seq_trace_clock < - unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { - c_p->seq_trace_clock = - unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - } - msg = ERL_MESSAGE_TERM(msgp); - seq_trace_output(SEQ_TRACE_TOKEN(c_p), - msg, - SEQ_TRACE_RECEIVE, - c_p->common.id, - c_p); -#ifdef USE_VM_PROBES - } -#endif - } -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_receive)) { - Eterm token2 = NIL; - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - Sint len = erts_proc_sig_privqs_len(c_p); - - dtrace_proc_str(c_p, receiver_name); - token2 = SEQ_TRACE_TOKEN(c_p); - if (have_seqtrace(token2)) { - tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); - } - DTRACE6(message_receive, - receiver_name, - size_object(ERL_MESSAGE_TERM(msgp)), - len, /* This is NOT message queue len, but its something... */ - tok_label, - tok_lastcnt, - tok_serial); - } -#endif - erts_msgq_unlink_msg(c_p, msgp); - erts_msgq_set_save_first(c_p); - CANCEL_TIMER(c_p); - - erts_save_message_in_proc(c_p, msgp); - c_p->flags &= ~F_DELAY_GC; - - if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { - /* - * We want to GC soon but we leave a few - * reductions giving the message some time - * to turn into garbage. - */ - ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); - } - - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - return FCALLS; -} - void BeamModuleAssembler::emit_remove_message() { /* HTOP and E are passed explicitly and only read from, so we don't need to * swap them out. */ @@ -415,21 +286,17 @@ void BeamModuleAssembler::emit_remove_message() { a.mov(ARG1, c_p); a.mov(ARG2, FCALLS); a.mov(ARG5, active_code_ix); - runtime_call<5>(remove_message); + runtime_call<5>(beam_jit_remove_message); a.mov(FCALLS, RET); emit_leave_runtime(); } -static void save_message(Process *c_p) { - erts_msgq_set_save_next(c_p); -} - void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(save_message); + runtime_call<1>(erts_msgq_set_save_next); emit_leave_runtime(); @@ -437,32 +304,12 @@ void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) { a.jmp(labels[Dest.getValue()]); } -static void take_receive_lock(Process *c_p) { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); -} - -static void wait_locked(Process *c_p, ErtsCodePtr cp) { - c_p->arity = 0; - if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) { - erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->current = NULL; - c_p->i = cp; -} - -static void wait_unlocked(Process *c_p, ErtsCodePtr cp) { - take_receive_lock(c_p); - wait_locked(c_p, cp); -} - void BeamModuleAssembler::emit_wait_unlocked(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); - runtime_call<2>(wait_unlocked); + runtime_call<2>(beam_jit_wait_unlocked); emit_leave_runtime(); @@ -474,54 +321,19 @@ void BeamModuleAssembler::emit_wait_locked(const ArgVal &Dest) { a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); - runtime_call<2>(wait_locked); + runtime_call<2>(beam_jit_wait_locked); emit_leave_runtime(); abs_jmp(ga->get_do_schedule()); } -enum tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 }; - -static enum tmo_ret wait_timeout(Process *c_p, - Eterm timeout_value, - ErtsCodePtr next) { - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - if (timeout_value == make_small(0)) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - return RET_next; - } else if (timeout_value == am_infinity) { - c_p->flags |= F_TIMO; - } else { - int tres = erts_set_proc_timer_term(c_p, timeout_value); - if (tres == 0) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - */ - c_p->def_arg_reg[0] = (Eterm)next; - } else { /* Wrong time */ - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->freason = EXC_TIMEOUT_VALUE; - return RET_badarg; - } - } - } - return RET_wait; -} - void BeamModuleAssembler::emit_wait_timeout_unlocked(const ArgVal &Src, const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(take_receive_lock); + runtime_call<1>(beam_jit_take_receive_lock); emit_leave_runtime(); @@ -538,14 +350,18 @@ void BeamModuleAssembler::emit_wait_timeout_locked(const ArgVal &Src, a.mov(ARG1, c_p); a.lea(ARG3, x86::qword_ptr(next)); - runtime_call<3>(wait_timeout); + runtime_call<3>(beam_jit_wait_timeout); emit_leave_runtime(); ERTS_CT_ASSERT(RET_next < RET_wait && RET_wait < RET_badarg); a.cmp(RET, RET_wait); a.short_().je(wait); +#ifdef JIT_HARD_DEBUG + a.jl(next); +#else a.short_().jl(next); +#endif emit_handle_error(currLabel, (ErtsCodeMFA *)nullptr); @@ -556,27 +372,11 @@ void BeamModuleAssembler::emit_wait_timeout_locked(const ArgVal &Src, a.bind(next); } -static void timeout(Process *c_p) { - if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_clock_service, am_timeout, NULL); - } - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_timeout); - } - c_p->flags &= ~F_TIMO; - erts_msgq_set_save_first(c_p); -} - -static void timeout_locked(Process *c_p) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - timeout(c_p); -} - void BeamModuleAssembler::emit_timeout_locked() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(timeout_locked); + runtime_call<1>(beam_jit_timeout_locked); emit_leave_runtime(); } @@ -585,7 +385,7 @@ void BeamModuleAssembler::emit_timeout() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(timeout); + runtime_call<1>(beam_jit_timeout); emit_leave_runtime(); } diff --git a/erts/emulator/beam/jit/instr_select.cpp b/erts/emulator/beam/jit/x86/instr_select.cpp index 8aaa80267f..8aaa80267f 100644 --- a/erts/emulator/beam/jit/instr_select.cpp +++ b/erts/emulator/beam/jit/x86/instr_select.cpp diff --git a/erts/emulator/beam/jit/instr_trace.cpp b/erts/emulator/beam/jit/x86/instr_trace.cpp index a1b87624c7..a1b87624c7 100644 --- a/erts/emulator/beam/jit/instr_trace.cpp +++ b/erts/emulator/beam/jit/x86/instr_trace.cpp diff --git a/erts/emulator/beam/jit/ops.tab b/erts/emulator/beam/jit/x86/ops.tab index e33cc110a5..e33cc110a5 100644 --- a/erts/emulator/beam/jit/ops.tab +++ b/erts/emulator/beam/jit/x86/ops.tab diff --git a/erts/emulator/beam/jit/predicates.tab b/erts/emulator/beam/jit/x86/predicates.tab index a4e00586f6..a4e00586f6 100644 --- a/erts/emulator/beam/jit/predicates.tab +++ b/erts/emulator/beam/jit/x86/predicates.tab diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c index b709ed5a6f..2710c586c6 100644 --- a/erts/emulator/nifs/common/zlib_nif.c +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -370,6 +370,7 @@ static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env, d->s.next_in = input_vec[vec_idx].iov_base; d->s.avail_in = block_size; + /* We don't flush until reaching the end of our input. */ res = codec(&d->s, Z_NO_FLUSH); ASSERT(d->s.avail_in == 0 || d->s.avail_out == 0 || res != Z_OK); @@ -395,7 +396,12 @@ static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env, res = Z_BUF_ERROR; } - if(res == Z_OK && flush != Z_NO_FLUSH && (*bytes_remaining == 0)) { + if(res == Z_OK && (*bytes_remaining == 0) && d->s.avail_out > 0) { + /* We've reached the end of our input and need to flush the zlib state. + * + * Note that we do this even when the flush parameter is Z_NO_FLUSH as + * we may have filled our output buffer on the previous call. It will + * nop when there's nothing left to flush. */ d->s.next_in = NULL; d->s.avail_in = 0; diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl index e9ace3cd99..4ab84117c7 100644 --- a/erts/emulator/test/persistent_term_SUITE.erl +++ b/erts/emulator/test/persistent_term_SUITE.erl @@ -22,8 +22,10 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0,suite/0,init_per_suite/1,end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, basic/1,purging/1,sharing/1,get_trapping/1, destruction/1, + get_all_race/1, info/1,info_trapping/1,killed_while_trapping/1, off_heap_values/1,keys/1,collisions/1, init_restart/1, put_erase_trapping/1, @@ -45,6 +47,7 @@ suite() -> all() -> [basic,purging,sharing,get_trapping,info,info_trapping, destruction, + get_all_race, killed_while_trapping,off_heap_values,keys,collisions, init_restart, put_erase_trapping, killed_while_trapping_put, killed_while_trapping_erase, @@ -62,6 +65,15 @@ end_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, false), Config. +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok; +end_per_testcase(get_all_race, _Config) -> + get_all_race_cleanup(), + ok. + basic(_Config) -> Chk = chk(), N = 777, @@ -950,3 +962,31 @@ eval_bif_error(F, Args, Opts, T, Errors0) -> do_error_info(T, Errors) end end. + + +%% OTP-17298 +get_all_race(_Config) -> + N = 20 * erlang:system_info(schedulers_online), + persistent_term:put(get_all_race, N), + SPs = [spawn_link(fun() -> gar_setter(Seq) end) || Seq <- lists:seq(1, N)], + GPs = [spawn_link(fun gar_getter/0) || _ <- lists:seq(1, N)], + receive after 2000 -> ok end, + [begin unlink(Pid), exit(Pid,kill) end || Pid <- (SPs ++ GPs)], + ok. + +get_all_race_cleanup() -> + N = persistent_term:get(get_all_race, 0), + _ = persistent_term:erase(get_all_race), + [_ = persistent_term:erase(Seq) || Seq <- lists:seq(1, N)], + ok. + +gar_getter() -> + erts_debug:set_internal_state(reds_left, 1), + _ = persistent_term:get(), + gar_getter(). + +gar_setter(Key) -> + erts_debug:set_internal_state(reds_left, 1), + persistent_term:erase(Key), + persistent_term:put(Key, {complex, term}), + gar_setter(Key). diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 0e39622540..bacf052f47 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -47,7 +47,8 @@ process_info_reductions/1, bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1, otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1, - process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1, + process_info_messages/1, process_flag_badarg/1, + process_flag_fullsweep_after/1, process_flag_heap_size/1, spawn_opt_heap_size/1, spawn_opt_max_heap_size/1, processes_large_tab/1, processes_default_tab/1, processes_small_tab/1, processes_this_tab/1, processes_apply_trap/1, @@ -106,7 +107,8 @@ all() -> process_info_reductions, bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, garbage_collect, process_info_messages, - process_flag_badarg, process_flag_heap_size, + process_flag_badarg, + process_flag_fullsweep_after, process_flag_heap_size, spawn_opt_heap_size, spawn_opt_max_heap_size, spawn_huge_arglist, spawn_request_bif, @@ -1595,6 +1597,7 @@ process_flag_badarg(Config) when is_list(Config) -> chk_badarg(fun () -> process_flag(gurka, banan) end), chk_badarg(fun () -> process_flag(trap_exit, gurka) end), chk_badarg(fun () -> process_flag(error_handler, 1) end), + chk_badarg(fun () -> process_flag(fullsweep_after, gurka) end), chk_badarg(fun () -> process_flag(min_heap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, -1) end), @@ -2213,6 +2216,15 @@ processes_gc_trap(Config) when is_list(Config) -> exit(Suspendee, bang), ok. +process_flag_fullsweep_after(Config) when is_list(Config) -> + {fullsweep_after, OldFSA} = process_info(self(), fullsweep_after), + OldFSA = process_flag(fullsweep_after, 12345), + {fullsweep_after, 12345} = process_info(self(), fullsweep_after), + 12345 = process_flag(fullsweep_after, 0), + {fullsweep_after, 0} = process_info(self(), fullsweep_after), + 0 = process_flag(fullsweep_after, OldFSA), + ok. + process_flag_heap_size(Config) when is_list(Config) -> HSize = 2586, % must be gc fib+ number VHSize = 318187, % must be gc fib+ number diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl index 52cf029915..678134bf1c 100644 --- a/erts/emulator/test/trace_call_time_SUITE.erl +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -65,6 +65,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). -export([basic/1, on_and_off/1, info/1, + apply_bif_bug/1, abb_worker/1, disable_ongoing/1, pause_and_restart/1, scheduling/1, called_function/1, combo/1, bif/1, nif/1]). @@ -88,6 +89,7 @@ suite() -> all() -> [basic, on_and_off, info, pause_and_restart, scheduling, disable_ongoing, + apply_bif_bug, combo, bif, nif, called_function, dead_tracer, return_stop, catch_crash]. @@ -824,3 +826,26 @@ loop() -> Pid ! {self(), answer, erlang:apply(M, F, A)}, loop() end. + +%% OTP-17290, GH-4635 +apply_bif_bug(_Config) -> + Pid = spawn(?MODULE, abb_worker, [self()]), + erlang:trace(Pid, true, [call]), + erlang:trace_pattern({?MODULE,abb_foo,'_'}, true, [call_time]), + erlang:trace_pattern({erlang,display,1}, true, [call_time]), + Pid ! {call, erlang, display, ["Hej"]}, + receive + done -> ok + end, + erlang:trace_pattern({'_','_','_'}, false, [call_time]). + +abb_worker(Papa) -> + receive + {call, M, F, Args} -> + abb_foo(M, F, Args), + Papa ! done + end. + + +abb_foo(M,F,Args) -> + apply(M,F,Args). diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c index efa7ac3493..b8551eaeaf 100644 --- a/erts/etc/common/ct_run.c +++ b/erts/etc/common/ct_run.c @@ -188,6 +188,7 @@ int main(int argc, char** argv) eargv = eargv_base; eargc = 0; push_words(emulator); + free(emulator); eargc_base = eargc; eargv = eargv + eargv_size/2; eargc = 0; @@ -516,7 +517,7 @@ get_default_emulator(char* progname) char* s; if (strlen(progname) >= sizeof(sbuf)) - return ERL_NAME; + return strsave(ERL_NAME); strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { @@ -527,7 +528,7 @@ get_default_emulator(char* progname) break; } } - return ERL_NAME; + return strsave(ERL_NAME); } #ifdef __WIN32__ diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c index 37e07878ff..ae16d0c32b 100644 --- a/erts/etc/common/dialyzer.c +++ b/erts/etc/common/dialyzer.c @@ -214,6 +214,9 @@ int main(int argc, char** argv) eargv = eargv_base; eargc = 0; push_words(emulator); + if (emulator != env) { + free(emulator); + } eargc_base = eargc; eargv = eargv + eargv_size/2; eargc = 0; @@ -485,7 +488,7 @@ get_default_emulator(char* progname) char* s; if (strlen(progname) >= sizeof(sbuf)) - return ERL_NAME; + return strsave(ERL_NAME); strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { @@ -496,7 +499,7 @@ get_default_emulator(char* progname) break; } } - return ERL_NAME; + return strsave(ERL_NAME); } #ifdef __WIN32__ diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c index 523ead86ed..b972e14c6c 100644 --- a/erts/etc/common/erlc.c +++ b/erts/etc/common/erlc.c @@ -186,6 +186,9 @@ set_env(char *key, char *value) efree(str); #endif #endif + /* codechecker_intentional [Malloc] we may leak str if we don't + have copying putenv but that is fine since we only have a + constant amount of environment variables */ } static void @@ -892,6 +895,8 @@ find_executable(char* progname) struct stat s; if (stat(real_name, &s) == 0 && s.st_mode & S_IFREG) { return real_name; + } else { + free(real_name); } } } while (*path++ == ':'); @@ -908,7 +913,11 @@ safe_realpath(char* file) * Solaris. */ char* real_name = emalloc(PATH_MAX + 1); - return realpath(file, real_name); + char* result = realpath(file, real_name); + if (result != real_name) { + free(real_name); + } + return result; } #endif diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 0efacd67a1..2a0def7f48 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -574,7 +574,7 @@ int main(int argc, char **argv) in_index = bindir_slug_index + bindir_slug_length; out_index += block_length; } - + efree((void*)bindir_slug); strcpy(out_index, in_index); } @@ -1041,6 +1041,8 @@ int main(int argc, char **argv) } } + efree(emu_name); + if (process_args) { ADD_BOOT_CONFIG; } @@ -1198,7 +1200,9 @@ int main(int argc, char **argv) if (flavor == NULL) { flavor = type; } else { - currbuff += sprintf(currbuff,"-emu_type %s ", strndup(type,flavor - type)); + char* emu_type = strndup(type,flavor - type); + currbuff += sprintf(currbuff,"-emu_type %s ", emu_type); + free(emu_type); flavor++; } currbuff += sprintf(currbuff,"-emu_flavor %s", flavor); diff --git a/erts/etc/common/inet_gethost.c b/erts/etc/common/inet_gethost.c index 0e4ec402ab..961b3288a6 100644 --- a/erts/etc/common/inet_gethost.c +++ b/erts/etc/common/inet_gethost.c @@ -1878,7 +1878,8 @@ static int worker_loop(void) DEBUGF(5,("getnameinfo returned %d", error_num)); if (error_num) { error_num = map_netdb_error_ai(error_num); - sa = NULL; + FREE(sa); + sa = NULL; } #elif defined(HAVE_GETIPNODEBYADDR) /*#ifdef HAVE_GETNAMEINFO*/ struct in6_addr ia; @@ -1980,10 +1981,10 @@ static int worker_loop(void) } close_mesq(readfrom); close_mesq(writeto); +#endif if (reply) { FREE(reply); } -#endif return 1; } diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c index f13135d883..c7af00bb46 100644 --- a/erts/etc/common/typer.c +++ b/erts/etc/common/typer.c @@ -163,6 +163,7 @@ main(int argc, char** argv) eargv = eargv_base; eargc = 0; push_words(emulator); + free(emulator); eargc_base = eargc; eargv = eargv + eargv_size/2; eargc = 0; @@ -410,7 +411,7 @@ get_default_emulator(char* progname) char* s; if (strlen(progname) >= sizeof(sbuf)) - return ERL_NAME; + return strsave(ERL_NAME); strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { @@ -421,7 +422,7 @@ get_default_emulator(char* progname) break; } } - return ERL_NAME; + return strsave(ERL_NAME); } #ifdef __WIN32__ diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index f617736e43..adc20f5ac6 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -34,6 +34,8 @@ # You have to start beam in gdb using "run". # -rgdb Run the debug compiled emulator in gdb. # You have to start beam in gdb using "run". +# -lldb Run the debug compiled emulator in lldb. +# You have to start beam in lldb using "run". # -dump Dump the bt of all threads in a core. # -break F Run the debug compiled emulator in emacs and gdb and set break. # The session is started, i.e. "run" is already don for you. @@ -170,6 +172,10 @@ while [ $# -gt 0 ]; do shift GDB=gdb ;; + "-lldb") + shift + GDB=lldb + ;; "-break") shift GDB=gdb @@ -421,6 +427,20 @@ elif [ "x$GDB" = "xgdb" ]; then echo "source $ROOTDIR/erts/etc/unix/etp-commands" > $cmdfile # Fire up gdb in emacs... exec gdb $GDBBP -x $cmdfile $gdbcmd +elif [ "x$GDB" = "xlldb" ]; then + case "x$core" in + x) + beam_args=`$EXEC -emu_args_exit ${1+"$@"}` + lldbcmd="-- $beam_args" + ;; + *) + lldbcmd="--core ${core}" + ;; + esac + cmdfile="/tmp/.cerllldb.$$" + echo "env TERM=dumb" > $cmdfile + echo "command script import $ROOTDIR/erts/etc/unix/etp.py" >> $cmdfile + exec lldb -s $cmdfile $EMU_NAME $lldbcmd elif [ "x$GDB" = "xegdb" ]; then if [ "x$EMACS" = "x" ]; then EMACS=emacs diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 1648daefa6..ba35ee7091 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3024,51 +3024,57 @@ define etp-aux-work-flags # Args: int # if ($arg0 & 0x1) - printf " delayed-dealloc" + printf " delayed-aw-wakeup" end if ($arg0 & 0x2) - printf " delayed-dealloc-thr-prgr" + printf " delayed-dealloc" end if ($arg0 & 0x4) - printf " fix-alloc-dealloc" + printf " delayed-dealloc-thr-prgr" end if ($arg0 & 0x8) - printf " fix-alloc-lower-lim" + printf " fix-alloc-dealloc" end if ($arg0 & 0x10) - printf " later-op" + printf " fix-alloc-lower-lim" end if ($arg0 & 0x20) - printf " canceled-timers" + printf " later-op" end if ($arg0 & 0x40) - printf " canceled-timers-thr-prgr" + printf " canceled-timers" end if ($arg0 & 0x80) - printf " async-ready" + printf " canceled-timers-thr-prgr" end if ($arg0 & 0x100) - printf " async-ready-clean" + printf " async-ready" end if ($arg0 & 0x200) - printf " misc-thr-prgr" + printf " async-ready-clean" end if ($arg0 & 0x400) - printf " misc" + printf " misc-thr-prgr" end if ($arg0 & 0x800) - printf " set-tmo" + printf " misc" end if ($arg0 & 0x1000) - printf " mseg-cache-check" + printf " set-tmo" end if ($arg0 & 0x2000) + printf " mseg-cache-check" + end + if ($arg0 & 0x4000) printf " yield" end - if ($arg0 & 0x1000) + if ($arg0 & 0x8000) printf " reap-ports" end - if ($arg0 & ~0x7ff) + if ($arg0 & 0x10000) + printf " debug-wait-completed" + end + if ($arg0 & ~0x1ffff) printf " GARBAGE" end printf "\n" diff --git a/erts/etc/unix/etp.py b/erts/etc/unix/etp.py new file mode 100644 index 0000000000..ca8a15d69d --- /dev/null +++ b/erts/etc/unix/etp.py @@ -0,0 +1,652 @@ +# coding=utf-8 +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2013-2022. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +# This script was orinally written by Anthony Ramine (aka nox) in 2013 +# A lot of things have changed since then, but the same base remains. +# + +import re +import lldb +import shlex + +unquoted_atom_re = re.compile(u'^[a-zß-öø-ÿ][a-zA-Zß-öø-ÿ0-9@]*$') + +def __lldb_init_module(debugger, internal_dict): + debugger.HandleCommand('type format add -f hex Eterm') + debugger.HandleCommand('type format add -f hex BeamInstr') + debugger.HandleCommand('type summary add -F etp.eterm_summary Eterm') + debugger.HandleCommand('command script add -f etp.processes_cmd etp-processes') + debugger.HandleCommand('command script add -f etp.process_info_cmd etp-process-info') + debugger.HandleCommand('command script add -f etp.stacktrace_cmd etp-stacktrace') + debugger.HandleCommand('command script add -f etp.stackdump_cmd etp-stackdump') + debugger.HandleCommand('command script add -f etp.eterm_cmd etp') + +#################################### +## Print all processes in the system +#################################### +def processes_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = erts_proc(target) + proc_r_o = proc.GetChildMemberWithName('r').GetChildMemberWithName('o') + proc_max_ix = proc_r_o.GetChildMemberWithName('max') + proc_tab = proc_r_o.GetChildMemberWithName('tab').Cast(ProcessPtrPtr(target)) + proc_cnt = proc.GetChildMemberWithName('vola').GetChildMemberWithName('tile').GetChildMemberWithName('count').GetChildMemberWithName('counter').unsigned + invalid_proc = global_var('erts_invalid_process', target).address_of + for proc_ix in range(0, proc_max_ix.unsigned): + proc = offset(proc_ix, proc_tab).deref + if proc.unsigned != 0 and proc.unsigned != invalid_proc.unsigned: + print('---') + print(' Pix: %d' % proc_ix) + process_info(proc) + proc_cnt -= 1 + if proc_cnt == 0: + break + +############################################ +## Print process-info about a single process +############################################ +def process_info_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + process_info(proc) + +def process_info(proc): + print(' Pid: %s' % eterm(proc.GetChildMemberWithName('common').GetChildMemberWithName('id'))) + print(' State: %s' % process_state(proc)) + print(' Flags: %s' % process_flags(proc)) + current = proc.GetChildMemberWithName('current') + if current.unsigned != 0 and proc.GetChildMemberWithName('state').GetChildMemberWithName('counter').unsigned & 0x800 == 0: + print(' Current function: %s' % mfa(current)) + else: + print(' Current function: %s' % 'unknown') + i = proc.GetChildMemberWithName('i') + if i.unsigned != 0: + print(' I: %s' % eterm(i)) + else: + print(' I: %s' % 'unknown') + print(' Pointer: %#x' % proc.unsigned) + +def process_state(proc): + state = proc.GetChildMemberWithName('state').unsigned + res = '' + if state & 0x80000000: + res += "GARBAGE<0x80000000> | " + if state & 0x40000000: + res += "dirty-running-sys | " + if state & 0x20000000: + res += "dirty-running | " + if state & 0x10000000: + res += "dirty-active-sys | " + if state & 0x8000000: + res += "dirty-io-proc | " + if state & 0x4000000: + res += "dirty-cpu-proc | " + if state & 0x2000000: + res += "sig-q | " + if state & 0x1000000: + res += "off-heap-msgq | " + if state & 0x800000: + res += "delayed-sys | " + if state & 0x400000: + res += "proxy | " + proxy_process = True + else: + proxy_process = False + if state & 0x200000: + res += "running-sys | " + if state & 0x100000: + res += "active-sys | " + if state & 0x80000: + res += "sig-in-q | " + if state & 0x40000: + res += "sys-tasks | " + if state & 0x20000: + res += "garbage-collecting | " + if state & 0x10000: + res += "suspended | " + if state & 0x8000: + res += "running | " + if state & 0x4000: + res += "in-run-queue | " + if state & 0x2000: + res += "active | " + if state & 0x1000: + res += "unused | " + if state & 0x800: + res += "exiting | " + if state & 0x400: + res += "free | " + if state & 0x200: + res += "in-prq-low | " + if state & 0x100: + res += "in-prq-normal | " + if state & 0x80: + res += "in-prq-high | " + if state & 0x40: + res += "in-prq-max | " + if state & 0x30 == 0x0: + res += "prq-prio-max | " + elif state & 0x30 == 0x10: + res += "prq-prio-high | " + elif state & 0x30 == 0x20: + res += "prq-prio-normal | " + else: + res += "prq-prio-low | " + if state & 0xc == 0x0: + res += "usr-prio-max | " + elif state & 0xc == 0x4: + res += "usr-prio-high | " + elif state & 0xc == 0x8: + res += "usr-prio-normal | " + else: + res += "usr-prio-low | " + if state & 0x3 == 0x0: + res += "act-prio-max" + elif state & 0x3 == 0x1: + res += "act-prio-high" + elif state & 0x3 == 0x2: + res += "act-prio-normal" + else: + res += "act-prio-low" + return res + +def process_flags(proc): + flags = proc.GetChildMemberWithName('flags').unsigned + res = '' + if flags & ~((1 << 24)-1): + res += "GARBAGE<%#x> " % (flags & ~((1 << 24)-1)) + if flags & (1 << 22): + res += "trap-exit " + if flags & (1 << 21): + res += "hibernated " + if flags & (1 << 20): + res += "dirty-minor-gc " + if flags & (1 << 19): + res += "dirty-major-gc " + if flags & (1 << 18): + res += "dirty-gc-hibernate " + if flags & (1 << 17): + res += "dirty-cla " + if flags & (1 << 16): + res += "delayed-del-proc " + if flags & (1 << 15): + res += "have-blocked-nmsb " + if flags & (1 << 14): + res += "shdlr-onln-wait-q " + if flags & (1 << 13): + res += "delay-gc " + if flags & (1 << 12): + res += "abandoned-heap-use " + if flags & (1 << 11): + res += "disable-gc " + if flags & (1 << 10): + res += "force-gc " + if flags & (1 << 9): + res += "ets-super-user " + if flags & (1 << 8): + res += "have-blocked-msb " + if flags & (1 << 7): + res += "using-ddll " + if flags & (1 << 6): + res += "distribution " + if flags & (1 << 5): + res += "using-db " + if flags & (1 << 4): + res += "need-fullsweep " + if flags & (1 << 3): + res += "heap-grow " + if flags & (1 << 2): + res += "timo " + if flags & (1 << 1): + res += "inslpqueue " + if flags & (1 << 0): + res += "hibernate-sched " + return res + +############################################ +## Print the stacktrace of a single process +############################################ +def stacktrace_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + stackdump(proc, False) + +############################################ +## Print the stackdump of a single process +############################################ +def stackdump_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + stackdump(proc, True) + +def stackdump(proc, dump): + stop = proc.GetChildMemberWithName('stop') + send = proc.GetChildMemberWithName('hend') + cnt = 0 + if proc.GetChildMemberWithName('state').GetChildMemberWithName('counter').unsigned & 0x8000: + print('%%%%%% WARNING: The process is currently running, so c_p->stop will not be correct') + print(F'%% Stacktrace ({send.unsigned - stop.unsigned})'); + i = proc.GetChildMemberWithName('i') + if i.unsigned != 0: + print(F'I: {eterm(i)}') + while stop.unsigned < send.unsigned: + if stop.deref.unsigned & 0x3 == 0x0 or dump: + print(F'{cnt}: {eterm(stop.deref)}') + cnt += 1 + stop = offset(1, stop) + +############################################ +## Print an eterm +############################################ +def eterm_cmd(debugger, command, result, internal_dict): + args = shlex.split(command) + target = debugger.GetSelectedTarget() + term = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(args[0]).Cast(EtermPtr(target)) + if len(args) >= 2: + print(eterm(term, int(args[1]))) + else: + print(eterm(term)) + +############################################ +## Print the summary of an Eterm +############################################ +def eterm_summary(valobj, internal_dict): + if valobj.TypeIsPointerType(): + return '' + return F'{valobj.unsigned:#x} {eterm(valobj, 5)}' + +def eterm(valobj, depth = float('inf')): + val = valobj.unsigned + tag = val & 0x3 + if tag == 0x1: + return cons(valobj, depth) + elif tag == 0x2: + return boxed(valobj, depth) + elif tag == 0x3: + return imm(valobj) + elif val == 0x0: + return '<the non-value>' + elif val == 0x4: + return '<the non-value debug>' + else: + return cp(valobj) + +def cons(valobj, depth = float('inf')): + items = [] + cdr = valobj + improper = False + truncated = False + depth *= 20 + + while True: + ptr = cdr.CreateValueFromData( + "unconsed", + lldb.SBData.CreateDataFromInt(cdr.unsigned - 1), + EtermPtr(cdr.target)) + items.append((ptr.deref, depth // 20)); # Append depth, car + if ptr.deref.unsigned & 0xF == 0xF: + depth -= 1 + else: + depth -= 20 + cdr = offset(1,ptr).deref + if is_nil(cdr): + break + if cdr.unsigned & 0x1 == 0: + improper = True + break + if depth <= 1: + truncated = True + break + + if improper: + return '#ImproperList' + + ## Try to print as ascii first + chars = '' + isprintable = True + for car, car_depth in items: + if car.unsigned & 0xF == 0xF: + if car.unsigned >> 4 == 10: + chars += '\\n' + elif car.unsigned >> 4 == 9: + chars += '\\t' + else: + chars += f'{car.unsigned >> 4:c}' + else: + isprintable = False + break + isprintable = isprintable and chars.isprintable() + if isprintable: + if not truncated: + return F'"{chars}"' + else: + return F'"{chars}..."' + + ## If not printable, we print the objects + objs = [] + chars = '[' + for car, car_depth in items: + objs.append(eterm(car, car_depth)) + if not truncated: + return '[' + ','.join(objs) + ']' + else: + return '[' + ','.join(objs) + '|...]' + +def boxed(valobj, depth = float('inf')): + ptr = valobj.CreateValueFromData( + "unboxed", + lldb.SBData.CreateDataFromInt(valobj.unsigned - 2), + EtermPtr(valobj.target)) + boxed_hdr = ptr.deref.unsigned + if boxed_hdr & 0x3f == 0x00: + arity = (boxed_hdr >> 6) + terms = [] + for x in range(1, arity+1): + if depth <= 1: + terms.append('...') + break + depth -= 1 + terms.append(eterm(offset(x, ptr).deref, depth)) + res = ','.join(terms) + return F"{{{res}}}" + if boxed_hdr & 0x3c == 0x3c: + if boxed_hdr & 0xc0 == 0x0: + return "flat_map" + else: + return "hash_map" + boxed_type = (boxed_hdr >> 2) & 0xF + if boxed_type == 0xC: + return '#ExternalPid' + if boxed_type == 0xD: + return '#ExternalPort' + if boxed_type == 0x2 or boxed_type == 0x3: + return '#Bignum' + if boxed_type == 0x6: + return '#Float' + if boxed_type == 0x4: + return '#Ref' + if boxed_type == 0xE: + return '#ExternalRef' + if boxed_type == 0x5: + return '#Fun' + if boxed_type == 0x8: + return '#RefcBin' + if boxed_type == 0x9: + return '#HeapBin' + if boxed_type == 0xA: + return '#SubBin' + return F'#Boxed<{valobj.unsigned}>' + +def imm(valobj): + val = valobj.unsigned + if (val & 0x3) != 3: + return '#NotImmediate<%#x>' % val + tag = val & 0xF + if tag == 0x3: + return pid(valobj) + elif tag == 0x7: + return port(valobj) + elif tag == 0xF: + return str(val >> 4) + elif tag == 0xB: + # Immediate2 + tag2 = val & 0x3F + if tag2 == 0x0B: + return atom(valobj) + elif tag2 == 0x1B: + return F'#Catch<{val>>6:#x}>' + elif is_nil(valobj): + return '[]' + return '#UnknownImmediate<%#x>' % val + +# Continuation pointers + +def cp(valobj): + mfaptr = erts_lookup_function_info(valobj) + if mfaptr == None: + return '#Cp<%#x>' % valobj.unsigned + else: + return '#Cp<%s>' % mfa(mfaptr) + +# Pids and ports + +def pid(valobj): + val = valobj.unsigned + if (val & 0xF) == 0x3: + target = valobj.target + if etp_arch_bits(target) == 64: + if etp_big_endian(target): + data = (val >> 35) & 0x0FFFFFFF + else: + data = (val >> 4) & 0x0FFFFFFF + else: + data = pixdata2data(valobj) + return '<0.%u.%u>' % (data & 0x7FFF, (data >> 15) & 0x1FFF) + else: + return '#NotPid<%#x>' % val + +def port(valobj): + val = valobj.unsigned + if (val & 0xF) == 0x7: + target = valobj.target + if etp_arch_bits(target) == 64 and not etp_halfword(target): + if etp_big_endian(target): + data = (val >> 36) & 0x0FFFFFFF + else: + data = (val >> 4) & 0x0FFFFFFF + else: + data = pixdata2data(valobj) + return '#Port<0.%u>' % data + else: + return '#NotPort<%#x>' % val + +# Strings and atoms + +def atom(valobj): + val = valobj.unsigned + if (val & 0x3F) == 0x0B: + name = atom_name(atom_tab(valobj)) + if unquoted_atom_re.match(name): + return str(name) + else: + return quoted_name(name, "'") + else: + return '#NotAtom<%#x>' % val + +def atom_name(entry): + name = entry.GetChildMemberWithName('name') + length = entry.GetChildMemberWithName('len').unsigned + data = name.GetPointeeData(0, length).uint8s + return ''.join(map(chr, data)) + +def quoted_name(name, quote): + return quote + ''.join(map(lambda c: quoted_char(c, quote), name)) + quote + +def quoted_char(c, quote): + point = ord(c) + if c == quote: + return '\\' + quote + elif point == 0x08: + return '\\b' + elif point == 0x09: + return '\\t' + elif point == 0x0A: + return '\\n' + elif point == 0x0B: + return '\\v' + elif point == 0x0C: + return '\\f' + elif point == 0x0D: + return '\\e' + elif point >= 0x20 and point <= 0x7E or point >= 0xA0: + return c + elif (point > 0xFF): + return '#NotChar<%#x>' % c + else: + return '\\%03o' % point + +# Constants + +MI_FUNCTIONS = 13 +MI_NUM_FUNCTIONS = 0 + +def is_nil(value): + ## We handle both -5 and 0x3b as NIL values so that this script + ## works with more versions + return value.signed == -5 or value.unsigned == 0x3b + +# Types + +def Atom(target): + return target.FindFirstType('Atom') + +def Char(target): + return target.FindFirstType('char') +def CharPtr(target): + return Char(target).GetPointerType() + +def Eterm(target): + return target.FindFirstType('Eterm') +def EtermPtr(target): + return Eterm(target).GetPointerType() +def EtermPtrPtr(target): + return EtermPtr(target).GetPointerType() + +def Range(target): + return target.FindFirstType('Range') + +def BeamInstr(target): + return target.FindFirstType('BeamInstr') +def BeamInstrPtr(target): + return BeamInstr(target).GetPointerType() + +def ErtsCodeInfo(target): + return target.FindFirstType('ErtsCodeInfo') +def ErtsCodeInfoPtr(target): + return ErtsCodeInfo(target).GetPointerType() + +def Process(target): + return target.FindFirstType('Process') +def ProcessPtr(target): + return Process(target).GetPointerType() +def ProcessPtrPtr(target): + return ProcessPtr(target).GetPointerType() + +# Globals + +def erts_atom_table(target): + return global_var('erts_atom_table', target) + +def erts_proc(target): + return global_var('erts_proc', target) + +def etp_arch_bits(target): + return global_var('etp_arch_bits', target).unsigned + +def etp_big_endian(target): + return global_var('etp_endianness', target).unsigned > 0 + +def etp_halfword(target): + return False + +def the_active_code_index(target): + return global_var('the_active_code_index', target) + +# Functions + +def atom_tab(valobj): + idx = valobj.unsigned + target = valobj.target + seg = erts_atom_table(target).GetChildMemberWithName('seg_table') + slot = offset(idx >> 16, seg).deref + entry = offset(idx >> 6 & 0x3FF, slot).deref + return entry.Cast(Atom(target).GetPointerType()) + +def erts_lookup_function_info(valobj): + r = find_range(valobj) + if r is None: + return None + pc = valobj.unsigned + target = valobj.target + start = r.GetChildMemberWithName('start').Cast(EtermPtr(target)) + curr = offset(MI_FUNCTIONS, start).Cast(ErtsCodeInfoPtr(target).GetPointerType()) + prev = curr + cnt = offset(MI_NUM_FUNCTIONS, start).deref.unsigned + for x in range(0, cnt): + prev = curr + curr = offset(1, curr) + if pc < curr.deref.unsigned: + return prev.deref.GetChildMemberWithName('mfa') + return None + +def find_range(valobj): + pc = valobj.unsigned + target = valobj.target + active = the_active_code_index(target).unsigned + ranges = offset(active, global_var('r', target)) + n = ranges.GetChildMemberWithName('n').unsigned + low = ranges.GetChildMemberWithName('modules') + high = offset(n, low) + range_type = Range(target) + range_pointer_type = range_type.GetPointerType() + mid = ranges.GetChildMemberWithName('mid').Cast(range_pointer_type) + while low.unsigned < high.unsigned: + if pc < mid.GetChildMemberWithName('start').unsigned: + high = mid + elif pc > mid.GetChildMemberWithName('end').GetChildMemberWithName('counter').unsigned: + low = offset(1, mid).Cast(range_pointer_type) + else: + return mid + length = (high.unsigned - low.unsigned) // range_type.size + mid = offset(length // 2, low) + return None + +def mfa(mfa): + return '%s:%s/%d' % (eterm(mfa.GetChildMemberWithName('module')), + eterm(mfa.GetChildMemberWithName('function')), + mfa.GetChildMemberWithName('arity').unsigned) + +def pixdata2data(valobj): + pixdata = valobj.unsigned + proc = erts_proc(target) + ro = proc.GetChildMemberWithName('r').GetChildMemberWithName('o') + pix_mask = ro.GetChildMemberWithName('pix_mask').unsigned + pix_cl_mask = ro.GetChildMemberWithName('pix_cl_mask').unsigned + pix_cl_shift = ro.GetChildMemberWithName('pix_cl_shift').unsigned + pix_cli_mask = ro.GetChildMemberWithName('pix_cli_mask').unsigned + pix_cli_shift = ro.GetChildMemberWithName('pix_cli_shift').unsigned + data = pixdata & ~pix_mask + data |= (pixdata >> pix_cl_shift) & pix_cl_mask + data |= (pixdata & pix_cli_mask) << pix_cli_shift + return data + +# LLDB utils + +def global_var(name, target): + return target.FindGlobalVariables(name, 1)[0] + +def offset(i, valobj): + # print("offset(" + str(i) + ", " + str(valobj.unsigned) + ")") + val = valobj.GetChildAtIndex(i, lldb.eNoDynamicValues, True) + if valobj.TypeIsPointerType(): + return val.address_of.Cast(valobj.GetType()) + else: + return val diff --git a/erts/etc/unix/jit-reader.c b/erts/etc/unix/jit-reader.c index 6419570249..83481ba4f2 100644 --- a/erts/etc/unix/jit-reader.c +++ b/erts/etc/unix/jit-reader.c @@ -224,14 +224,14 @@ static void destroy(struct gdb_reader_funcs *self){ struct gdb_reader_funcs *gdb_init_reader(void){ struct gdb_reader_funcs *funcs = malloc(sizeof(struct gdb_reader_funcs)); - priv *priv = malloc(sizeof(priv)); - priv->num_ranges = 1; - priv->ranges = malloc(sizeof(range)); - priv->ranges[0].start = 0; - priv->ranges[0].end = 0; + priv *priv_data = malloc(sizeof(priv)); + priv_data->num_ranges = 1; + priv_data->ranges = malloc(sizeof(range)); + priv_data->ranges[0].start = 0; + priv_data->ranges[0].end = 0; funcs->reader_version = GDB_READER_INTERFACE_VERSION; - funcs->priv_data = priv; + funcs->priv_data = priv_data; funcs->read = read_debug_info; funcs->unwind = unwind; diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 2983966ade..e0f216ca5e 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -2359,6 +2359,9 @@ open_port(PortName, PortSettings) -> (error_handler, Module) -> OldModule when Module :: atom(), OldModule :: atom(); + (fullsweep_after, FullsweepAfter) -> OldFullsweepAfter when + FullsweepAfter :: non_neg_integer(), + OldFullsweepAfter :: non_neg_integer(); (min_heap_size, MinHeapSize) -> OldMinHeapSize when MinHeapSize :: non_neg_integer(), OldMinHeapSize :: non_neg_integer(); diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 661ba04fe8..90bb824873 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -11,7 +11,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -29,12 +29,12 @@ <rev></rev> <file>write_test_chapter.xml</file> </header> - + <section> <marker id="intro"></marker> <title>Support for Test Suite Authors</title> - <p>The <seeerl marker="ct"><c>ct</c></seeerl> module provides the main + <p>The <seeerl marker="ct"><c>ct</c></seeerl> module provides the main interface for writing test cases. This includes for example, the following:</p> <list type="bulleted"> @@ -45,8 +45,8 @@ </list> <p>For details about these functions, see module <seeerl marker="ct"><c>ct</c></seeerl>.</p> - - <p>The <c>Common Test</c> application also includes other modules named + + <p>The <c>Common Test</c> application also includes other modules named <c><![CDATA[ct_<component>]]></c>, which provide various support, mainly simplified use of communication protocols such as RPC, SNMP, FTP, Telnet, and others.</p> @@ -55,10 +55,10 @@ <section> <title>Test Suites</title> - + <p>A test suite is an ordinary Erlang module that contains test cases. It is recommended that the module has a name on the form - <c>*_SUITE.erl</c>. Otherwise, the directory and auto compilation + <c>*_SUITE.erl</c>. Otherwise, the directory and auto compilation function in <c>Common Test</c> cannot locate it (at least not by default). </p> @@ -66,10 +66,10 @@ in all test suite modules. </p> - <p>Each test suite module must export function + <p>Each test suite module must export function <seemfa marker="ct_suite#Module:all/0"><c>all/0</c></seemfa>, - which returns the list of all test case groups and test cases - to be executed in that module. + which returns the list of all test case groups and test cases + to be executed in that module. </p> <p>The callback functions to be implemented by the test suite are @@ -84,16 +84,16 @@ <p>Each test suite module can contain the optional configuration functions <seemfa marker="ct_suite#Module:init_per_suite/1"><c>init_per_suite/1</c></seemfa> - and <seemfa marker="ct_suite#Module:end_per_suite/1"><c>end_per_suite/1</c></seemfa>. + and <seemfa marker="ct_suite#Module:end_per_suite/1"><c>end_per_suite/1</c></seemfa>. If the init function is defined, so must the end function be. - </p> + </p> <p>If <c>init_per_suite</c> exists, it is called initially before the test cases are executed. It typically contains initializations common - for all test cases in the suite, which are only to be performed once. - <c>init_per_suite</c> is recommended for setting up and verifying state - and environment on the System Under Test (SUT) or the <c>Common Test</c> - host node, or both, so that the test cases in the suite executes correctly. + for all test cases in the suite, which are only to be performed once. + <c>init_per_suite</c> is recommended for setting up and verifying state + and environment on the System Under Test (SUT) or the <c>Common Test</c> + host node, or both, so that the test cases in the suite executes correctly. The following are examples of initial configuration operations: </p> <list type="bulleted"> @@ -103,10 +103,10 @@ </list> <p><c>end_per_suite</c> is called as the final stage of the test suite execution - (after the last test case has finished). The function is meant to be used - for cleaning up after <c>init_per_suite</c>. + (after the last test case has finished). The function is meant to be used + for cleaning up after <c>init_per_suite</c>. </p> - + <p><c>init_per_suite</c> and <c>end_per_suite</c> execute on dedicated Erlang processes, just like the test cases do. The result of these functions is however not included in the test run statistics of successful, failed, and @@ -115,16 +115,16 @@ <p>The argument to <c>init_per_suite</c> is <c>Config</c>, that is, the same key-value list of runtime configuration data that each test case takes - as input argument. <c>init_per_suite</c> can modify this parameter with + as input argument. <c>init_per_suite</c> can modify this parameter with information that the test cases need. The possibly modified <c>Config</c> list is the return value of the function. </p> <p>If <c>init_per_suite</c> fails, all test cases in the test - suite are skipped automatically (so called <em>auto skipped</em>), + suite are skipped automatically (so called <em>auto skipped</em>), including <c>end_per_suite</c>. </p> - + <p>Notice that if <c>init_per_suite</c> and <c>end_per_suite</c> do not exist in the suite, <c>Common Test</c> calls dummy functions (with the same names) instead, so that output generated by hook functions can be saved to the log @@ -139,17 +139,20 @@ <p>Each test suite module can contain the optional configuration functions <seemfa marker="ct_suite#Module:init_per_testcase/2"><c>init_per_testcase/2</c></seemfa> - and <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase/2</c></seemfa>. + and <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase/2</c></seemfa>. If the init function is defined, so must the end function be.</p> <p>If <c>init_per_testcase</c> exists, it is called before each test case in the suite. It typically contains initialization that - must be done for each test case (analog to <c>init_per_suite</c> for the + must be done for each test case (analog to <c>init_per_suite</c> for the suite).</p> <p><c>end_per_testcase/2</c> is called after each test case has finished, enabling cleanup after <c>init_per_testcase</c>.</p> + <note><p>If <c>end_per_testcase</c> crashes, however, test results are unaffected. + At the same time, this occurrence is reported in the test execution logs.</p></note> + <p>The first argument to these functions is the name of the test case. This value can be used with pattern matching in function clauses or conditional expressions to choose different initialization and cleanup @@ -159,17 +162,17 @@ <p>The second argument is the <c>Config</c> key-value list of runtime configuration data, which has the same value as the list returned by <c>init_per_suite</c>. <c>init_per_testcase/2</c> can modify this - parameter or return it "as is". The return value of <c>init_per_testcase/2</c> + parameter or return it "as is". The return value of <c>init_per_testcase/2</c> is passed as parameter <c>Config</c> to the test case itself.</p> - + <p>The return value of <c>end_per_testcase/2</c> is ignored by the - test server, with exception of the + test server, with exception of the <seeguide marker="dependencies_chapter#save_config"><c>save_config</c></seeguide> and <c>fail</c> tuple.</p> - <p><c>end_per_testcase</c> can check if the test case was successful. - (which in turn can determine how cleanup is to be performed). - This is done by reading the value tagged with <c>tc_status</c> from + <p><c>end_per_testcase</c> can check if the test case was successful. + (which in turn can determine how cleanup is to be performed). + This is done by reading the value tagged with <c>tc_status</c> from <c>Config</c>. The value is one of the following: </p> <list type="bulleted"> @@ -178,7 +181,7 @@ </item> <item> <p><c>{failed,Reason}</c></p> - <p>where <c>Reason</c> is <c>timetrap_timeout</c>, information from <c>exit/1</c>, + <p>where <c>Reason</c> is <c>timetrap_timeout</c>, information from <c>exit/1</c>, or details of a runtime error</p></item> <item> <p><c>{skipped,Reason}</c></p> @@ -186,7 +189,7 @@ </list> <p>Function <c>end_per_testcase/2</c> is even called if a - test case terminates because of a call to + test case terminates because of a call to <seemfa marker="ct#abort_current_testcase/1"><c>ct:abort_current_testcase/1</c></seemfa>, or after a timetrap time-out. However, <c>end_per_testcase</c> then executes on a different process than the test case @@ -198,7 +201,7 @@ </p> <list type="bulleted"> <item>If <c>init_per_testcase</c> crashes (called <em>auto skipped</em>).</item> - <item>If <c>init_per_testcase</c> returns a tuple <c>{skip,Reason}</c> + <item>If <c>init_per_testcase</c> returns a tuple <c>{skip,Reason}</c> (called <em>user skipped</em>).</item> </list> <p>The test case can also be marked as failed without executing it @@ -209,7 +212,7 @@ </p></note> <p>If it is determined during execution of <c>end_per_testcase</c> that - the status of a successful test case is to be changed to failed, + the status of a successful test case is to be changed to failed, <c>end_per_testcase</c> can return the tuple <c>{fail,Reason}</c> (where <c>Reason</c> describes why the test case fails).</p> @@ -221,20 +224,20 @@ <section> <marker id="test_cases"></marker> <title>Test Cases</title> - + <p>The smallest unit that the test server is concerned with is a test case. Each test case can test many things, for example, make several calls to the same interface function with different parameters. </p> - + <p>The author can choose to put many or few tests into each test case. Some things to keep in mind follows: - </p> + </p> <list type="bulleted"> <item><p>Many small test cases tend to result in extra, and possibly duplicated code, as well as slow test execution because of - large overhead for initializations and cleanups. Avoid duplicated + large overhead for initializations and cleanups. Avoid duplicated code, for example, by using common help functions. Otherwise, the resulting suite becomes difficult to read and understand, and expensive to maintain. @@ -243,52 +246,52 @@ fails. Also, large portions of test code risk being skipped when errors occur.</p> </item> - <item><p>Readability and maintainability suffer - when test cases become too large and extensive. It is not certain - that the resulting log files reflect very well the number of tests + <item><p>Readability and maintainability suffer + when test cases become too large and extensive. It is not certain + that the resulting log files reflect very well the number of tests performed. </p></item> </list> <p>The test case function takes one argument, <c>Config</c>, which contains configuration information such as <c>data_dir</c> and - <c>priv_dir</c>. (For details about these, see section + <c>priv_dir</c>. (For details about these, see section <seeguide marker="#data_priv_dir">Data and Private Directories</seeguide>. - The value of <c>Config</c> at the time of the call, is the same + The value of <c>Config</c> at the time of the call, is the same as the return value from <c>init_per_testcase</c>, mentioned earlier. </p> - <note><p>The test case function argument <c>Config</c> is not to be + <note><p>The test case function argument <c>Config</c> is not to be confused with the information that can be retrieved from the configuration files (using <seemfa marker="ct#get_config/1"><c> ct:get_config/1/2</c></seemfa>). The test case argument <c>Config</c> - is to be used for runtime configuration of the test suite and the - test cases, while configuration files are to contain data - related to the SUT. These two types of configuration data are handled + is to be used for runtime configuration of the test suite and the + test cases, while configuration files are to contain data + related to the SUT. These two types of configuration data are handled differently.</p></note> - + <p>As parameter <c>Config</c> is a list of key-value tuples, that is, a data type called a property list, it can be handled by the <seeerl marker="stdlib:proplists"><c>proplists</c></seeerl> module. - A value can, for example, be searched for and returned with function + A value can, for example, be searched for and returned with function <seemfa marker="stdlib:proplists#get_value/2"><c>proplists:get_value/2</c></seemfa>. Also, or alternatively, the general <seeerl marker="stdlib:lists"><c>lists</c></seeerl> - module contains useful functions. Normally, the only operations - performed on <c>Config</c> is insert (adding a tuple to the head of the list) - and lookup. <c>Common Test</c> provides a simple macro named <c>?config</c>, - which returns a value of an item in <c>Config</c> given the key (exactly like + module contains useful functions. Normally, the only operations + performed on <c>Config</c> is insert (adding a tuple to the head of the list) + and lookup. <c>Common Test</c> provides a simple macro named <c>?config</c>, + which returns a value of an item in <c>Config</c> given the key (exactly like <c>proplists:get_value</c>). Example: <c>PrivDir = ?config(priv_dir, Config)</c>. </p> - <p>If the test case function crashes or exits purposely, it is considered - <em>failed</em>. If it returns a value (no matter what value), it is - considered successful. An exception to this rule is the return value - <c>{skip,Reason}</c>. If this tuple is returned, the test case is considered - skipped and is logged as such.</p> + <p>If the test case function crashes or exits purposely, it is considered + <em>failed</em>. If it returns a value (no matter what value), it is + considered successful. An exception to this rule is the return value + <c>{skip,Reason}</c>. If this tuple is returned, the test case is considered + skipped and is logged as such.</p> <p>If the test case returns the tuple <c>{comment,Comment}</c>, the case - is considered successful and <c>Comment</c> is printed in the overview - log file. This is equal to calling + is considered successful and <c>Comment</c> is printed in the overview + log file. This is equal to calling <seemfa marker="ct#comment/1"><c>ct:comment(Comment)</c></seemfa>. </p> @@ -297,13 +300,13 @@ <section> <marker id="info_function"></marker> <title>Test Case Information Function</title> - + <p>For each test case function there can be an extra function with the same name but without arguments. This is the test case - information function. It is expected to return a list of tagged + information function. It is expected to return a list of tagged tuples that specifies various properties regarding the test case. </p> - + <p>The following tags have special meaning:</p> <taglist> <tag><c>timetrap</c></tag> @@ -311,9 +314,9 @@ <p> Sets the maximum time the test case is allowed to execute. If this time is exceeded, the test case fails with - reason <c>timetrap_timeout</c>. Notice that <c>init_per_testcase</c> + reason <c>timetrap_timeout</c>. Notice that <c>init_per_testcase</c> and <c>end_per_testcase</c> are included in the timetrap time. - For details, see section + For details, see section <seeguide marker="write_test_chapter#timetraps">Timetrap Time-Outs</seeguide>. </p> </item> @@ -321,7 +324,7 @@ <item> <p> Specifies any data related to the test case. This - data can be retrieved at any time using the + data can be retrieved at any time using the <seemfa marker="ct#userdata/3"><c>ct:userdata/3</c></seemfa> utility function. </p> @@ -329,7 +332,7 @@ <tag><c>silent_connections</c></tag> <item> <p> - For details, see section + For details, see section <seeguide marker="run_test_chapter#silent_connections">Silent Connections</seeguide>. </p> </item> @@ -339,25 +342,25 @@ Specifies configuration variables required by the test case. If the required configuration variables are not found in any of the test system configuration files, the test case is - skipped.</p> + skipped.</p> <p> - A required variable can also be given a default value to - be used if the variable is not found in any configuration file. To specify - a default value, add a tuple on the form - <c>{default_config,ConfigVariableName,Value}</c> to the test case information list + A required variable can also be given a default value to + be used if the variable is not found in any configuration file. To specify + a default value, add a tuple on the form + <c>{default_config,ConfigVariableName,Value}</c> to the test case information list (the position in the list is irrelevant). </p> <p><em>Examples:</em></p> - + <pre> - testcase1() -> + testcase1() -> [{require, ftp}, {default_config, ftp, [{ftp, "my_ftp_host"}, {username, "aladdin"}, {password, "sesame"}]}}].</pre> <pre> - testcase2() -> + testcase2() -> [{require, unix_telnet, unix}, {require, {unix, [telnet, username, password]}}, {default_config, unix, [{telnet, "my_telnet_host"}, @@ -369,16 +372,16 @@ <p>For more information about <c>require</c>, see section <seeguide marker="config_file_chapter#require_config_data"> Requiring and Reading Configuration Data</seeguide> - in section External Configuration Data and function + in section External Configuration Data and function <seemfa marker="ct#require/1"><c>ct:require/1/2</c></seemfa>.</p> <note><p>Specifying a default value for a required variable can result in a test case always getting executed. This might not be a desired behavior.</p> - </note> - + </note> + <p>If <c>timetrap</c> or <c>require</c>, or both, is not set specifically for a particular test case, default values specified by function - <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> + <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> are used. </p> @@ -395,7 +398,7 @@ {require,interfaces}, {userdata, [{description,"System Upgrade: RpuAddition Normal RebootNode"}, - {fts,"http://someserver.ericsson.se/test_doc4711.pdf"}]} + {fts,"http://someserver.ericsson.se/test_doc4711.pdf"}]} ].</pre> </section> @@ -404,23 +407,23 @@ <marker id="suite"></marker> <title>Test Suite Information Function</title> - <p>Function <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> - can, for example, be used in a test suite module to set a default - <c>timetrap</c> value and to <c>require</c> external configuration data. + <p>Function <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> + can, for example, be used in a test suite module to set a default + <c>timetrap</c> value and to <c>require</c> external configuration data. If a test case, or a group information function also specifies any of the information tags, it - overrides the default values set by <c>suite/0</c>. For details, - see + overrides the default values set by <c>suite/0</c>. For details, + see <seeguide marker="#info_function">Test Case Information Function</seeguide> and <seeguide marker="#test_case_groups">Test Case Groups</seeguide>. </p> - + <p>The following options can also be specified with the suite information list:</p> <list type="bulleted"> - <item><c>stylesheet</c>, + <item><c>stylesheet</c>, see <seeguide marker="run_test_chapter#html_stylesheet">HTML Style Sheets</seeguide></item> - <item><c>userdata</c>, + <item><c>userdata</c>, see <seeguide marker="#info_function">Test Case Information Function</seeguide></item> - <item><c>silent_connections</c>, + <item><c>silent_connections</c>, see <seeguide marker="run_test_chapter#silent_connections">Silent Connections</seeguide></item> </list> @@ -435,16 +438,16 @@ {userdata,[{info,"This suite tests database transactions."}]}, {silent_connections,[telnet]}, {stylesheet,"db_testing.css"} - ].</pre> - + ].</pre> + </section> <section> <marker id="test_case_groups"></marker> <title>Test Case Groups</title> - <p>A test case group is a set of test cases sharing configuration + <p>A test case group is a set of test cases sharing configuration functions and execution properties. Test case groups are defined by - function + function <seemfa marker="ct_suite#Module:groups/0"><c>groups/0</c></seemfa> according to the following syntax:</p> <pre> @@ -461,9 +464,9 @@ TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre> <p><c>GroupName</c> is the name of the group and must be unique within - the test suite module. Groups can be nested, by including a group definition - within the <c>GroupsAndTestCases</c> list of another group. - <c>Properties</c> is the list of execution + the test suite module. Groups can be nested, by including a group definition + within the <c>GroupsAndTestCases</c> list of another group. + <c>Properties</c> is the list of execution properties for the group. The possible values are as follows:</p> <pre> Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] @@ -484,24 +487,24 @@ <tag><c>shuffle</c></tag> <item><p>The cases in the group are executed in random order.</p></item> <tag><c>repeat, repeat_until_*</c></tag> - <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the + <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the group a given number of times, or until any, or all, cases fail or succeed.</p></item> </taglist> - + <p><em>Example:</em></p> <pre> groups() -> [{group1, [parallel], [test1a,test1b]}, {group2, [shuffle,sequence], [test2a,test2b,test2c]}].</pre> <p>To specify in which order groups are to be executed (also with respect - to test cases that are not part of any group), add tuples on the form + to test cases that are not part of any group), add tuples on the form <c>{group,GroupName}</c> to the <c>all/0</c> list.</p> <p><em>Example:</em></p> <pre> all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre> - <p>Execution properties with a group tuple in - <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. + <p>Execution properties with a group tuple in + <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. These properties override those specified in the group definition (see <c>groups/0</c> earlier). This way, the same set of tests can be run, but with different properties, without having to make copies of the group @@ -544,7 +547,7 @@ {tests3, [{repeat,2}], [t3a,t3b,t3c]}]. all() -> - [{group, tests1, default, + [{group, tests1, default, [{tests2, default, [{tests3, [parallel,{repeat,100}]}]}]}].</pre> @@ -561,33 +564,33 @@ execution is immediately stopped and the remaining cases are skipped.</p> <p>Before execution of a group begins, the configuration function - <seemfa marker="ct_suite#Module:init_per_group/2"><c>init_per_group(GroupName, Config)</c></seemfa> - is called. The list of tuples returned from this function is passed to the - test cases in the usual manner by argument <c>Config</c>. - <c>init_per_group/2</c> is meant to be used for initializations common + <seemfa marker="ct_suite#Module:init_per_group/2"><c>init_per_group(GroupName, Config)</c></seemfa> + is called. The list of tuples returned from this function is passed to the + test cases in the usual manner by argument <c>Config</c>. + <c>init_per_group/2</c> is meant to be used for initializations common for the test cases in the group. After execution of the group is finished, function - <seemfa marker="ct_suite#Module:end_per_group/2"><c>end_per_group(GroupName, Config)</c></seemfa> - is called. This function is meant to be used for cleaning up after + <seemfa marker="ct_suite#Module:end_per_group/2"><c>end_per_group(GroupName, Config)</c></seemfa> + is called. This function is meant to be used for cleaning up after <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p> <p>Whenever a group is executed, if <c>init_per_group</c> and <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls dummy functions (with the same names) instead. Output generated by hook functions are saved to the log files for these dummies. - For more information, see section + For more information, see section <seeguide marker="ct_hooks_chapter#manipulating">Manipulating Tests</seeguide> in section Common Test Hooks. </p> <note><p><c>init_per_testcase/2</c> and <c>end_per_testcase/2</c> - are always called for each individual test case, no matter if the case + are always called for each individual test case, no matter if the case belongs to a group or not.</p></note> - <p>The properties for a group are always printed in the top of the HTML log + <p>The properties for a group are always printed in the top of the HTML log for <c>init_per_group/2</c>. The total execution time for a group is included at the bottom of the log for <c>end_per_group/2</c>.</p> - <p>Test case groups can be nested so sets of groups can be + <p>Test case groups can be nested so sets of groups can be configured with the same <c>init_per_group/2</c> and <c>end_per_group/2</c> functions. Nested groups can be defined by including a group definition, or a group name reference, in the test case list of another group.</p> @@ -602,7 +605,7 @@ {group5, [sequence], [test5a,test5b,test5c]}].</pre> <p>In the previous example, if <c>all/0</c> returns group name references - in the order <c>[{group,group1},{group,group3}]</c>, the order of the + in the order <c>[{group,group1},{group,group3}]</c>, the order of the configuration functions and test cases becomes the following (notice that <c>init_per_testcase/2</c> and <c>end_per_testcase/2:</c> are also always called, but not included in this example for simplification):</p> @@ -613,7 +616,7 @@ test2a(Config2), test2b(Config2) end_per_group(group2, Config2) test1b(Config1) - end_per_group(group1, Config1) + end_per_group(group1, Config1) init_per_group(group3, Config) -> Config3 init_per_group(group4, Config3) -> Config4 test4a(Config4), test4b(Config4) (**) @@ -626,36 +629,36 @@ <p>(*) The order of test case <c>test1a</c>, <c>test1b</c>, and <c>group2</c> is undefined, as <c>group1</c> has a shuffle property.</p> <p>(**) These cases are not executed in order, but in parallel.</p> - <p>Properties are not inherited from top-level groups to nested - subgroups. For instance, in the previous example, the test cases in <c>group2</c> + <p>Properties are not inherited from top-level groups to nested + subgroups. For instance, in the previous example, the test cases in <c>group2</c> are not executed in random order (which is the property of <c>group1</c>).</p> </section> <section> <title>Parallel Property and Nested Groups</title> <p>If a group has a parallel property, its test cases are spawned - simultaneously and get executed in parallel. However, a test case is not + simultaneously and get executed in parallel. However, a test case is not allowed to execute in parallel with <c>end_per_group/2</c>, which means that the time to execute a parallel group is equal to the execution time of the slowest test case in the group. A negative side effect of running test cases in parallel is that the HTML summary pages - are not updated with links to the individual test case logs until function + are not updated with links to the individual test case logs until function <c>end_per_group/2</c> for the group has finished.</p> - <p>A group nested under a parallel group starts executing in parallel - with previous (parallel) test cases (no matter what properties the nested - group has). However, as test cases are never executed in parallel with - <c>init_per_group/2</c> or <c>end_per_group/2</c> of the same group, it is - only after a nested group has finished that remaining parallel cases + <p>A group nested under a parallel group starts executing in parallel + with previous (parallel) test cases (no matter what properties the nested + group has). However, as test cases are never executed in parallel with + <c>init_per_group/2</c> or <c>end_per_group/2</c> of the same group, it is + only after a nested group has finished that remaining parallel cases in the previous group become spawned.</p> </section> <section> <title>Parallel Test Cases and I/O</title> - <p>A parallel test case has a private I/O server as its group leader. + <p>A parallel test case has a private I/O server as its group leader. (For a description of the group leader concept, see <seeapp marker="erts:index">ERTS</seeapp>). - The central I/O server process, which handles the output from + The central I/O server process, which handles the output from regular test cases and configuration functions, does not respond to I/O messages during execution of parallel groups. This is important to understand to avoid certain traps, like the following:</p> @@ -676,13 +679,13 @@ (specified by an integer) or indefinitely (specified by <c>forever</c>). The repetition can also be stopped too early if any or all cases fail or succeed, that is, if any of the properties <c>repeat_until_any_fail</c>, - <c>repeat_until_any_ok</c>, <c>repeat_until_all_fail</c>, or + <c>repeat_until_any_ok</c>, <c>repeat_until_all_fail</c>, or <c>repeat_until_all_ok</c> is used. If the basic <c>repeat</c> - property is used, status of test cases is irrelevant for the repeat + property is used, status of test cases is irrelevant for the repeat operation.</p> - + <p>The status of a subgroup can be returned (<c>ok</c> or - <c>failed</c>), to affect the execution of the group on the level above. + <c>failed</c>), to affect the execution of the group on the level above. This is accomplished by, in <c>end_per_group/2</c>, looking up the value of <c>tc_group_properties</c> in the <c>Config</c> list and checking the result of the test cases in the group. If status <c>failed</c> is to be @@ -692,9 +695,9 @@ group is to be repeated or not (unless the basic <c>repeat</c> property is used).</p> - <p>The value of <c>tc_group_properties</c> is a list of status tuples, + <p>The value of <c>tc_group_properties</c> is a list of status tuples, each with the key <c>ok</c>, <c>skipped</c>, and <c>failed</c>. The - value of a status tuple is a list with names of test cases + value of a status tuple is a list with names of test cases that have been executed with the corresponding status as result.</p> <p>The following is an example of how to return the status from a group:</p> @@ -702,7 +705,7 @@ end_per_group(_Group, Config) -> Status = ?config(tc_group_result, Config), case proplists:get_value(failed, Status) of - [] -> % no failed cases + [] -> % no failed cases {return_group_result,ok}; _Failed -> % one or more failed {return_group_result,failed} @@ -721,27 +724,27 @@ case lists:member({group_result,group2}, Failed) of true -> {return_group_result,failed}; - false -> + false -> {return_group_result,ok} - end; + end; ...</pre> - <note><p>When a test case group is repeated, the configuration - functions <c>init_per_group/2</c> and <c>end_per_group/2</c> are + <note><p>When a test case group is repeated, the configuration + functions <c>init_per_group/2</c> and <c>end_per_group/2</c> are also always called with each repetition.</p></note> </section> <section> <title>Shuffled Test Case Order</title> <p>The order in which test cases in a group are executed is under normal - circumstances the same as the order specified in the test case list + circumstances the same as the order specified in the test case list in the group definition. With property <c>shuffle</c> set, however, <c>Common Test</c> instead executes the test cases in random order.</p> <p>You can provide a seed value (a tuple of three integers) with the shuffle property <c>{shuffle,Seed}</c>. This way, the same shuffling order can be created every time the group is executed. If no seed value - is specified, <c>Common Test</c> creates a "random" seed for the shuffling operation + is specified, <c>Common Test</c> creates a "random" seed for the shuffling operation (using the return value of <c>erlang:timestamp/0</c>). The seed value is always printed to the <c>init_per_group/2</c> log file so that it can be used to recreate the same execution order in a subsequent test run.</p> @@ -750,9 +753,9 @@ reset between turns.</p></note> <p>If a subgroup is specified in a group with a <c>shuffle</c> property, - the execution order of this subgroup in relation to the test cases + the execution order of this subgroup in relation to the test cases (and other subgroups) in the group, is random. The order of the - test cases in the subgroup is however not random (unless the + test cases in the subgroup is however not random (unless the subgroup has a <c>shuffle</c> property).</p> </section> @@ -770,10 +773,10 @@ group(connection_tests) -> [{require,login_data}, {timetrap,1000}].</pre> - + <p>The group information properties override those set with the suite information function, and can in turn be overridden by test - case information properties. For a list of valid information properties + case information properties. For a list of valid information properties and more general information, see the <seeguide marker="#info_function">Test Case Information Function</seeguide>. </p> @@ -784,10 +787,10 @@ <p>Information functions can also be used for functions <c>init_per_suite</c>, <c>end_per_suite</c>, <c>init_per_group</c>, and <c>end_per_group</c>, and they work the same way as with the - <seeguide marker="#info_function">Test Case Information Function</seeguide>. - This is useful, for example, for setting timetraps and requiring - external configuration data relevant only for the configuration - function in question (without affecting properties set for groups + <seeguide marker="#info_function">Test Case Information Function</seeguide>. + This is useful, for example, for setting timetraps and requiring + external configuration data relevant only for the configuration + function in question (without affecting properties set for groups and test cases in the suite).</p> <p>The information function <c>init/end_per_suite()</c> is called for @@ -797,7 +800,7 @@ cannot be used with <c>init/end_per_testcase(TestCase, Config)</c>, as these configuration functions execute on the test case process and use the same properties as the test case (that is, the properties - set by the test case information function, <c>TestCase()</c>). For a list + set by the test case information function, <c>TestCase()</c>). For a list of valid information properties and more general information, see the <seeguide marker="#info_function">Test Case Information Function</seeguide>. </p> @@ -807,9 +810,9 @@ <marker id="data_priv_dir"></marker> <title>Data and Private Directories</title> - <p>In the data directory, <c>data_dir</c>, the test module has - its own files needed for the testing. The name of <c>data_dir</c> - is the the name of the test suite followed by <c>"_data"</c>. + <p>In the data directory, <c>data_dir</c>, the test module has + its own files needed for the testing. The name of <c>data_dir</c> + is the the name of the test suite followed by <c>"_data"</c>. For example, <c>"some_path/foo_SUITE.beam"</c> has the data directory <c>"some_path/foo_SUITE_data/"</c>. Use this directory for portability, that is, to avoid hardcoding directory names in your suite. As the data @@ -829,11 +832,11 @@ Especially if the same test cases are executed multiple times during a test run (that is, if they belong to a test case group with property <c>repeat</c>) and there is a risk that files in the private directory get - overwritten. Under these circumstances, <c>Common Test</c> can be + overwritten. Under these circumstances, <c>Common Test</c> can be configured to create one dedicated private directory per test case and execution instead. This is accomplished with the flag/option <c>create_priv_dir</c> (to be used with the - <seecom marker="ct_run"><c>ct_run</c></seecom> program, the + <seecom marker="ct_run"><c>ct_run</c></seecom> program, the <seemfa marker="ct#run_test/1"><c>ct:run_test/1</c></seemfa> function, or as test specification term). There are three possible values for this option as follows: @@ -848,19 +851,19 @@ one private directory created per test run. The two latter values tell <c>Common Test</c> to generate a unique test directory name per test case and execution. If the auto version is used, <em>all</em> - private directories are created automatically. This can become very - inefficient for test runs with many test cases or repetitions, or both. - Therefore, if the manual version is used instead, the test case must tell + private directories are created automatically. This can become very + inefficient for test runs with many test cases or repetitions, or both. + Therefore, if the manual version is used instead, the test case must tell <c>Common Test</c> to create <c>priv_dir</c> when it needs it. - It does this by calling the function + It does this by calling the function <seemfa marker="ct#make_priv_dir/0"><c>ct:make_priv_dir/0</c></seemfa>. </p> <note><p>Do not depend on the current working directory for - reading and writing data files, as this is not portable. All - scratch files are to be written in the <c>priv_dir</c> and all - data files are to be located in <c>data_dir</c>. Also, - the <c>Common Test</c> server sets the current working directory to + reading and writing data files, as this is not portable. All + scratch files are to be written in the <c>priv_dir</c> and all + data files are to be located in <c>data_dir</c>. Also, + the <c>Common Test</c> server sets the current working directory to the test case log directory at the start of every case. </p></note> @@ -871,12 +874,12 @@ <p>Each test case is executed by a dedicated Erlang process. The process is spawned when the test case starts, and terminated when - the test case is finished. The configuration functions - <c>init_per_testcase</c> and <c>end_per_testcase</c> execute on the + the test case is finished. The configuration functions + <c>init_per_testcase</c> and <c>end_per_testcase</c> execute on the same process as the test case. </p> - <p>The configuration functions <c>init_per_suite</c> and + <p>The configuration functions <c>init_per_suite</c> and <c>end_per_suite</c> execute, like test cases, on dedicated Erlang processes. </p> @@ -898,30 +901,30 @@ for a subgroup, it overrides that of its higher level groups. Timetrap values set by individual test cases (by the test case information function) override both group- and suite- level timetraps.</p> - + <p>A timetrap can also be set or reset dynamically during the - execution of a test case, or configuration function. + execution of a test case, or configuration function. This is done by calling - <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>. - This function cancels the current timetrap and starts a new one + <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>. + This function cancels the current timetrap and starts a new one (that stays active until time-out, or end of the current function).</p> - + <p>Timetrap values can be extended with a multiplier value specified at startup with option <c>multiply_timetraps</c>. It is also possible to let the test server decide to scale up timetrap time-out values - automatically. That is, if tools such as <c>cover</c> or <c>trace</c> - are running during the test. This feature is disabled by default and + automatically. That is, if tools such as <c>cover</c> or <c>trace</c> + are running during the test. This feature is disabled by default and can be enabled with start option <c>scale_timetraps</c>.</p> - + <p>If a test case needs to suspend itself for a time that also gets multipled by <c>multiply_timetraps</c> (and possibly also scaled up if - <c>scale_timetraps</c> is enabled), the function + <c>scale_timetraps</c> is enabled), the function <seemfa marker="ct#sleep/1"><c>ct:sleep/1</c></seemfa> can be used (instead of, for example, <c>timer:sleep/1</c>).</p> - - <p>A function (<c>fun/0</c> or <c>{Mod,Func,Args}</c> (MFA) tuple) can be - specified as timetrap value in the suite-, group- and test case information - function, and as argument to function + + <p>A function (<c>fun/0</c> or <c>{Mod,Func,Args}</c> (MFA) tuple) can be + specified as timetrap value in the suite-, group- and test case information + function, and as argument to function <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>.</p> <p><em>Examples:</em></p> @@ -936,12 +939,12 @@ </list> <p>Before execution of the timetrap function (which is performed on a parallel, dedicated timetrap process), <c>Common Test</c> cancels - any previously set timer for the test case or configuration function. + any previously set timer for the test case or configuration function. When the timetrap function returns, the time-out is triggered, <em>unless</em> the return value is a valid timetrap time, such as an integer, or a <c>{SecMinOrHourTag,Time}</c> tuple (for details, see module - <seeerl marker="common_test">common_test</seeerl>). If a time value - is returned, a new timetrap is started to generate a time-out after + <seeerl marker="common_test">common_test</seeerl>). If a time value + is returned, a new timetrap is started to generate a time-out after the specified time.</p> <p>The user timetrap function can return a time value after a delay. @@ -952,19 +955,19 @@ <section> <marker id="logging"></marker> <title>Logging - Categories and Verbosity Levels</title> - <p><c>Common Test</c> provides the following three main functions for + <p><c>Common Test</c> provides the following three main functions for printing strings:</p> <list type="bulleted"> <item><c>ct:log(Category, Importance, Format, FormatArgs, Opts)</c></item> <item><c>ct:print(Category, Importance, Format, FormatArgs)</c></item> <item><c>ct:pal(Category, Importance, Format, FormatArgs)</c></item> </list> - <p>The <seemfa marker="ct#log/1"><c>log/1,2,3,4,5</c></seemfa> function - prints a string to the test case log file. - The <seemfa marker="ct#print/1"><c>print/1,2,3,4</c></seemfa> function + <p>The <seemfa marker="ct#log/1"><c>log/1,2,3,4,5</c></seemfa> function + prints a string to the test case log file. + The <seemfa marker="ct#print/1"><c>print/1,2,3,4</c></seemfa> function prints the string to screen. - The <seemfa marker="ct#pal/1"><c>pal/1,2,3,4</c></seemfa> function - prints the same string both to file and screen. The functions are described + The <seemfa marker="ct#pal/1"><c>pal/1,2,3,4</c></seemfa> function + prints the same string both to file and screen. The functions are described in module <seeerl marker="ct">ct</seeerl>. </p> @@ -984,7 +987,7 @@ exist in the <c>ct.hrl</c> header file. The default importance level, <c>?STD_IMPORTANCE</c> (used if argument <c>Importance</c> is not provided), is 50. This is also the importance used for standard I/O, - for example, from printouts made with <c>io:format/2</c>, + for example, from printouts made with <c>io:format/2</c>, <c>io:put_chars/1</c>, and so on.</p> <p><c>Importance</c> is compared to a verbosity level set by the @@ -1046,11 +1049,11 @@ ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</pre> ct:log(?INFO, "Info report: ~p", [Info]) ct:pal(?ERROR, "Error report: ~p", [Error])</pre> - + <p>The functions <seemfa marker="ct#set_verbosity/2"><c>ct:set_verbosity/2</c></seemfa> and <seemfa marker="ct#get_verbosity/1"><c>ct:get_verbosity/1</c></seemfa> may be used to modify and read verbosity levels during test execution.</p> - + <p>The arguments <c>Format</c> and <c>FormatArgs</c> in <c>ct:log/print/pal</c> are always passed on to the STDLIB function <c>io:format/3</c> (For details, see the <seeerl marker="stdlib:io"><c>io</c></seeerl> manual page).</p> @@ -1064,7 +1067,7 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> <p>How categories can be mapped to CSS tags is documented in section <seeguide marker="run_test_chapter#html_stylesheet">HTML Style Sheets</seeguide> in section Running Tests and Analyzing Results.</p> - + <p>Common Test will escape special HTML characters (<, > and &) in printouts to the log file made with <c>ct:pal/4</c> and <c>io:format/2</c>. In order to print strings with HTML tags to the log, use the <c>ct:log/3,4,5</c> function. The character @@ -1078,55 +1081,55 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> supported in test specifications).</p> <p>For more information about log files, see section - <seeguide marker="run_test_chapter#log_files">Log Files</seeguide> + <seeguide marker="run_test_chapter#log_files">Log Files</seeguide> in section Running Tests and Analyzing Results.</p> </section> - + <section> <title>Illegal Dependencies</title> <p>Even though it is highly efficient to write test suites with the <c>Common Test</c> framework, mistakes can be made, - mainly because of illegal dependencies. Some of the - more frequent mistakes from our own experience with running the + mainly because of illegal dependencies. Some of the + more frequent mistakes from our own experience with running the Erlang/OTP test suites follows:</p> <list type="bulleted"> <item><p>Depending on current directory, and writing there:</p> - + <p>This is a common error in test suites. It is assumed that the current directory is the same as the author used as current directory when the test case was developed. Many test cases even try to write scratch files to this directory. Instead - <c>data_dir</c> and <c>priv_dir</c> are to be used to locate + <c>data_dir</c> and <c>priv_dir</c> are to be used to locate data and for writing scratch files. </p> </item> <item><p>Depending on execution order:</p> - - <p>During development of test suites, make no assumptions on the - execution order of the test cases or suites. For example, a test - case must not assume that a server it depends on is already + + <p>During development of test suites, make no assumptions on the + execution order of the test cases or suites. For example, a test + case must not assume that a server it depends on is already started by a previous test case. Reasons for this follows: </p> <list type="bulleted"> <item>The user/operator can specify the order at will, and maybe - a different execution order is sometimes more relevant or + a different execution order is sometimes more relevant or efficient.</item> - <item>If the user specifies a whole directory of test suites - for the test, the execution order of the suites depends on - how the files are listed by the operating system, which varies + <item>If the user specifies a whole directory of test suites + for the test, the execution order of the suites depends on + how the files are listed by the operating system, which varies between systems.</item> - <item>If a user wants to run only a subset of a test suite, - there is no way one test case could successfully depend on + <item>If a user wants to run only a subset of a test suite, + there is no way one test case could successfully depend on another.</item> </list> </item> <item><p>Depending on Unix:</p> - - <p>Running Unix commands through <c>os:cmd</c> are likely + + <p>Running Unix commands through <c>os:cmd</c> are likely not to work on non-Unix platforms. </p> </item> @@ -1134,36 +1137,36 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> <item><p>Nested test cases:</p> <p>Starting a test case from another not only tests the same - thing twice, but also makes it harder to follow what is being + thing twice, but also makes it harder to follow what is being tested. Also, if the called test case fails for some reason, so do the caller. This way, one error gives cause to several error reports, which is to be avoided. </p> - <p>Functionality common for many test case functions can be - implemented in common help functions. If these functions are - useful for test cases across suites, put the help functions + <p>Functionality common for many test case functions can be + implemented in common help functions. If these functions are + useful for test cases across suites, put the help functions into common help modules. </p> </item> - + <item><p>Failure to crash or exit when things go wrong:</p> - + <p>Making requests without checking that the return value indicates success can be OK if the test case fails later, but it is never acceptable just to print an error - message (into the log file) and return successfully. Such test - cases do harm, as they create a false sense of security when + message (into the log file) and return successfully. Such test + cases do harm, as they create a false sense of security when overviewing the test results. </p> </item> <item><p>Messing up for subsequent test cases:</p> - + <p>Test cases are to restore as much of the execution environment as possible, so that subsequent test cases - do not crash because of their execution order. - The function - <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase</c></seemfa> + do not crash because of their execution order. + The function + <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase</c></seemfa> is suitable for this. </p> </item> diff --git a/lib/compiler/doc/src/Makefile b/lib/compiler/doc/src/Makefile index e1c445662c..39c00935cb 100644 --- a/lib/compiler/doc/src/Makefile +++ b/lib/compiler/doc/src/Makefile @@ -36,6 +36,9 @@ EDOC_REF3_FILES = cerl.xml cerl_trees.xml cerl_clauses.xml XML_PART_FILES = internal.xml XML_NOTES_FILES = notes.xml +XML_INTERNAL_FILES = beam_ssa.xml + +XML_GEN_FILES = $(XML_INTERNAL_FILES:%=$(XMLDIR)/%) BOOK_FILES = book.xml @@ -43,4 +46,7 @@ XML_FILES = \ $(BOOK_FILES) $(XML_NOTES_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) +$(XMLDIR)/%.xml: ../../internal_doc/%.md $(ERL_TOP)/make/emd2exml + $(ERL_TOP)/make/emd2exml $< $@ + include $(ERL_TOP)/make/doc.mk diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 26555438b8..8906b1c2eb 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -363,7 +363,7 @@ module.beam: module.erl \ since R13B04.</p> </item> - <tag><c>error_location</c></tag> + <tag><c>{error_location,line | column}</c></tag> <item> <p>If the value of this flag is <c>line</c>, the location <seeerl marker="#error_information"><c>ErrorLocation</c></seeerl> diff --git a/lib/compiler/doc/src/internal.xml b/lib/compiler/doc/src/internal.xml index f24b363c1c..4167f43e95 100644 --- a/lib/compiler/doc/src/internal.xml +++ b/lib/compiler/doc/src/internal.xml @@ -34,5 +34,6 @@ <xi:include href="cerl.xml"/> <xi:include href="cerl_trees.xml"/> <xi:include href="cerl_clauses.xml"/> + <xi:include href="beam_ssa.xml"/> </internal> diff --git a/lib/compiler/internal_doc/beam_ssa.md b/lib/compiler/internal_doc/beam_ssa.md new file mode 100644 index 0000000000..29ad019194 --- /dev/null +++ b/lib/compiler/internal_doc/beam_ssa.md @@ -0,0 +1,43 @@ +Invariants on the Structure and Format of BEAM SSA +================================================== + +Function Calls +-------------- + +All function calls not in a tail call position must be followed by a +succeeded:body-instruction unless one of the following exceptions +apply: + +* The function call can statically be proven to always fail. + +* The function call is to the `erlang`-module and can statically be + proven to always succeed or fail. + +Variable Naming +--------------- + +A variable name in BEAM SSA is either an atom, a non-negative integer +or a tuple: `atom() | non_neg_integer() | {atom() | non_neg_integer(), +non_neg_integer()}`. In order to generate fresh unused variable names, +all compiler transforms maintain a counter, the `cnt`-field in the +`opt_st`-record, which is incremented each time a new variable or +label is created. In the following description the value of the +`cnt`-field is called `Cnt`. + +Due to peculiarities in the BEAM SSA code generator, a compiler +transformation unfortunately cannot just use the `cnt`-value directly +as a fresh name. There are three basic strategies for creating fresh +variable names which can by used by a compiler pass: + +1) A name can be derived from an existing name of the form `V :: + atom() | non_neg_integer()` by selecting an atom, which is unique to + the compiler pass, to form a new name `{A, V}`. The same `A` cannot + be used by strategy 3) below. + +2) A name can be derived from an existing name of the form `V :: + non_neg_integer()` by combining it with the `cnt`-field into `{V, + Cnt}`. + +3) A fresh name can be created by selecting an atom `A`, which is + unique to the compiler pass, to form the new name `{A, Cnt}`. The + same `A` cannot be used by strategy 1) above. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index c9b902d8ff..515f5facd8 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -1049,7 +1049,10 @@ do_parse_module(DefEncoding, #compile{ifile=File,options=Opts,dir=Dir}=St) -> end. with_columns(Opts) -> - proplists:get_value(error_location, Opts, column) =:= column. + case proplists:get_value(error_location, Opts, column) of + column -> true; + line -> false + end. consult_abstr(_Code, St) -> case file:consult(St#compile.ifile) of @@ -1138,16 +1141,15 @@ foldl_transform([T|Ts], Code0, St) -> T:parse_transform(Code, S#compile.options) end, Run = runner(none, St), - try Run({Name, Fun}, Code0, St) of + StrippedCode = maybe_strip_columns(Code0, T, St), + try Run({Name, Fun}, StrippedCode, St) of {error,Es,Ws} -> {error,St#compile{warnings=St#compile.warnings ++ Ws, errors=St#compile.errors ++ Es}}; - {warning, Forms0, Ws} -> - Forms = maybe_strip_columns(Forms0, T), + {warning, Forms, Ws} -> foldl_transform(Ts, Forms, St#compile{warnings=St#compile.warnings ++ Ws}); - Forms0 -> - Forms = maybe_strip_columns(Forms0, T), + Forms -> foldl_transform(Ts, Forms, St) catch error:Reason:Stk -> @@ -1160,28 +1162,43 @@ foldl_transform([T|Ts], Code0, St) -> {undef_parse_transform,T}}]}], {error,St#compile{errors=St#compile.errors ++ Es}} end; -foldl_transform([], Code, St) -> {ok,Code,St}. - -%%% It is possible--although unlikely--that parse transforms add -%%% columns to the abstract code, why this function is called for -%%% every parse transform. -maybe_strip_columns(Code, T) -> - case erlang:function_exported(T, parse_transform_info, 0) of - true -> - Info = T:parse_transform_info(), - case maps:get(error_location, Info, column) of - line -> - strip_columns(Code); - _ -> - Code - end; - false -> - Code +foldl_transform([], Code, St) -> + %% We may need to strip columns added by parse transforms before returning + %% them back to the compiler. We pass ?MODULE as a bit of a hack to get the + %% correct default. + {ok, maybe_strip_columns(Code, ?MODULE, St), St}. + +%%% If a parse transform does not support column numbers it can say so using +%%% the parse_transform_info callback. The error_location is taken from both +%%% compiler options and from the parse transform and if either of them want +%%% to only use lines, we strip columns. +maybe_strip_columns(Code, T, St) -> + PTErrorLocation = + case erlang:function_exported(T, parse_transform_info, 0) of + true -> + maps:get(error_location, T:parse_transform_info(), column); + false -> + column + end, + ConfigErrorLocation = proplists:get_value(error_location, St#compile.options, column), + if + PTErrorLocation =:= line; ConfigErrorLocation =:= line -> + strip_columns(Code); + true -> Code end. strip_columns(Code) -> F = fun(A) -> erl_anno:set_location(erl_anno:line(A), A) end, - [erl_parse:map_anno(F, Form) || Form <- Code]. + [case Form of + {eof,{Line,_Col}} -> + {eof,Line}; + {ErrorOrWarning,{{Line,_Col},Module,Reason}} + when ErrorOrWarning =:= error; + ErrorOrWarning =:= warning -> + {ErrorOrWarning,{Line,Module,Reason}}; + Form -> + erl_parse:map_anno(F, Form) + end || Form <- Code]. get_core_transforms(Opts) -> [M || {core_transform,M} <- Opts]. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index e3f03d4959..7c47d8a289 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1724,11 +1724,44 @@ do_transforms(Config) -> %% Compile a file using line_pt and verify that column numbers %% have been stripped. Big = filename:join(DataDir, "big"), - {[],[_|_]} = compile_partition_warnings(Big, line_pt), + {[],[_|_]} = compile_partition_warnings(Big, [{parse_transform,line_pt}]), %% Compile a file using column_pt and verify that column numbers %% have NOT been stripped. - {[_|_],[]} = compile_partition_warnings(Big, column_pt), + {[_|_],[]} = compile_partition_warnings(Big, [{parse_transform,column_pt}]), + + %% Compile a file using column_pt and error_location=line and verify + %% that column numbers have been stripped. + {[],[_|_]} = compile_partition_warnings(Big, [{error_location,line}, + {parse_transform,column_pt}]), + + %% Compile a file using column_pt, line_pt and verify + %% that column numbers have been stripped. + {[],[_|_]} = compile_partition_warnings(Big, [{parse_transform,column_pt}, + {parse_transform,line_pt}]), + + %% Compile a file using column_pt that adds columns and error_location=line and + %% verify that column numbers have been stripped. + {[],[_|_]} = compile_partition_warnings(Big, [{error_location,line}, + add_columns, + {parse_transform,column_pt}]), + + %% Compile a file using column_pt that adds columns and error_location=line and + %% then call column_pt again to check that columns are stripped in between calls. + %% and then verify that column numbers have been stripped from output. + {[],[_|_]} = compile_partition_warnings(Big, [{error_location,line}, + add_columns, + {parse_transform,column_pt}, + {parse_transform,column_pt}]), + + %% Compile a file using column_pt that adds columns and en error and error_location=line and + %% verify that column numbers have been stripped. + {error,[{_What,[{Line,_,_}]}],[_|_]} = + compile_partition_warnings(Big, [{error_location,line}, + add_columns, + add_error, + {parse_transform,column_pt}]), + true = is_integer(Line), %% Cover transform code implementing the `time` option. {ok,big,_} = compile:file(Big, [binary, time, report, @@ -1754,15 +1787,18 @@ do_transforms(Config) -> ok. -compile_partition_warnings(Source, PT) -> - Opts = [binary, return, {core_transform,generic_pt}, {parse_transform,PT}], - {ok,big,<<_/binary>>,Ws0} = compile:file(Source, Opts), - [{_SourcePath,Ws}] = Ws0, - - %% Return {[ColumnWarning], [LineWarning]}. - lists:partition(fun({{L,C},_,_}) when is_integer(L), is_integer(C) -> true; - ({L,_,_}) when is_integer(L) -> false - end, Ws). +compile_partition_warnings(Source, Opts) -> + case compile:file(Source, [binary, return | Opts]) of + {ok,big,<<_/binary>>,Ws0} -> + [{_SourcePath,Ws}] = Ws0, + + %% Return {[ColumnWarning], [LineWarning]}. + lists:partition(fun({{L,C},_,_}) when is_integer(L), is_integer(C) -> true; + ({L,_,_}) when is_integer(L) -> false + end, Ws); + Error -> + Error + end. %% Cover the erl_compile API used by erlc. erl_compile_api(Config) -> diff --git a/lib/compiler/test/compile_SUITE_data/column_pt.erl b/lib/compiler/test/compile_SUITE_data/column_pt.erl index 93d56cb5ed..bf599bb38f 100644 --- a/lib/compiler/test/compile_SUITE_data/column_pt.erl +++ b/lib/compiler/test/compile_SUITE_data/column_pt.erl @@ -19,11 +19,68 @@ %% -module(column_pt). --export([parse_transform/2, parse_transform_info/0]). +-export([parse_transform/2, parse_transform_info/0, format_error/1]). parse_transform_info() -> %% Will default to {error_location,column}. #{}. -parse_transform(Forms, _Options) -> - Forms. +parse_transform(Forms, Options) -> + + HasColumn = + case proplists:get_value(error_location, Options, column) of + column -> + true; + line -> + false + end, + + AddColumns = proplists:get_value(add_columns, Options, false), + AddError = proplists:get_value(add_error, Options, false), + + {eof,LastLocation} = lists:keyfind(eof, 1, Forms), + LastLine = erl_anno:line(erl_anno:new(LastLocation)), + ExtraFormsAnno = + if HasColumn -> + erl_anno:new({LastLine,1}); + not HasColumn -> + erl_anno:new(LastLine) + end, + + ExtraForms = [{warning,{ExtraFormsAnno,?MODULE,"injected warning"}}] ++ + [{error,{ExtraFormsAnno,?MODULE,"injected error"}} || AddError], + + lists:map( + fun + ({eof,Location}) -> + Anno = erl_anno:new(Location), + HasColumn = erl_anno:column(Anno) =/= undefined, + if AddColumns -> + {eof,{erl_anno:line(Anno), 1}}; + not AddColumns -> + {eof,Location} + end; + ({ErrorOrWarning,{Location,Module,Text}}) + when ErrorOrWarning =:= error; + ErrorOrWarning =:= warning -> + Anno = erl_anno:new(Location), + HasColumn = erl_anno:column(Anno) =/= undefined, + if AddColumns -> + {ErrorOrWarning,{{erl_anno:line(Anno), 1}, Module, Text}}; + not AddColumns -> + {ErrorOrWarning, {Location, Module, Text}} + end; + (Form) -> + erl_parse:map_anno( + fun(Anno) -> + HasColumn = erl_anno:column(Anno) =/= undefined, + if AddColumns -> + erl_anno:set_location({erl_anno:line(Anno), 1}, Anno); + not AddColumns -> + Anno + end + end, Form) + end, Forms ++ ExtraForms). + +format_error(Error) -> + Error. diff --git a/lib/compiler/test/compile_SUITE_data/line_pt.erl b/lib/compiler/test/compile_SUITE_data/line_pt.erl index 0e77072d4c..32078081d2 100644 --- a/lib/compiler/test/compile_SUITE_data/line_pt.erl +++ b/lib/compiler/test/compile_SUITE_data/line_pt.erl @@ -24,5 +24,17 @@ parse_transform_info() -> #{error_location => line}. -parse_transform(Forms, _Options) -> - Forms. +parse_transform(Forms,_Options) -> + lists:map( + fun + ({eof,Location}) -> + Anno = erl_anno:new(Location), + false = erl_anno:column(Anno) =/= undefined, + {eof,Location}; + (Form) -> + erl_parse:map_anno( + fun(Anno) -> + false = erl_anno:column(Anno) =/= undefined, + Anno + end, Form) + end, Forms). diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 858ffe39f7..5671e6d801 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -52,6 +52,7 @@ ERL_NIF_TERM atom_not_supported; ERL_NIF_TERM atom_type; ERL_NIF_TERM atom_size; ERL_NIF_TERM atom_block_size; +ERL_NIF_TERM atom_prop_aead; ERL_NIF_TERM atom_key_length; ERL_NIF_TERM atom_iv_length; ERL_NIF_TERM atom_mode; @@ -186,6 +187,7 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_type = enif_make_atom(env,"type"); atom_size = enif_make_atom(env,"size"); atom_block_size = enif_make_atom(env,"block_size"); + atom_prop_aead = enif_make_atom(env,"prop_aead"); atom_key_length = enif_make_atom(env,"key_length"); atom_iv_length = enif_make_atom(env,"iv_length"); atom_mode = enif_make_atom(env,"mode"); diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index ac8b4e2b74..414e3045ea 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -57,6 +57,7 @@ extern ERL_NIF_TERM atom_type; extern ERL_NIF_TERM atom_size; extern ERL_NIF_TERM atom_block_size; extern ERL_NIF_TERM atom_key_length; +extern ERL_NIF_TERM atom_prop_aead; extern ERL_NIF_TERM atom_iv_length; extern ERL_NIF_TERM atom_mode; extern ERL_NIF_TERM atom_encrypt; diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index c872922f6b..132c98f17d 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -229,6 +229,9 @@ ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] enif_make_int(env, EVP_CIPHER_iv_length(cipher)), &ret); enif_make_map_put(env, ret, atom_block_size, enif_make_int(env, EVP_CIPHER_block_size(cipher)), &ret); + enif_make_map_put(env, ret, atom_prop_aead, + (((EVP_CIPHER_flags(cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) != 0) ? atom_true : atom_false), + &ret); mode = EVP_CIPHER_mode(cipher); switch (mode) { diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index b2e145840b..9c26f926e0 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -1189,7 +1189,7 @@ <name name="cipher_info" arity="1" since="OTP 22.0"/> <fsummary>Information about supported ciphers.</fsummary> <desc> - <p>Provides a map with information about block_size, key_length, iv_length and possibly other properties of the + <p>Provides a map with information about block_size, key_length, iv_length, aead support and possibly other properties of the cipher algorithm in question. </p> <note> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index a9c18a3779..81154e0884 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -725,7 +725,8 @@ mac_final_nif(_Ref) -> ?nif_stub. iv_length := integer(), block_size := integer(), mode := CipherModes, - type := undefined | integer() + type := undefined | integer(), + prop_aead := boolean() }, CipherModes :: undefined | cbc_mode @@ -1360,7 +1361,8 @@ rand_seed_nif(_Seed) -> ?nif_stub. when Algorithm :: pk_sign_verify_algs(), DigestType :: rsa_digest_type() | dss_digest_type() - | ecdsa_digest_type(), + | ecdsa_digest_type() + | none, Msg :: iodata() | {digest,iodata()}, Key :: rsa_private() | dss_private() @@ -1426,7 +1428,8 @@ verify(Algorithm, Type, Data, Signature, Key) -> when Algorithm :: pk_sign_verify_algs(), DigestType :: rsa_digest_type() | dss_digest_type() - | ecdsa_digest_type(), + | ecdsa_digest_type() + | none, Msg :: iodata() | {digest,iodata()}, Signature :: binary(), Key :: rsa_public() diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 4c89cdafe9..40b6ad46d6 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -57,6 +57,8 @@ bad_verify_name/1, cipher_info/0, cipher_info/1, + cipher_info_prop_aead_attr/0, + cipher_info_prop_aead_attr/1, cipher_padding/1, compute/0, compute/1, @@ -1242,6 +1244,45 @@ cipher_info(Config) when is_list(Config) -> ct:fail('Cipher unsupported',[]) end. +cipher_info_prop_aead_attr() -> + [{doc, "crypto cipher_info prop_aead attribute testing"}]. +cipher_info_prop_aead_attr(Config) when is_list(Config) -> + AeadCiphers = [aes_128_ccm, aes_192_ccm, aes_256_ccm, aes_128_gcm, aes_192_gcm, aes_256_gcm, chacha20_poly1305], + case lists:foldl(fun(C,Ok) -> + case crypto:cipher_info(C) of + #{prop_aead := true} -> + true and Ok; + _ -> + false + end + end, + true, + AeadCiphers + ) + of + true -> + ok; + false -> + ct:fail('AEAD Cipher attribute reported false',[]) + end, + NonAeadCiphers = [aes_ige256, blowfish_cbc, blowfish_cfb64], + case lists:foldl(fun(C,Ok) -> + case crypto:cipher_info(C) of + #{prop_aead := false} -> + true and Ok; + _ -> + false + end + end, + true, + NonAeadCiphers + ) + of + true -> + ok; + false -> + ct:fail('Non-AEAD Cipher attribute reported true',[]) + end. %%-------------------------------------------------------------------- hash_info() -> diff --git a/lib/edoc/include/edoc_doclet.hrl b/lib/edoc/include/edoc_doclet.hrl index 40ed77cff9..33e28785c0 100644 --- a/lib/edoc/include/edoc_doclet.hrl +++ b/lib/edoc/include/edoc_doclet.hrl @@ -26,6 +26,9 @@ %% Author contact: carlsson.richard@gmail.com %% ===================================================================== +%% Keep this define for backwards compatibility +-define(NO_APP, no_app). + -record(doclet_context, {dir = "", env, opts = []}). diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index a811037b5f..6ec5565fb0 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -77,7 +77,8 @@ exports :: ordset(function_name()), attributes :: ordset({atom(), term()}), records :: [{atom(), [{atom(), term()}]}], - encoding :: epp:source_encoding()}. + encoding :: epp:source_encoding(), + file :: file:filename()}. %% Module information. -type env() :: #env{}. diff --git a/lib/edoc/src/edoc.hrl b/lib/edoc/src/edoc.hrl index 9365d2ea5b..82d1fe932f 100644 --- a/lib/edoc/src/edoc.hrl +++ b/lib/edoc/src/edoc.hrl @@ -50,7 +50,8 @@ exports = [], attributes = [], records = [], - encoding = latin1}). + encoding = latin1, + file}). -record(env, {module = [], root = "", diff --git a/lib/edoc/src/edoc_extract.erl b/lib/edoc/src/edoc_extract.erl index 75df6637ab..cef5e55703 100644 --- a/lib/edoc/src/edoc_extract.erl +++ b/lib/edoc/src/edoc_extract.erl @@ -328,7 +328,8 @@ get_module_info(Forms, File) -> exports = ordsets:intersection(Exports, Functions), attributes = Attributes, records = Records, - encoding = Encoding}. + encoding = Encoding, + file = File}. get_list_keyval(Key, L) -> case lists:keyfind(Key, 1, L) of @@ -483,9 +484,19 @@ insert_specs(As, Ss, Mod) -> Specs = maps:from_list(SpecList), %% Assert that we've not skipped redundant specs for the same {Fun, Arity}. %% This should never happen, as such a module would not compile. - true = length(SpecList) == maps:size(Specs), + case length(SpecList) == maps:size(Specs) of + true -> ok; + false -> error_redundant_specs(Mod, SpecList, Specs) + end, insert_specs_(ModName, As, Specs). +error_redundant_specs(Mod, SpecList, Specs) -> + [{RedundantMFA, [Form]} | _] = lists:sort(SpecList) -- lists:sort(maps:to_list(Specs)), + {_, Line, _, _} = erl_syntax:revert(Form), + {_, F, A} = RedundantMFA, + edoc_report:error(Line, {Mod#module.file, {F, A}}, "Redundant -spec attribute found. Try setting {preprocess, true}."), + erlang:exit({redundant_spec, RedundantMFA}). + insert_specs_(_, [], _) -> []; insert_specs_(ModName, [#entry{} = A | As], Specs) -> #entry{name = {F, Arity}, data = {Cs, Cbs, _, Ts, Rs}} = A, diff --git a/lib/erl_docgen/doc/src/inline_tags.xml b/lib/erl_docgen/doc/src/inline_tags.xml index d56eac41ca..2be4fa6755 100644 --- a/lib/erl_docgen/doc/src/inline_tags.xml +++ b/lib/erl_docgen/doc/src/inline_tags.xml @@ -122,8 +122,8 @@ <p>Points to an Erlang module or a custom <seeguide marker="#markerTAG">marker</seeguide> within a module. Example:</p> <pre><![CDATA[ -<seeerl marker="stdlib:string">string(3)</seemfa>, -<seeerl marker="stdlib:string#oldapi">Old API in string</seemfa> +<seeerl marker="stdlib:string">string(3)</seeerl>, +<seeerl marker="stdlib:string#oldapi">Old API in string</seeerl> ]]></pre> results in: <seeerl marker="stdlib:string">string(3)</seeerl>,<seeerl marker="stdlib:string#oldapi">Old API in string</seeerl>. </item> diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index 6862eaf41a..e4dd926a62 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -919,18 +919,26 @@ <xsl:with-param name="chapnum" select="$chapnum"/> </xsl:call-template> </xsl:if> - <xsl:if test="(local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)"> - <!-- .../internal or .../internal/chapter --> - <xsl:call-template name="menu.internal.ug"> - <xsl:with-param name="chapnum" select="$chapnum"/> - </xsl:call-template> - </xsl:if> - <xsl:if test="(local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal)"> - <!-- .../internal,.../internal/erlref, .../internal/comref or .../internal/cref or .../internal/fileref or .../internal/appref --> - <xsl:call-template name="menu.internal.ref"> - <xsl:with-param name="curModule" select="$curModule"/> - </xsl:call-template> - </xsl:if> + + <xsl:choose> + <!-- Ugly hack to avoid two menus when we have internal documentation covering both modules and chapters --> + <xsl:when test="((local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)) and ((local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal))"> + <xsl:call-template name="menu.internal.ug_ref"> + </xsl:call-template> + </xsl:when> + <xsl:when test="(local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)"> + <!-- .../internal or .../internal/chapter --> + <xsl:call-template name="menu.internal.ug"> + <xsl:with-param name="chapnum" select="$chapnum"/> + </xsl:call-template> + </xsl:when> + <xsl:when test="(local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal)"> + <!-- .../internal,.../internal/erlref, .../internal/comref or .../internal/cref or .../internal/fileref or .../internal/appref --> + <xsl:call-template name="menu.internal.ref"> + <xsl:with-param name="curModule" select="$curModule"/> + </xsl:call-template> + </xsl:when> + </xsl:choose> <xsl:if test="(local-name() = 'application') or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::application)"> <!-- .../application,.../application/erlref, .../application/comref or .../application/cref or .../application/fileref or .../application/appref --> <xsl:call-template name="menu.ref"> @@ -1483,6 +1491,39 @@ </div> </xsl:template> + <!-- Menu.internal.chapter combined when we have both modules and free-form chapters --> + <xsl:template name="menu.internal.ug_ref"> + <div id="leftnav"> + <div class="leftnav-tube"> + + <xsl:call-template name="erlang_logo"/> + <p class="section-title"><xsl:value-of select="/book/header/title"/></p> + <p class="section-subtitle">Internal Documentation</p> + <p class="section-version">Version <xsl:value-of select="$appver"/></p> + + <xsl:call-template name="menu_top"/> + + <xsl:call-template name="menu_middle"/> + + <h3>Chapters</h3> + + <ul class="flipMenu" imagepath="{$topdocdir}/js/flipmenu"> + <xsl:call-template name="menu.chapter"> + <xsl:with-param name="entries" select="/book/internals/internal/chapter[header/title]"/> + </xsl:call-template> + </ul> + + <h3>Modules</h3> + + <ul class="flipMenu"> + <xsl:call-template name="menu.ref2"> + <xsl:with-param name="entries" select="/book/internals/internal/erlref[module]|/book/internals/internal/cref[lib]|/book/internals/internal/comref[com]|/book/internals/internal/fileref[file]|/book/internals/internal/appref[app]"/> + <!--xsl:with-param name="genFuncMenu" select="true"/--> + </xsl:call-template> + </ul> + </div> + </div> + </xsl:template> <!--Users Guide --> diff --git a/lib/erl_interface/doc/src/ei_connect.xml b/lib/erl_interface/doc/src/ei_connect.xml index c5ef9440c5..cd9465bb18 100644 --- a/lib/erl_interface/doc/src/ei_connect.xml +++ b/lib/erl_interface/doc/src/ei_connect.xml @@ -1047,6 +1047,7 @@ if (ei_reg_send(&ec, fd, x.buff, x.index) < 0) <func> <name since=""><ret>int</ret><nametext>ei_rpc(ei_cnode *ec, int fd, char *mod, char *fun, const char *argbuf, int argbuflen, ei_x_buff *x)</nametext></name> <name since=""><ret>int</ret><nametext>ei_rpc_to(ei_cnode *ec, int fd, char *mod, char *fun, const char *argbuf, int argbuflen)</nametext></name> + <name since="OTP-17048"><ret>int</ret><nametext>ei_xrpc_to(ei_cnode *ec, int fd, char *mod, char *fun, const char *argbuf, int argbuflen, int flags)</nametext></name> <name since=""><ret>int</ret><nametext>ei_rpc_from(ei_cnode *ec, int fd, int timeout, erlang_msg *msg, ei_x_buff *x)</nametext></name> <fsummary>Remote Procedure Call from C to Erlang.</fsummary> <type> @@ -1055,13 +1056,27 @@ if (ei_reg_send(&ec, fd, x.buff, x.index) < 0) <v><seecref marker="#erlang_msg"><c>erlang_msg</c></seecref></v> </type> <desc> - <p>Supports calling Erlang functions on remote nodes. - <c>ei_rpc_to()</c> sends an RPC request to a remote node - and <c>ei_rpc_from()</c> receives the results of such a - call. <c>ei_rpc()</c> combines the functionality of these - two functions by sending an RPC request and waiting for the results. - See also <seemfa marker="kernel:rpc#call/4"> - <c>rpc:call/4</c></seemfa> in Kernel.</p> + <p> + Supports calling Erlang functions on remote nodes. + <c>ei_rpc_to()</c> sends an RPC request to a remote node and + <c>ei_rpc_from()</c> receives the results of such a + call. <c>ei_rpc()</c> combines the functionality of these two + functions by sending an RPC request and waiting for the + results. + </p> + <p> + The <c>ei_xrpc_to()</c> function is equivalent to + <c>ei_rpc_to()</c> when its <c>flags</c> parameter is set to + <c>0</c>. When the flags parameter of <c>ei_xrpc_to()</c> is + set to <c>EI_RPC_FETCH_STDOUT</c>, stdout (standard output) + data are forwarded. See the documentation for the flags + parameter for more information about the + <c>EI_RPC_FETCH_STDOUT</c> flag. + </p> + <p> + <seemfa marker="kernel:rpc#call/4"><c>rpc:call/4</c></seemfa> + in Kernel. + </p> <list type="bulleted"> <item> <p><c>ec</c> is the C-node structure previously @@ -1104,21 +1119,75 @@ if (ei_reg_send(&ec, fd, x.buff, x.index) < 0) <c>ei_receive_msg</c></seecref>.</p> </item> <item> - <p><c>x</c> points to the dynamic buffer that receives - the result. For <c>ei_rpc()</c> this is the result - without the version magic number. For - <c>ei_rpc_from()</c> the result returns a version - magic number and a 2-tuple <c>{rex,Reply}</c>.</p> + <p><c>x</c> points to the dynamic buffer that receives the + result. For <c>ei_rpc()</c> this is the result without the + version magic number. For an <c>ei_rpc_from()</c> call the + result consists of a version magic number and a 2-tuple. + The 2-tuple can be in one of the following two forms:</p> + <taglist> + <tag><c>{rex,Reply}</c></tag> + <item> + This response value means that the RPC has + completed. The result value is the <c>Reply</c> + term. This is the only type of response that one can + get from an RPC triggered by a call to + <c>ei_rpc_to()</c> or <c>ei_xrpc_to()</c> without the + <c>EI_RPC_FETCH_STDOUT</c> flag. If the RPC was + triggered by a call to <c>ei_xrpc_to()</c> with the + <c>EI_RPC_FETCH_STDOUT</c> flag set, then all forwarded + stdout data has been received. + </item> + <tag><c>{rex_stdout,StdOutUTF8Binary}</c></tag> + <item> + This response value can only be obtained if the RPC + call was triggered by a call to <c>ei_xrpc_to()</c> + with the <c>EI_RPC_FETCH_STDOUT</c> flag set. This + response value means that forwarded stdout data has + been received. The stdout data is stored in a binary + and is UTF-8 encoded. One may need to call + <c>ei_rpc_from()</c> multiple times to read all the + stdout data. The stdout data is received in the same + order as it was written. All forwarded stdout data have + been received when a <c>{rex,Reply}</c> tuple has been + obtained from an <c>ei_rpc_from()</c> call. + </item> + </taglist> + </item> + <item> + <p><c>flags</c> The flag <c>EI_RPC_FETCH_STDOUT</c> is + currently the only flag that is supported by + <c>ei_xrpc_to()</c>. When <c>EI_RPC_FETCH_STDOUT</c> is + set, the called function is executed in a new process with + a <seemfa marker="erts:erlang#group_leader/0">group + leader</seemfa> that forwards all stdout data. This means + that stdout data that are written during the execution of + the called function, by the called function and by + descendant processes, will be forwarded (given that the + group leader has not been changed by a call to <seemfa + marker="erts:erlang#group_leader/2"><c>erlang:group_leader/2</c></seemfa>). + The forwarded stdout data need to be collected by a + sequence of calls to <c>ei_rpc_from()</c>. See the + description of the <c>x</c> parameter for how + <c>ei_rpc_from()</c> is used to receive stdout data. See + the documentation of the <seeguide + marker="stdlib:io_protocol">the I/O protocol</seeguide>, + for more information about the group leader concept.</p> + <note> + <p> + The flag <c>EI_RPC_FETCH_STDOUT</c> only works when + interacting with a node with a version greater or + equal to OTP-24. + </p> + </note> </item> </list> - <p><c>ei_rpc()</c> returns the number of bytes in the - result on success and <c>-1</c> on failure. - <c>ei_rpc_from()</c> returns the - number of bytes, otherwise one of <c>ERL_TICK</c>, - <c>ERL_TIMEOUT</c>, - and <c>ERL_ERROR</c>. When failing, all three - functions set <c>erl_errno</c> to one of the - following:</p> + <p><c>ei_rpc()</c> returns the number of bytes in the result + on success and <c>-1</c> on failure. <c>ei_rpc_from()</c> + returns the number of bytes, otherwise one of <c>ERL_TICK</c>, + <c>ERL_TIMEOUT</c>, and <c>ERL_ERROR</c>. The functions + <c>ei_rpc_to()</c> and <c>ei_xrpc_to()</c> returns 0 if + successful, otherwise -1. When failing, all four functions set + <c>erl_errno</c> to one of the following:</p> <taglist> <tag><c>EIO</c></tag> <item>I/O error.</item> diff --git a/lib/erl_interface/doc/src/erl_call_cmd.xml b/lib/erl_interface/doc/src/erl_call_cmd.xml index 04b5ec74bf..19b159d7a3 100644 --- a/lib/erl_interface/doc/src/erl_call_cmd.xml +++ b/lib/erl_interface/doc/src/erl_call_cmd.xml @@ -116,6 +116,41 @@ expressions and returns the result from the last expression. Returns <c>{ok,Result}</c> on success.</p> </item> + <tag><c>-fetch_stdout</c></tag> + <item> + <p> + (<em>Optional.</em>) Executes the code, specified with + the <c>-a</c> or <c>-e</c> option, in a new process that + has a <seemfa marker="erts:erlang#group_leader/0">group + leader</seemfa> that forwards all stdout (standard + output) data so that it is printed to stdout of the + <c>erl_call</c> process. This means that stdout data + that are written during the execution of the called code, + by the code and by descendant processes, will be + forwarded (given that the group leader has not been + changed by a call to <seemfa + marker="erts:erlang#group_leader/2"><c>erlang:group_leader/2</c></seemfa>). + </p> + <p> + The printed data is UTF-8 encoded. + </p> + <p> + This option is only relevant together with the option + <c>-a</c> or <c>-e</c>. + </p> + <p> + See the documentation of <seeguide + marker="stdlib:io_protocol">the I/O protocol</seeguide>, + for more information about the group leader concept. + </p> + <note> + <p> + This option only works when <c>erl_call</c> is + interacting with a node with a version greater or equal + to OTP-24. + </p> + </note> + </item> <tag><c>-h HiddenName</c></tag> <item> <p>(<em>Optional.</em>) Specifies the name of the hidden node @@ -145,6 +180,12 @@ <c>-s</c> is specified, an Erlang node will (if necessary) be started with <c>erl -name</c>.</p> </item> + <tag><c>-no_result_term</c></tag> + <item> + <p>(<em>Optional.</em>) Do not print the result term. This + option is only relevant together with the options + <c>-a</c> and <c>-e</c>.</p> + </item> <tag><c>-q</c></tag> <item> <p>(<em>Optional.</em>) Halts the Erlang node specified @@ -292,6 +333,16 @@ start() -> {registered_name,user}}, {<madonna@chivas.du.etx.ericsson.se,38,0>, []}] + ]]></code> + <p>To forward standard output without printing the result term + (<em>again, the input ends with EOF (Control-D)</em>):</p> + <code type="none"><![CDATA[ +erl_call -s -e -sname madonna -fetch_stdout -no_result_term +io:format("Number of schedulers: ~p~n", [erlang:system_info(schedulers)]), +io:format("Number of logical cores: ~p~n", [erlang:system_info(logical_processors_available)]). +^D +Number of schedulers: 8 +Number of logical cores: 8 ]]></code> </section> </comref> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index f1b6112960..f171bf1a8d 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -122,7 +122,8 @@ typedef LONG_PTR ssize_t; /* Sigh... */ #define ERL_DEMONITOR_P 20 #define ERL_MONITOR_P_EXIT 21 - +/* For ei_xrpc_to */ +#define EI_RPC_FETCH_STDOUT 1 /* -------------------------------------------------------------------- */ /* Defines used for ei_get_type_internal() output */ /* -------------------------------------------------------------------- */ @@ -435,6 +436,8 @@ int ei_reg_send_tmo(ei_cnode* ec, int fd, char *server_name, char* buf, int len, int ei_rpc(ei_cnode* ec, int fd, char *mod, char *fun, const char* inbuf, int inbuflen, ei_x_buff* x); +int ei_xrpc_to(ei_cnode* ec, int fd, char *mod, char *fun, + const char* buf, int len, int flags); int ei_rpc_to(ei_cnode* ec, int fd, char *mod, char *fun, const char* buf, int len); int ei_rpc_from(ei_cnode* ec, int fd, int timeout, erlang_msg* msg, diff --git a/lib/erl_interface/src/Makefile.in b/lib/erl_interface/src/Makefile.in index 061fd05a1d..56fdaa608e 100644 --- a/lib/erl_interface/src/Makefile.in +++ b/lib/erl_interface/src/Makefile.in @@ -390,8 +390,7 @@ NEVERUSED = \ send_link.c ERLCALL = \ - prog/erl_call.c \ - prog/erl_start.c + prog/erl_call.c # Note that encode/decode_term.c defines ei functions that is diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index d1b1dea892..fede6606e2 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -1851,18 +1851,26 @@ int ei_xreceive_msg_tmo(int fd, erlang_msg *msg, ei_x_buff *x, unsigned ms) return ei_do_receive_msg(fd, 0, msg, x, ms); } -/* -* The RPC consists of two parts, send and receive. -* Here is the send part ! -* { PidFrom, { call, Mod, Fun, Args, user }} -*/ /* -* Now returns non-negative number for success, negative for failure. +* A remote process call consists of two parts, sending a request and +* receiving a response. This function sends the request and the +* ei_rpc_from function receives the response. +* +* Here is the term that is sent when (flags & EI_RPC_FETCH_STDOUT) != 0: +* +* { PidFrom, { call, Mod, Fun, Args, send_stdout_to_caller }} +* +* Here is the term that is sent otherwise: +* +* { PidFrom, { call, Mod, Fun, Args, user }} +* +* Returns a non-negative number for success and a negative number for +* failure. +* */ -int ei_rpc_to(ei_cnode *ec, int fd, char *mod, char *fun, - const char *buf, int len) +int ei_xrpc_to(ei_cnode *ec, int fd, char *mod, char *fun, + const char *buf, int len, int flags) { - ei_x_buff x; erlang_pid *self = ei_self(ec); int err = ERL_ERROR; @@ -1872,10 +1880,10 @@ int ei_rpc_to(ei_cnode *ec, int fd, char *mod, char *fun, goto einval; if (ei_x_encode_tuple_header(&x, 2) < 0) /* A */ goto einval; - + if (ei_x_encode_pid(&x, self) < 0) /* A 1 */ goto einval; - + if (ei_x_encode_tuple_header(&x, 5) < 0) /* B A 2 */ goto einval; if (ei_x_encode_atom(&x, "call") < 0) /* B 1 */ @@ -1886,14 +1894,19 @@ int ei_rpc_to(ei_cnode *ec, int fd, char *mod, char *fun, goto einval; if (ei_x_append_buf(&x, buf, len) < 0) /* B 4 */ goto einval; - if (ei_x_encode_atom(&x, "user") < 0) /* B 5 */ - goto einval; - + if (flags & EI_RPC_FETCH_STDOUT) { + if (ei_x_encode_atom(&x, "send_stdout_to_caller") < 0) /* B 5 */ + goto einval; + } else { + if (ei_x_encode_atom(&x, "user") < 0) /* B 5 */ + goto einval; + } + err = ei_send_reg_encoded(fd, self, "rex", x.buff, x.index); if (err) goto error; - - ei_x_free(&x); + + ei_x_free(&x); return 0; @@ -1904,6 +1917,13 @@ error: if (x.buff != NULL) ei_x_free(&x); return err; +} /* xrpc_to */ + + +int ei_rpc_to(ei_cnode *ec, int fd, char *mod, char *fun, + const char *buf, int len) +{ + return ei_xrpc_to(ec, fd, mod, fun, buf, len, 0); } /* rpc_to */ /* diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index 1c9bd69a96..60940d7ae6 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -20,13 +20,21 @@ */ /* - * Function: Makes it possible to send and receive Erlang - * messages from the (Unix) command line. - * Note: We don't free any memory at all since we only - * live for a short while. + * Description: This file implements the erl_call command line + * utility. The erl_call command can be used to: + * + * * Execute code on an Erlang node and get the result back + * * Start and stop Erlang nodes + * * Upload and compile a module on an Erlang node + * + * See the erl_call man page or HTML documentation for additional + * information. * */ +/* An exception from using eidef.h, use config.h directly */ +#include "config.h" + #ifdef __WIN32__ #include <winsock2.h> #include <direct.h> @@ -35,16 +43,23 @@ #else /* unix */ +#include <arpa/inet.h> +#include <errno.h> +#include <netdb.h> +#include <netinet/in.h> +#include <signal.h> +#include <stdio.h> +#include <string.h> +#include <sys/param.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/times.h> #include <sys/types.h> #include <sys/uio.h> -#include <sys/time.h> +#include <sys/wait.h> +#include <time.h> #include <unistd.h> -#include <sys/param.h> -#include <netdb.h> -#include <sys/times.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> #if TIME_WITH_SYS_TIME # include <sys/time.h> @@ -59,6 +74,8 @@ #endif +#include <sys/types.h> + #include <stdio.h> #include <stdlib.h> @@ -69,8 +86,24 @@ #include "ei.h" #include "ei_resolve.h" -#include "erl_start.h" /* FIXME remove dependency */ +#define ERL_START_MSG "gurka" /* make something up */ +#define ERL_START_TIME 10000 /* wait this long (ms) */ +#define ERL_START_LOGFILE ".erl_start.out" /* basename of logfile */ + +/* flags used by erl_connect and erl_xconnect */ +#define ERL_START_ENODE 0x0001 +#define ERL_START_EPMD 0x0002 +#define ERL_START_LONG 0x0004 +#define ERL_START_COOKIE 0x0008 +#define ERL_START_DEBUG 0x0010 +#define ERL_START_VERBOSE 0x0020 +#define ERL_START_REMOTE 0x0040 + +/* error return values */ +#define ERL_S_TIMEOUT -51 /* a timeout occurred */ +#define ERL_BADARG -52 /* an argument contained an incorrect value */ +#define ERL_SYS_ERROR -99 /* a system error occurred (check errno) */ struct call_flags { int startp; @@ -84,6 +117,8 @@ struct call_flags { int debugp; int verbosep; int haltp; + int fetch_stdout; + int print_result_term; long port; char *hostname; char *cookie; @@ -93,6 +128,9 @@ struct call_flags { char *script; }; +/* start an erlang system */ +int erl_start_sys(ei_cnode *ec, char *alive, Erl_IpAddr addr, int flags, + char *erl, char *add_args[]); static void usage_arg(const char *progname, const char *switchname); static void usage_error(const char *progname, const char *switchname); static void usage(const char *progname); @@ -105,6 +143,10 @@ static void* ei_chk_malloc(size_t size); static void* ei_chk_calloc(size_t nmemb, size_t size); static void* ei_chk_realloc(void *old, size_t size); static char* ei_chk_strdup(char *s); +static int rpc_print_node_stdout(ei_cnode* ec, int fd, char *mod, + char *fun, const char* inbuf, + int inbuflen, ei_x_buff* x); +static void exit_free_flags_fields(int exit_status, struct call_flags* flags); /* Converts the given hostname to a shortname, if required. */ static void format_node_hostname(const struct call_flags *flags, @@ -146,6 +188,13 @@ int main(int argc, char *argv[]) ei_cnode ec; flags.port = -1; flags.hostname = NULL; + flags.fetch_stdout = 0; + flags.print_result_term = 1; + flags.script = NULL; + flags.hidden = NULL; + flags.apply = NULL; + flags.cookie = NULL; + flags.node = NULL; ei_init(); @@ -159,7 +208,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-sname "); } - + if (flags.node != NULL) { + free(flags.node); + } flags.node = ei_chk_strdup(argv[i+1]); i++; flags.use_long_name = 0; @@ -167,7 +218,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-name "); } - + if (flags.node != NULL) { + free(flags.node); + } flags.node = ei_chk_strdup(argv[i+1]); i++; flags.use_long_name = 1; @@ -180,16 +233,24 @@ int main(int argc, char *argv[]) char* address_string_end = strchr(hostname_port_arg, ':'); if (address_string_end == NULL) { flags.port = strtol(hostname_port_arg, NULL, 10); + free(hostname_port_arg); + hostname_port_arg = NULL; } else { flags.port = strtol(address_string_end + 1, NULL, 10); /* Remove port part from hostname_port_arg*/ *address_string_end = '\0'; if (strlen(hostname_port_arg) > 0) { flags.hostname = hostname_port_arg; + } else { + free(hostname_port_arg); + hostname_port_arg = NULL; } } if (flags.port < 1 || flags.port > 65535) { + if (hostname_port_arg != NULL) { + free(hostname_port_arg); + } usage_error(progname, "-address"); } i++; @@ -208,6 +269,10 @@ int main(int argc, char *argv[]) start_timeout(timeout); i++; + } else if (strcmp(argv[i], "-fetch_stdout") == 0) { + flags.fetch_stdout = 1; + } else if (strcmp(argv[i], "-no_result_term") == 0) { + flags.print_result_term = 0; } else if (strcmp(argv[i], "-__uh_test__") == 0) { /* Fakes a failure in the call to ei_gethostbyname(h_hostname) so * we can test the localhost fallback. */ @@ -247,6 +312,9 @@ int main(int argc, char *argv[]) usage_arg(progname, "-c "); } flags.cookiep = 1; + if (flags.cookie != NULL) { + free(flags.cookie); + } flags.cookie = ei_chk_strdup(argv[i+1]); i++; break; @@ -254,6 +322,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-n "); } + if (flags.node != NULL) { + free(flags.node); + } flags.node = ei_chk_strdup(argv[i+1]); flags.use_long_name = 1; i++; @@ -262,6 +333,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-h "); } + if (flags.hidden != NULL) { + free(flags.hidden); + } flags.hidden = ei_chk_strdup(argv[i+1]); i++; break; @@ -269,6 +343,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-x "); } + if (flags.script != NULL) { + free(flags.script); + } flags.script = ei_chk_strdup(argv[i+1]); i++; break; @@ -276,6 +353,9 @@ int main(int argc, char *argv[]) if (i+1 >= argc) { usage_arg(progname, "-a "); } + if (flags.apply != NULL) { + free(flags.apply); + } flags.apply = ei_chk_strdup(argv[i+1]); i++; break; @@ -351,7 +431,7 @@ int main(int argc, char *argv[]) /* gethostname requires len to be max(hostname) + 1 */ if (gethostname(h_hostname, EI_MAXHOSTNAMELEN+1) < 0) { fprintf(stderr,"erl_call: failed to get host name: %d\n", errno); - exit(1); + exit_free_flags_fields(1, &flags); } if (flags.use_localhost_fallback || (hp = ei_gethostbyname(h_hostname)) == 0) { @@ -369,7 +449,7 @@ int main(int argc, char *argv[]) if (h_alivename) { if (strlen(h_alivename) + strlen(h_hostname) + 2 > sizeof(h_nodename_buf)) { fprintf(stderr,"erl_call: hostname too long: %s\n", h_hostname); - exit(1); + exit_free_flags_fields(1, &flags); } sprintf(h_nodename, "%s@%s", h_alivename, h_hostname); } @@ -383,7 +463,7 @@ int main(int argc, char *argv[]) (short) creation) < 0) { fprintf(stderr,"erl_call: can't create C node %s; %d\n", h_nodename, erl_errno); - exit(1); + exit_free_flags_fields(1, &flags); } } @@ -409,7 +489,7 @@ int main(int argc, char *argv[]) } else { if ((hp = ei_gethostbyname(host)) == 0) { fprintf(stderr,"erl_call: can't ei_gethostbyname(%s)\n", host); - exit(1); + exit_free_flags_fields(1, &flags); } format_node_hostname(&flags, hp->h_name, host_name); @@ -418,7 +498,7 @@ int main(int argc, char *argv[]) if (flags.port == -1) { if (strlen(flags.node) + strlen(host_name) + 2 > sizeof(nodename)) { fprintf(stderr,"erl_call: nodename too long: %s\n", flags.node); - exit(1); + exit_free_flags_fields(1, &flags); } sprintf(nodename, "%s@%s", flags.node, host_name); } @@ -433,11 +513,11 @@ int main(int argc, char *argv[]) /* We failed to connect ourself */ /* FIXME do we really know we failed because of node not up? */ if (flags.haltp) { - exit(0); + exit_free_flags_fields(0, &flags); } else { fprintf(stderr,"erl_call: failed to connect to node %s\n", nodename); - exit(1); + exit_free_flags_fields(1, &flags); } } } else { @@ -446,12 +526,12 @@ int main(int argc, char *argv[]) /* We failed to connect ourself */ /* FIXME do we really know we failed because of node not up? */ if (flags.haltp) { - exit(0); + exit_free_flags_fields(0, &flags); } else { fprintf(stderr,"erl_call: failed to connect to node with address \"%s:%ld\"\n", flags.hostname == NULL ? "" : flags.hostname, flags.port); - exit(1); + exit_free_flags_fields(1, &flags); } } } @@ -475,7 +555,7 @@ int main(int argc, char *argv[]) ei_rpc(&ec, fd, "erlang", "halt", p, i, &reply); free(p); ei_x_free(&reply); - exit(0); + exit_free_flags_fields(0, &flags); } if (flags.verbosep) { @@ -497,7 +577,7 @@ int main(int argc, char *argv[]) if (strlen(modname) + 4 + 1 > sizeof(fname)) { fprintf(stderr,"erl_call: module name too long: %s\n", modname); - exit(1); + exit_free_flags_fields(1, &flags); } strcpy(fname, modname); strcat(fname, ".erl"); @@ -531,7 +611,7 @@ int main(int argc, char *argv[]) ei_x_free(&reply); fprintf(stderr,"erl_call: can't write to source file %s\n", fname); - exit(1); + exit_free_flags_fields(1, &flags); } free(p); ei_x_free(&reply); @@ -596,7 +676,8 @@ int main(int argc, char *argv[]) len = read_stdin(&evalbuf); { int i = 0; - char *p; + int rpc_res; + char *p; ei_x_buff reply; ei_encode_list_header(NULL, &i, 1); @@ -611,19 +692,26 @@ int main(int argc, char *argv[]) ei_encode_empty_list(p, &i); ei_x_new_with_version(&reply); - /* erl_format("[~w]", erl_mk_binary(evalbuf,len))) */ - if (ei_rpc(&ec, fd, "erl_eval", "eval_str", p, i, &reply) < 0) { + if (flags.fetch_stdout) { + rpc_res = rpc_print_node_stdout(&ec, fd, "erl_eval", "eval_str", p, i, &reply); + } else { + rpc_res = ei_rpc(&ec, fd, "erl_eval", "eval_str", p, i, &reply); + } + + if (rpc_res < 0) { fprintf(stderr,"erl_call: evaluating input failed: %s\n", evalbuf); free(p); free(evalbuf); /* Allocated in read_stdin() */ ei_x_free(&reply); - exit(1); + exit_free_flags_fields(1, &flags); } - i = 0; - ei_print_term(stdout,reply.buff,&i); + if (flags.print_result_term) { + i = 0; + ei_print_term(stdout,reply.buff,&i); + } free(p); free(evalbuf); /* Allocated in read_stdin() */ ei_x_free(&reply); @@ -635,7 +723,7 @@ int main(int argc, char *argv[]) if (flags.apply != NULL) { char *mod,*fun,*args; ei_x_buff e, reply; - + int rpc_res; split_apply_string(flags.apply, &mod, &fun, &args); if (flags.verbosep) { fprintf(stderr,"erl_call: module = %s, function = %s, args = %s\n", @@ -646,24 +734,38 @@ int main(int argc, char *argv[]) if (ei_x_format_wo_ver(&e, args) < 0) { /* FIXME no error message and why -1 ? */ - exit(-1); + free(mod); + free(fun); + free(args); + exit_free_flags_fields(-1, &flags); } - + free(args); ei_x_new_with_version(&reply); - if (ei_rpc(&ec, fd, mod, fun, e.buff, e.index, &reply) < 0) { + if (flags.fetch_stdout) { + rpc_res = rpc_print_node_stdout(&ec, fd, mod, fun, e.buff, e.index, &reply); + } else { + rpc_res = ei_rpc(&ec, fd, mod, fun, e.buff, e.index, &reply); + } + if (rpc_res < 0) { /* FIXME no error message and why -1 ? */ ei_x_free(&e); ei_x_free(&reply); - exit(-1); + free(mod); + free(fun); + exit_free_flags_fields(-1, &flags); } else { - int i = 0; - ei_print_term(stdout,reply.buff,&i); + if (flags.print_result_term) { + int i = 0; + ei_print_term(stdout,reply.buff,&i); + } ei_x_free(&e); ei_x_free(&reply); } + free(mod); + free(fun); } - + exit_free_flags_fields(0, &flags); return(0); } @@ -720,7 +822,7 @@ static int do_connect(ei_cnode *ec, char *nodename, struct call_flags *flags) if ((r=erl_start_sys(ec,alive,(Erl_IpAddr)(h->h_addr_list[0]), start_flags,flags->script,args)) < 0) { fprintf(stderr,"erl_call: unable to start node, error = %d\n", r); - exit(1); + exit_free_flags_fields(1, flags); } if ((fd=ei_connect(ec, nodename)) >= 0) { @@ -734,19 +836,19 @@ static int do_connect(ei_cnode *ec, char *nodename, struct call_flags *flags) switch (fd) { case ERL_NO_DAEMON: fprintf(stderr,"erl_call: no epmd running\n"); - exit(1); + exit_free_flags_fields(1, flags); case ERL_CONNECT_FAIL: fprintf(stderr,"erl_call: connect failed\n"); - exit(1); + exit_free_flags_fields(1, flags); case ERL_NO_PORT: fprintf(stderr,"erl_call: node is not running\n"); - exit(1); + exit_free_flags_fields(1, flags); case ERL_TIMEOUT: fprintf(stderr,"erl_call: connect timed out\n"); - exit(1); + exit_free_flags_fields(1, flags); default: fprintf(stderr,"erl_call: error during connect\n"); - exit(1); + exit_free_flags_fields(1, flags); } } } @@ -908,6 +1010,13 @@ static void usage_noexit(const char *progname) { fprintf(stderr," -d direct Erlang output to ~/.erl_call.out.<Nodename>\n"); fprintf(stderr," -e evaluate contents of standard input (e.g., echo \"X=1,Y=2,{X,Y}.\"|%s -e ...)\n", progname); + fprintf(stderr, + " -fetch_stdout\n" + " execute the code, specified with the -a or -e option, in a new\n" + " process that has a group leader that forwards all stdout (standard\n" + " output) data so that it is printed to stdout of the\n" + " %s process. See the %s man page for additional information.\n", + progname, progname); fprintf(stderr," -h specify a name for the erl_call client node\n"); fprintf(stderr," -m read and compile Erlang module from stdin\n"); fprintf(stderr," -n name of Erlang node, same as -name\n"); @@ -918,6 +1027,7 @@ static void usage_noexit(const char *progname) { " (e.g., %s -address my_host:36303 ...)\n" " (cannot be combinated with -s, -n, -name and -sname)\n", progname); + fprintf(stderr," -no_result_term do not print the result term\n"); fprintf(stderr," -timeout command timeout, in seconds\n"); fprintf(stderr," -q halt the Erlang node (overrides the -s switch)\n"); fprintf(stderr," -r use a random name for the erl_call client node\n"); @@ -989,3 +1099,690 @@ static char* ei_chk_strdup(char *s) } return p; } + +/* + * Helper function that that: + * + * 1. Executes a function on a remote node + * + * 2. Forwards what the executed function and its subprocesses prints + * to stdout of this process + * + * 3. Returns the result of the executed function (the result term is + * written to the buffer pointed to by x) + * + * This function is similar to (and is based on) the function ei_rpc + */ +static int rpc_print_node_stdout(ei_cnode* ec, int fd, char *mod, + char *fun, const char* inbuf, + int inbuflen, ei_x_buff* x) +{ + int i, index; + int got_rex_response = 0; + int initial_buff_index = x->index; + ei_term t; + erlang_msg msg; + char rex[MAXATOMLEN]; + + if (ei_xrpc_to(ec, fd, mod, fun, inbuf, inbuflen, EI_RPC_FETCH_STDOUT) < 0) { + return ERL_ERROR; + } + + while (!got_rex_response) { + /* ei_rpc_from() responds with a tick if it gets one... */ + while ((i = ei_rpc_from(ec, fd, ERL_NO_TIMEOUT, &msg, x)) == ERL_TICK) + ; + + if (i == ERL_ERROR) return i; + + index = 0; + if (ei_decode_version(x->buff, &index, &i) < 0) + goto ebadmsg; + + if (ei_decode_ei_term(x->buff, &index, &t) < 0) + goto ebadmsg; + + if (t.ei_type != ERL_SMALL_TUPLE_EXT && t.ei_type != ERL_LARGE_TUPLE_EXT) + goto ebadmsg; + + if (t.arity != 2) + goto ebadmsg; + + if (ei_decode_atom(x->buff, &index, rex) < 0) + goto ebadmsg; + + if (strcmp("rex_stdout", rex) == 0) { + int type; + int size; + char* binary_buff; + long actual_size; + ei_get_type(x->buff, &index, &type, &size); + if(type != ERL_BINARY_EXT) { + goto ebadmsg; + } + binary_buff = ei_chk_malloc(size + 1); + ei_decode_binary(x->buff, &index, binary_buff, &actual_size); + binary_buff[size] = '\0'; + printf("%s", binary_buff); + free(binary_buff); + /* Reset the buffer as we need to read more and we have no + use for what we have already read */ + x->index = initial_buff_index; + } else { + if(strcmp("rex", rex) != 0) + goto ebadmsg; + got_rex_response = 1; + } + } + /* remove header */ + x->index -= index; + memmove(x->buff, &x->buff[index], x->index); + return 0; + +ebadmsg: + + return ERL_ERROR; +} + + +void exit_free_flags_fields(int exit_status, struct call_flags* flags) { + if (flags->script != NULL) { + free(flags->script); + } + if (flags->hidden != NULL) { + free(flags->hidden); + } + if (flags->cookie != NULL) { + free(flags->cookie); + } + if (flags->apply != NULL) { + free(flags->apply); + } + if (flags->hostname != NULL) { + free(flags->hostname); + } + exit(exit_status); +} + + +/* Constants and helper functions used by erl_start_sys */ + +/* FIXME is this a case a vfork can be used? */ +#if !HAVE_WORKING_VFORK +# define vfork fork +#endif + +#ifndef MAXPATHLEN +#define MAXPATHLEN 1024 +#endif + +#ifndef RSH +#define RSH "/usr/bin/ssh" +#endif + +#ifndef HAVE_SOCKLEN_T +typedef int SocklenType; +#else +typedef socklen_t SocklenType; +#endif + +/* FIXME check errors from malloc */ + +static struct in_addr *get_addr(const char *hostname, struct in_addr *oaddr); + +static int wait_for_erlang(int sockd, int magic, struct timeval *timeout); +#if defined(__WIN32__) +static int unique_id(void); +static HANDLE spawn_erlang_epmd(ei_cnode *ec, + char *alive, + Erl_IpAddr adr, + int flags, + char *erl_or_epmd, + char *args[], + int port, + int is_erlang); +#else +static int exec_erlang(ei_cnode *ec, char *alive, Erl_IpAddr adr, int flags, + char *erl, char *args[],int port); +#endif + +#if defined(__WIN32__) +#define DEF_ERL_COMMAND "erl" +#define DEF_EPMD_COMMAND "epmd" +#define ERL_REPLY_FMT "-s erl_reply reply \"%s\" \"%d\" \"%d\"" +#define ERL_NAME_FMT "-noinput -name %s" +#define ERL_SNAME_FMT "-noinput -sname %s" + +#define IP_ADDR_CHARS 15 +#define FORMATTED_INT_LEN 10 + +static int unique_id(void){ + return (int) GetCurrentThreadId(); +} + +static int enquote_args(char **oargs, char ***qargs){ + char **args; + int len; + int i; + int qwhole; + int extra; + char *ptr; + char *ptr2; + + if(oargs == NULL){ + *qargs = malloc(sizeof(char *)); + **qargs = NULL; + return 0; + }; + + for(len=0;oargs[len] != NULL; ++len) + ; + args = malloc(sizeof(char *) * (len + 1)); + + for(i = 0; i < len; ++i){ + qwhole = strchr(oargs[i],' ') != NULL; + extra = qwhole * 2; + for(ptr = oargs[i]; *ptr != '\0'; ++ptr) + extra += (*ptr == '"'); + args[i] = malloc(strlen(oargs[i]) + + extra + + 1); + ptr2 = args[i]; + if(qwhole) + *(ptr2++) = '"'; + for(ptr = oargs[i]; *ptr != '\0'; ++ptr){ + if(*ptr == '"') + *(ptr2++) = '\\'; + *(ptr2++) = *ptr; + } + if(qwhole) + *(ptr2++) = '"'; + *ptr2 = '\0'; + } + args[len] = NULL; + *qargs = args; + return len; +} + +static void free_args(char **args){ + char **ptr = args; + while(*ptr != NULL) + free(*(ptr++)); + free(args); +} + +/* In NT we cannot fork(), Erlang and Epmd gets + spawned by this function instead. */ + +static HANDLE spawn_erlang_epmd(ei_cnode *ec, + char *alive, + Erl_IpAddr adr, + int flags, + char *erl_or_epmd, + char *args[], + int port, + int is_erlang) +{ + STARTUPINFO sinfo; + SECURITY_ATTRIBUTES sa; + PROCESS_INFORMATION pinfo; + char *cmdbuf; + int cmdlen; + char *ptr; + int i; + int num_args; + char *name_format; + struct in_addr myaddr; + struct in_addr *hisaddr = (struct in_addr *)adr; + char iaddrbuf[IP_ADDR_CHARS + 1]; + HANDLE ret; + + if(is_erlang){ + get_addr(ei_thishostname(ec), &myaddr); + if((ptr = inet_ntoa(myaddr)) == NULL) + return INVALID_HANDLE_VALUE; + else + strcpy(iaddrbuf,ptr); + } + if ((flags & ERL_START_REMOTE) || + (is_erlang && (hisaddr->s_addr != myaddr.s_addr))) { + return INVALID_HANDLE_VALUE; + } else { + num_args = enquote_args(args, &args); + for(cmdlen = i = 0; args[i] != NULL; ++i) + cmdlen += strlen(args[i]) + 1; + if(!erl_or_epmd) + erl_or_epmd = (is_erlang) ? DEF_ERL_COMMAND : + DEF_EPMD_COMMAND; + if(is_erlang){ + name_format = (flags & ERL_START_LONG) ? ERL_NAME_FMT : + ERL_SNAME_FMT; + cmdlen += + strlen(erl_or_epmd) + (*erl_or_epmd != '\0') + + strlen(name_format) + 1 + strlen(alive) + + strlen(ERL_REPLY_FMT) + 1 + strlen(iaddrbuf) + 2 * FORMATTED_INT_LEN + 1; + ptr = cmdbuf = malloc(cmdlen); + if(*erl_or_epmd != '\0') + ptr += sprintf(ptr,"%s ",erl_or_epmd); + ptr += sprintf(ptr, name_format, + alive); + ptr += sprintf(ptr, " " ERL_REPLY_FMT, + iaddrbuf, port, unique_id()); + } else { /* epmd */ + cmdlen += strlen(erl_or_epmd) + (*erl_or_epmd != '\0') + 1; + ptr = cmdbuf = malloc(cmdlen); + if(*erl_or_epmd != '\0') + ptr += sprintf(ptr,"%s ",erl_or_epmd); + else + *(ptr++) = '\0'; + } + for(i= 0; args[i] != NULL; ++i){ + *(ptr++) = ' '; + strcpy(ptr,args[i]); + ptr += strlen(args[i]); + } + free_args(args); + if (flags & ERL_START_VERBOSE) { + fprintf(stderr,"erl_call: commands are %s\n",cmdbuf); + } + /* OK, one single command line... */ + /* Hmmm, hidden or unhidden window??? */ + memset(&sinfo,0,sizeof(sinfo)); + sinfo.cb = sizeof(STARTUPINFO); + sinfo.dwFlags = STARTF_USESHOWWINDOW /*| + STARTF_USESTDHANDLES*/; + sinfo.wShowWindow = SW_HIDE; /* Hidden! */ + sinfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + sinfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + sinfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + sa.nLength = sizeof(sa); + sa.lpSecurityDescriptor = NULL; + sa.bInheritHandle = /*TRUE*/ FALSE; + if(!CreateProcess( + NULL, + cmdbuf, + &sa, + NULL, + /*TRUE*/ FALSE, + 0 | CREATE_NEW_CONSOLE, + NULL, + NULL, + &sinfo, + &pinfo)) + ret = INVALID_HANDLE_VALUE; + else + ret = pinfo.hProcess; + free(cmdbuf); + return ret; + } + /* NOTREACHED */ +} +#else /* Unix */ + +/* call this from the child process to start an erlang system. This + * function just builds the erlang command line and then calls it. + * + * node - the nodename for the new node + * flags - various options that can be set (see erl_start.h) + * erl - name of the erlang executable, or NULL for default ("erl") + * args - additional arguments to pass to erlang executable + * port - the port number where we wait for acknowledgment from the enode + * + * we have a potential problem if args conflicts with any of the + * arguments we use here. + */ +static int exec_erlang(ei_cnode *ec, + char *alive, + Erl_IpAddr adr, + int flags, + char *erl, + char *args[], + int port) +{ +#if !defined(__WIN32__) + int fd,len,l,i; + char **s; + char *argv[4]; + char argbuf[BUFSIZ]; + struct in_addr myaddr; + struct in_addr *hisaddr = (struct in_addr *)adr; + + if (!get_addr(ei_thishostname(ec), &myaddr)) { + fprintf(stderr,"erl_call: failed to find hostname\r\n"); + return ERL_SYS_ERROR; + } + + /* on this host? */ + /* compare ip addresses, unless forced by flag setting to use rsh */ + if ((flags & ERL_START_REMOTE) || (hisaddr->s_addr != myaddr.s_addr)) { + argv[0] = RSH; + len = strlen(inet_ntoa(*hisaddr)); + argv[1] = malloc(len+1); + strcpy(argv[1],inet_ntoa(*hisaddr)); + } + else { + /* Yes - use sh to start local Erlang */ + argv[0] = "sh"; + argv[1] = "-c"; + } + argv[2] = argbuf; + argv[3] = NULL; + + len = 0; + *argbuf=(char)0; + + sprintf(argbuf,"exec %s ", (erl? erl: "erl")); + len = strlen(argbuf); + + /* *must* be noinput or node (seems to) hang... */ + /* long or short names? */ + sprintf(&argbuf[len], "-noinput %s %s ", + ((flags & ERL_START_LONG) ? "-name" : "-sname"), + alive); + len = strlen(argbuf); + + /* now make the new node report back when it's ready */ + /* add: myip, myport and replymsg */ + sprintf(&argbuf[len], + "-s erl_reply reply %s %d %d ", + inet_ntoa(myaddr),port,(int)getpid()); +#ifdef DEBUG + fprintf(stderr,"erl_call: debug %s\n",&argbuf[len]); +#endif + len = strlen(argbuf); + + /* additional arguments to be passed to the other system */ + /* make sure that they will fit first */ + for (l=0, s = args; s && *s; s++) l+= strlen(*s) + 1; + + if (len + l + 1 > BUFSIZ) return ERL_BADARG; + else { + for (s = args; s && *s; s++) { + strcat(argbuf," "); + strcat(argbuf,*s); + } + len += l + 1; + } + + if (flags & ERL_START_VERBOSE) { + fprintf(stderr,"erl_call: %s %s %s\n",argv[0],argv[1],argv[2]); + } + + /* close all descriptors in child */ + for (i=0; i<64; i++) close(i); + + /* debug output to file? */ + if (flags & ERL_START_DEBUG) { + char debugfile[MAXPATHLEN+1]; + char *home=getenv("HOME"); + sprintf(debugfile,"%s/%s.%s",home,ERL_START_LOGFILE,alive); + if ((fd=open(debugfile, O_WRONLY | O_CREAT | O_APPEND, 0644)) >= 0) { + time_t t = time(NULL); + dup2(fd,1); + dup2(fd,2); + fprintf(stderr,"\n\n===== Log started ======\n%s \n",ctime(&t)); + fprintf(stderr,"erl_call: %s %s %s\n",argv[0],argv[1],argv[2]); + } + } + + /* start the system */ + execvp(argv[0], argv); + + if (flags & ERL_START_DEBUG) { + fprintf(stderr,"erl_call: exec failed: (%d) %s %s %s\n", + errno,argv[0],argv[1],argv[2]); + } + +#endif + /* (hopefully) NOT REACHED */ + return ERL_SYS_ERROR; +} /* exec_erlang() */ + +#endif /* defined(WINDOWS) */ + +#if defined(__WIN32__) +static void gettimeofday(struct timeval *now,void *dummy){ + SYSTEMTIME systime; + FILETIME ft; + DWORD x; + GetSystemTime(&systime); + SystemTimeToFileTime(&systime,&ft); + x = ft.dwLowDateTime / 10; + now->tv_sec = x / 1000000; + now->tv_usec = x % 1000000; +} + +#endif + + +/* wait for the remote system to reply */ +/* + * sockd - an open socket where we expect a connection from the e-node + * magic - sign on message the e-node must provide for verification + * timeout - how long to wait before returning failure + * + * OBS: the socket is blocking, and there is a potential deadlock if we + * get an accept but the peer sends no data (and does not close). + * in normal cases the timeout will work ok however, i.e. either we + * never get any connection, or we get connection then close(). + */ +static int wait_for_erlang(int sockd, int magic, struct timeval *timeout) +{ + struct timeval to; + struct timeval stop_time; + struct timeval now; + fd_set rdset; + int fd; + int n,i; + char buf[16]; + struct sockaddr_in peer; + SocklenType len = (SocklenType) sizeof(peer); + + /* determine when we should exit this function */ + gettimeofday(&now,NULL); + stop_time.tv_sec = now.tv_sec + timeout->tv_sec; + stop_time.tv_usec = now.tv_usec + timeout->tv_usec; + while (stop_time.tv_usec > 1000000) { + stop_time.tv_sec++; + stop_time.tv_usec -= 1000000; + } + +#ifdef DEBUG + fprintf(stderr,"erl_call: debug time is %ld.%06ld, " + "will timeout at %ld.%06ld\n", + now.tv_sec,now.tv_usec,stop_time.tv_sec,stop_time.tv_usec); +#endif + + while (1) { + FD_ZERO(&rdset); + FD_SET(sockd,&rdset); + + /* adjust the timeout to (stoptime - now) */ + gettimeofday(&now,NULL); + to.tv_sec = stop_time.tv_sec - now.tv_sec; + to.tv_usec = stop_time.tv_usec - now.tv_usec; + while ((to.tv_usec < 0) && (to.tv_sec > 0)) { + to.tv_usec += 1000000; + to.tv_sec--; + } + if (to.tv_sec < 0) return ERL_TIMEOUT; + +#ifdef DEBUG + fprintf(stderr,"erl_call: debug remaining to timeout: %ld.%06ld\n", + to.tv_sec,to.tv_usec); +#endif + switch ((i = select(sockd+1,&rdset,NULL,NULL,&to))) { + case -1: + return ERL_SYS_ERROR; + break; + + case 0: /* timeout */ +#ifdef DEBUG + gettimeofday(&now,NULL); + fprintf(stderr,"erl_call: debug timed out at %ld.%06ld\n", + now.tv_sec,now.tv_usec); +#endif + return ERL_TIMEOUT; + break; + + default: /* ready descriptors */ +#ifdef DEBUG + gettimeofday(&now,NULL); + fprintf(stderr,"erl_call: debug got select at %ld.%06ld\n", + now.tv_sec,now.tv_usec); +#endif + if (FD_ISSET(sockd,&rdset)) { + if ((fd = accept(sockd,(struct sockaddr *)&peer,&len)) < 0) + return ERL_SYS_ERROR; + + /* now get sign-on message and terminate it */ +#if defined(__WIN32__) + if ((n=recv(fd,buf,16,0)) >= 0) buf[n]=0x0; + closesocket(fd); +#else + if ((n=read(fd,buf,16)) >= 0) buf[n]=0x0; + close(fd); +#endif +#ifdef DEBUG + fprintf(stderr,"erl_call: debug got %d, expected %d\n", + atoi(buf),magic); +#endif + if (atoi(buf) == magic) return 0; /* success */ + } /* if FD_SET */ + } /* switch */ + } /* while */ + + /* unreached? */ + return ERL_SYS_ERROR; +} /* wait_for_erlang() */ + + +static struct in_addr *get_addr(const char *hostname, struct in_addr *oaddr) +{ + struct hostent *hp; + +#if !defined (__WIN32__) + char buf[1024]; + struct hostent host; + int herror; + + hp = ei_gethostbyname_r(hostname,&host,buf,1024,&herror); +#else + hp = ei_gethostbyname(hostname); +#endif + + if (hp) { + memmove(oaddr,hp->h_addr_list[0],sizeof(*oaddr)); + return oaddr; + } + return NULL; +} + +/* Start an Erlang node. return value 0 indicates that node was + * started successfully, negative values indicate error. + * + * node - the name of the remote node to start (alivename@hostname). + * flags - turn on or off certain options. See erl_start.h for a list. + * erl - is the name of the erl script to call. If NULL, the default + * name "erl" will be used. + * args - a NULL-terminated list of strings containing + * additional arguments to be sent to the remote Erlang node. These + * strings are simply appended to the end of the command line, so any + * quoting of special characters, etc must be done by the caller. + * There may be some conflicts between some of these arguments and the + * default arguments hard-coded into this function, so be careful. + */ +int erl_start_sys(ei_cnode *ec, char *alive, Erl_IpAddr adr, int flags, + char *erl, char *args[]) +{ + struct timeval timeout; + struct sockaddr_in addr; + SocklenType namelen; + int port; + int sockd = 0; + int one = 1; +#if defined(__WIN32__) + HANDLE pid; +#else + int pid; +#endif + int r = 0; + + if (((sockd = socket(AF_INET, SOCK_STREAM, 0)) < 0) || + (setsockopt(sockd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(one)) < 0)) { + r = ERL_SYS_ERROR; + goto done; + } + + memset(&addr,0,sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = 0; + + if (bind(sockd,(struct sockaddr *)&addr,sizeof(addr))<0) { + return ERL_SYS_ERROR; + } + namelen = sizeof(addr); + if (getsockname(sockd,(struct sockaddr *)&addr,&namelen)<0) { + return ERL_SYS_ERROR; + } + port = ntohs(addr.sin_port); + + listen(sockd,5); + +#if defined(__WIN32__) + pid = spawn_erlang_epmd(ec,alive,adr,flags,erl,args,port,1); + if (pid == INVALID_HANDLE_VALUE) + return ERL_SYS_ERROR; + timeout.tv_usec = 0; + timeout.tv_sec = 10; /* ignoring ERL_START_TIME */ + if((r = wait_for_erlang(sockd,unique_id(),&timeout)) + == ERL_TIMEOUT) { + /* Well, this is not a nice way to do it, and it does not + always kill the emulator, but the alternatives are few.*/ + TerminateProcess(pid,1); + } +#else /* Unix */ + switch ((pid = fork())) { + case -1: + r = ERL_SYS_ERROR; + break; + + case 0: + /* child - start the erlang node */ + exec_erlang(ec, alive, adr, flags, erl, args, port); + + /* error if reached - parent reports back to caller after timeout + so we just exit here */ + exit(1); + break; + + default: + + /* parent - waits for response from Erlang node */ + /* child pid used here as magic number */ + timeout.tv_usec = 0; + timeout.tv_sec = 10; /* ignoring ERL_START_TIME */ + if ((r = wait_for_erlang(sockd,pid,&timeout)) == ERL_TIMEOUT) { + /* kill child if no response */ + kill(pid,SIGINT); + sleep(1); + if (waitpid(pid,NULL,WNOHANG) != pid) { + /* no luck - try harder */ + kill(pid,SIGKILL); + sleep(1); + waitpid(pid,NULL,WNOHANG); + } + } + + } +#endif /* defined(__WIN32__) */ + +done: +#if defined(__WIN32__) + if (sockd) closesocket(sockd); +#else + if (sockd) close(sockd); +#endif + return r; +} /* erl_start_sys() */ diff --git a/lib/erl_interface/src/prog/erl_start.c b/lib/erl_interface/src/prog/erl_start.c deleted file mode 100644 index 9c876feb5e..0000000000 --- a/lib/erl_interface/src/prog/erl_start.c +++ /dev/null @@ -1,634 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2020. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - * - */ - -/* An exception from using eidef.h, use config.h directly */ -#include "config.h" - -#include <stdlib.h> -#include <sys/types.h> -#include <fcntl.h> - -#ifdef __WIN32__ -#include <winsock2.h> -#include <windows.h> -#include <winbase.h> - -#else /* unix */ -#include <errno.h> -#include <netdb.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <signal.h> -#include <stdio.h> -#include <string.h> -#include <sys/param.h> -#include <sys/socket.h> -#include <sys/stat.h> -#include <sys/wait.h> -#include <sys/time.h> -#include <time.h> -#include <unistd.h> -#include <sys/types.h> -#include <signal.h> -#endif - -#include "ei.h" -#include "ei_resolve.h" -#include "erl_start.h" - -/* FIXME is this a case a vfork can be used? */ -#if !HAVE_WORKING_VFORK -# define vfork fork -#endif - -#ifndef MAXPATHLEN -#define MAXPATHLEN 1024 -#endif - -#ifndef RSH -#define RSH "/usr/bin/ssh" -#endif - -#ifndef HAVE_SOCKLEN_T -typedef int SocklenType; -#else -typedef socklen_t SocklenType; -#endif - -/* FIXME check errors from malloc */ - -static struct in_addr *get_addr(const char *hostname, struct in_addr *oaddr); - -static int wait_for_erlang(int sockd, int magic, struct timeval *timeout); -#if defined(__WIN32__) -static int unique_id(void); -static HANDLE spawn_erlang_epmd(ei_cnode *ec, - char *alive, - Erl_IpAddr adr, - int flags, - char *erl_or_epmd, - char *args[], - int port, - int is_erlang); -#else -static int exec_erlang(ei_cnode *ec, char *alive, Erl_IpAddr adr, int flags, - char *erl, char *args[],int port); -#endif -/* Start an Erlang node. return value 0 indicates that node was - * started successfully, negative values indicate error. - * - * node - the name of the remote node to start (alivename@hostname). - * flags - turn on or off certain options. See erl_start.h for a list. - * erl - is the name of the erl script to call. If NULL, the default - * name "erl" will be used. - * args - a NULL-terminated list of strings containing - * additional arguments to be sent to the remote Erlang node. These - * strings are simply appended to the end of the command line, so any - * quoting of special characters, etc must be done by the caller. - * There may be some conflicts between some of these arguments and the - * default arguments hard-coded into this function, so be careful. - */ -int erl_start_sys(ei_cnode *ec, char *alive, Erl_IpAddr adr, int flags, - char *erl, char *args[]) -{ - struct timeval timeout; - struct sockaddr_in addr; - SocklenType namelen; - int port; - int sockd = 0; - int one = 1; -#if defined(__WIN32__) - HANDLE pid; -#else - int pid; -#endif - int r = 0; - - if (((sockd = socket(AF_INET, SOCK_STREAM, 0)) < 0) || - (setsockopt(sockd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(one)) < 0)) { - r = ERL_SYS_ERROR; - goto done; - } - - memset(&addr,0,sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_ANY); - addr.sin_port = 0; - - if (bind(sockd,(struct sockaddr *)&addr,sizeof(addr))<0) { - return ERL_SYS_ERROR; - } - namelen = sizeof(addr); - if (getsockname(sockd,(struct sockaddr *)&addr,&namelen)<0) { - return ERL_SYS_ERROR; - } - port = ntohs(addr.sin_port); - - listen(sockd,5); - -#if defined(__WIN32__) - pid = spawn_erlang_epmd(ec,alive,adr,flags,erl,args,port,1); - if (pid == INVALID_HANDLE_VALUE) - return ERL_SYS_ERROR; - timeout.tv_usec = 0; - timeout.tv_sec = 10; /* ignoring ERL_START_TIME */ - if((r = wait_for_erlang(sockd,unique_id(),&timeout)) - == ERL_TIMEOUT) { - /* Well, this is not a nice way to do it, and it does not - always kill the emulator, but the alternatives are few.*/ - TerminateProcess(pid,1); - } -#else /* Unix */ - switch ((pid = fork())) { - case -1: - r = ERL_SYS_ERROR; - break; - - case 0: - /* child - start the erlang node */ - exec_erlang(ec, alive, adr, flags, erl, args, port); - - /* error if reached - parent reports back to caller after timeout - so we just exit here */ - exit(1); - break; - - default: - - /* parent - waits for response from Erlang node */ - /* child pid used here as magic number */ - timeout.tv_usec = 0; - timeout.tv_sec = 10; /* ignoring ERL_START_TIME */ - if ((r = wait_for_erlang(sockd,pid,&timeout)) == ERL_TIMEOUT) { - /* kill child if no response */ - kill(pid,SIGINT); - sleep(1); - if (waitpid(pid,NULL,WNOHANG) != pid) { - /* no luck - try harder */ - kill(pid,SIGKILL); - sleep(1); - waitpid(pid,NULL,WNOHANG); - } - } - - } -#endif /* defined(__WIN32__) */ - -done: -#if defined(__WIN32__) - if (sockd) closesocket(sockd); -#else - if (sockd) close(sockd); -#endif - return r; -} /* erl_start_sys() */ - -#if defined(__WIN32__) -#define DEF_ERL_COMMAND "erl" -#define DEF_EPMD_COMMAND "epmd" -#define ERL_REPLY_FMT "-s erl_reply reply \"%s\" \"%d\" \"%d\"" -#define ERL_NAME_FMT "-noinput -name %s" -#define ERL_SNAME_FMT "-noinput -sname %s" - -#define IP_ADDR_CHARS 15 -#define FORMATTED_INT_LEN 10 - -static int unique_id(void){ - return (int) GetCurrentThreadId(); -} - -static int enquote_args(char **oargs, char ***qargs){ - char **args; - int len; - int i; - int qwhole; - int extra; - char *ptr; - char *ptr2; - - if(oargs == NULL){ - *qargs = malloc(sizeof(char *)); - **qargs = NULL; - return 0; - }; - - for(len=0;oargs[len] != NULL; ++len) - ; - args = malloc(sizeof(char *) * (len + 1)); - - for(i = 0; i < len; ++i){ - qwhole = strchr(oargs[i],' ') != NULL; - extra = qwhole * 2; - for(ptr = oargs[i]; *ptr != '\0'; ++ptr) - extra += (*ptr == '"'); - args[i] = malloc(strlen(oargs[i]) + - extra + - 1); - ptr2 = args[i]; - if(qwhole) - *(ptr2++) = '"'; - for(ptr = oargs[i]; *ptr != '\0'; ++ptr){ - if(*ptr == '"') - *(ptr2++) = '\\'; - *(ptr2++) = *ptr; - } - if(qwhole) - *(ptr2++) = '"'; - *ptr2 = '\0'; - } - args[len] = NULL; - *qargs = args; - return len; -} - -static void free_args(char **args){ - char **ptr = args; - while(*ptr != NULL) - free(*(ptr++)); - free(args); -} - -/* In NT we cannot fork(), Erlang and Epmd gets - spawned by this function instead. */ - -static HANDLE spawn_erlang_epmd(ei_cnode *ec, - char *alive, - Erl_IpAddr adr, - int flags, - char *erl_or_epmd, - char *args[], - int port, - int is_erlang) -{ - STARTUPINFO sinfo; - SECURITY_ATTRIBUTES sa; - PROCESS_INFORMATION pinfo; - char *cmdbuf; - int cmdlen; - char *ptr; - int i; - int num_args; - char *name_format; - struct in_addr myaddr; - struct in_addr *hisaddr = (struct in_addr *)adr; - char iaddrbuf[IP_ADDR_CHARS + 1]; - HANDLE ret; - - if(is_erlang){ - get_addr(ei_thishostname(ec), &myaddr); - if((ptr = inet_ntoa(myaddr)) == NULL) - return INVALID_HANDLE_VALUE; - else - strcpy(iaddrbuf,ptr); - } - if ((flags & ERL_START_REMOTE) || - (is_erlang && (hisaddr->s_addr != myaddr.s_addr))) { - return INVALID_HANDLE_VALUE; - } else { - num_args = enquote_args(args, &args); - for(cmdlen = i = 0; args[i] != NULL; ++i) - cmdlen += strlen(args[i]) + 1; - if(!erl_or_epmd) - erl_or_epmd = (is_erlang) ? DEF_ERL_COMMAND : - DEF_EPMD_COMMAND; - if(is_erlang){ - name_format = (flags & ERL_START_LONG) ? ERL_NAME_FMT : - ERL_SNAME_FMT; - cmdlen += - strlen(erl_or_epmd) + (*erl_or_epmd != '\0') + - strlen(name_format) + 1 + strlen(alive) + - strlen(ERL_REPLY_FMT) + 1 + strlen(iaddrbuf) + 2 * FORMATTED_INT_LEN + 1; - ptr = cmdbuf = malloc(cmdlen); - if(*erl_or_epmd != '\0') - ptr += sprintf(ptr,"%s ",erl_or_epmd); - ptr += sprintf(ptr, name_format, - alive); - ptr += sprintf(ptr, " " ERL_REPLY_FMT, - iaddrbuf, port, unique_id()); - } else { /* epmd */ - cmdlen += strlen(erl_or_epmd) + (*erl_or_epmd != '\0') + 1; - ptr = cmdbuf = malloc(cmdlen); - if(*erl_or_epmd != '\0') - ptr += sprintf(ptr,"%s ",erl_or_epmd); - else - *(ptr++) = '\0'; - } - for(i= 0; args[i] != NULL; ++i){ - *(ptr++) = ' '; - strcpy(ptr,args[i]); - ptr += strlen(args[i]); - } - free_args(args); - if (flags & ERL_START_VERBOSE) { - fprintf(stderr,"erl_call: commands are %s\n",cmdbuf); - } - /* OK, one single command line... */ - /* Hmmm, hidden or unhidden window??? */ - memset(&sinfo,0,sizeof(sinfo)); - sinfo.cb = sizeof(STARTUPINFO); - sinfo.dwFlags = STARTF_USESHOWWINDOW /*| - STARTF_USESTDHANDLES*/; - sinfo.wShowWindow = SW_HIDE; /* Hidden! */ - sinfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); - sinfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); - sinfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); - sa.nLength = sizeof(sa); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = /*TRUE*/ FALSE; - if(!CreateProcess( - NULL, - cmdbuf, - &sa, - NULL, - /*TRUE*/ FALSE, - 0 | CREATE_NEW_CONSOLE, - NULL, - NULL, - &sinfo, - &pinfo)) - ret = INVALID_HANDLE_VALUE; - else - ret = pinfo.hProcess; - free(cmdbuf); - return ret; - } - /* NOTREACHED */ -} -#else /* Unix */ - -/* call this from the child process to start an erlang system. This - * function just builds the erlang command line and then calls it. - * - * node - the nodename for the new node - * flags - various options that can be set (see erl_start.h) - * erl - name of the erlang executable, or NULL for default ("erl") - * args - additional arguments to pass to erlang executable - * port - the port number where we wait for acknowledgment from the enode - * - * we have a potential problem if args conflicts with any of the - * arguments we use here. - */ -static int exec_erlang(ei_cnode *ec, - char *alive, - Erl_IpAddr adr, - int flags, - char *erl, - char *args[], - int port) -{ -#if !defined(__WIN32__) - int fd,len,l,i; - char **s; - char *argv[4]; - char argbuf[BUFSIZ]; - struct in_addr myaddr; - struct in_addr *hisaddr = (struct in_addr *)adr; - - if (!get_addr(ei_thishostname(ec), &myaddr)) { - fprintf(stderr,"erl_call: failed to find hostname\r\n"); - return ERL_SYS_ERROR; - } - - /* on this host? */ - /* compare ip addresses, unless forced by flag setting to use rsh */ - if ((flags & ERL_START_REMOTE) || (hisaddr->s_addr != myaddr.s_addr)) { - argv[0] = RSH; - len = strlen(inet_ntoa(*hisaddr)); - argv[1] = malloc(len+1); - strcpy(argv[1],inet_ntoa(*hisaddr)); - } - else { - /* Yes - use sh to start local Erlang */ - argv[0] = "sh"; - argv[1] = "-c"; - } - argv[2] = argbuf; - argv[3] = NULL; - - len = 0; - *argbuf=(char)0; - - sprintf(argbuf,"exec %s ", (erl? erl: "erl")); - len = strlen(argbuf); - - /* *must* be noinput or node (seems to) hang... */ - /* long or short names? */ - sprintf(&argbuf[len], "-noinput %s %s ", - ((flags & ERL_START_LONG) ? "-name" : "-sname"), - alive); - len = strlen(argbuf); - - /* now make the new node report back when it's ready */ - /* add: myip, myport and replymsg */ - sprintf(&argbuf[len], - "-s erl_reply reply %s %d %d ", - inet_ntoa(myaddr),port,(int)getpid()); -#ifdef DEBUG - fprintf(stderr,"erl_call: debug %s\n",&argbuf[len]); -#endif - len = strlen(argbuf); - - /* additional arguments to be passed to the other system */ - /* make sure that they will fit first */ - for (l=0, s = args; s && *s; s++) l+= strlen(*s) + 1; - - if (len + l + 1 > BUFSIZ) return ERL_BADARG; - else { - for (s = args; s && *s; s++) { - strcat(argbuf," "); - strcat(argbuf,*s); - } - len += l + 1; - } - - if (flags & ERL_START_VERBOSE) { - fprintf(stderr,"erl_call: %s %s %s\n",argv[0],argv[1],argv[2]); - } - - /* close all descriptors in child */ - for (i=0; i<64; i++) close(i); - - /* debug output to file? */ - if (flags & ERL_START_DEBUG) { - char debugfile[MAXPATHLEN+1]; - char *home=getenv("HOME"); - sprintf(debugfile,"%s/%s.%s",home,ERL_START_LOGFILE,alive); - if ((fd=open(debugfile, O_WRONLY | O_CREAT | O_APPEND, 0644)) >= 0) { - time_t t = time(NULL); - dup2(fd,1); - dup2(fd,2); - fprintf(stderr,"\n\n===== Log started ======\n%s \n",ctime(&t)); - fprintf(stderr,"erl_call: %s %s %s\n",argv[0],argv[1],argv[2]); - } - } - - /* start the system */ - execvp(argv[0], argv); - - if (flags & ERL_START_DEBUG) { - fprintf(stderr,"erl_call: exec failed: (%d) %s %s %s\n", - errno,argv[0],argv[1],argv[2]); - } - -#endif - /* (hopefully) NOT REACHED */ - return ERL_SYS_ERROR; -} /* exec_erlang() */ - -#endif /* defined(WINDOWS) */ - -#if defined(__WIN32__) -static void gettimeofday(struct timeval *now,void *dummy){ - SYSTEMTIME systime; - FILETIME ft; - DWORD x; - GetSystemTime(&systime); - SystemTimeToFileTime(&systime,&ft); - x = ft.dwLowDateTime / 10; - now->tv_sec = x / 1000000; - now->tv_usec = x % 1000000; -} - -#endif - - -/* wait for the remote system to reply */ -/* - * sockd - an open socket where we expect a connection from the e-node - * magic - sign on message the e-node must provide for verification - * timeout - how long to wait before returning failure - * - * OBS: the socket is blocking, and there is a potential deadlock if we - * get an accept but the peer sends no data (and does not close). - * in normal cases the timeout will work ok however, i.e. either we - * never get any connection, or we get connection then close(). - */ -static int wait_for_erlang(int sockd, int magic, struct timeval *timeout) -{ - struct timeval to; - struct timeval stop_time; - struct timeval now; - fd_set rdset; - int fd; - int n,i; - char buf[16]; - struct sockaddr_in peer; - SocklenType len = (SocklenType) sizeof(peer); - - /* determine when we should exit this function */ - gettimeofday(&now,NULL); - stop_time.tv_sec = now.tv_sec + timeout->tv_sec; - stop_time.tv_usec = now.tv_usec + timeout->tv_usec; - while (stop_time.tv_usec > 1000000) { - stop_time.tv_sec++; - stop_time.tv_usec -= 1000000; - } - -#ifdef DEBUG - fprintf(stderr,"erl_call: debug time is %ld.%06ld, " - "will timeout at %ld.%06ld\n", - now.tv_sec,now.tv_usec,stop_time.tv_sec,stop_time.tv_usec); -#endif - - while (1) { - FD_ZERO(&rdset); - FD_SET(sockd,&rdset); - - /* adjust the timeout to (stoptime - now) */ - gettimeofday(&now,NULL); - to.tv_sec = stop_time.tv_sec - now.tv_sec; - to.tv_usec = stop_time.tv_usec - now.tv_usec; - while ((to.tv_usec < 0) && (to.tv_sec > 0)) { - to.tv_usec += 1000000; - to.tv_sec--; - } - if (to.tv_sec < 0) return ERL_TIMEOUT; - -#ifdef DEBUG - fprintf(stderr,"erl_call: debug remaining to timeout: %ld.%06ld\n", - to.tv_sec,to.tv_usec); -#endif - switch ((i = select(sockd+1,&rdset,NULL,NULL,&to))) { - case -1: - return ERL_SYS_ERROR; - break; - - case 0: /* timeout */ -#ifdef DEBUG - gettimeofday(&now,NULL); - fprintf(stderr,"erl_call: debug timed out at %ld.%06ld\n", - now.tv_sec,now.tv_usec); -#endif - return ERL_TIMEOUT; - break; - - default: /* ready descriptors */ -#ifdef DEBUG - gettimeofday(&now,NULL); - fprintf(stderr,"erl_call: debug got select at %ld.%06ld\n", - now.tv_sec,now.tv_usec); -#endif - if (FD_ISSET(sockd,&rdset)) { - if ((fd = accept(sockd,(struct sockaddr *)&peer,&len)) < 0) - return ERL_SYS_ERROR; - - /* now get sign-on message and terminate it */ -#if defined(__WIN32__) - if ((n=recv(fd,buf,16,0)) >= 0) buf[n]=0x0; - closesocket(fd); -#else - if ((n=read(fd,buf,16)) >= 0) buf[n]=0x0; - close(fd); -#endif -#ifdef DEBUG - fprintf(stderr,"erl_call: debug got %d, expected %d\n", - atoi(buf),magic); -#endif - if (atoi(buf) == magic) return 0; /* success */ - } /* if FD_SET */ - } /* switch */ - } /* while */ - - /* unreached? */ - return ERL_SYS_ERROR; -} /* wait_for_erlang() */ - - -static struct in_addr *get_addr(const char *hostname, struct in_addr *oaddr) -{ - struct hostent *hp; - -#if !defined (__WIN32__) - char buf[1024]; - struct hostent host; - int herror; - - hp = ei_gethostbyname_r(hostname,&host,buf,1024,&herror); -#else - hp = ei_gethostbyname(hostname); -#endif - - if (hp) { - memmove(oaddr,hp->h_addr_list[0],sizeof(*oaddr)); - return oaddr; - } - return NULL; -} diff --git a/lib/erl_interface/src/prog/erl_start.h b/lib/erl_interface/src/prog/erl_start.h deleted file mode 100644 index 1d0d584c45..0000000000 --- a/lib/erl_interface/src/prog/erl_start.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - * - - */ -#ifndef _ERL_START_H -#define _ERL_START_H - -#define ERL_START_MSG "gurka" /* make something up */ -#define ERL_START_TIME 10000 /* wait this long (ms) */ -#define ERL_START_LOGFILE ".erl_start.out" /* basename of logfile */ - -/* flags used by erl_connect and erl_xconnect */ -#define ERL_START_ENODE 0x0001 -#define ERL_START_EPMD 0x0002 -#define ERL_START_LONG 0x0004 -#define ERL_START_COOKIE 0x0008 -#define ERL_START_DEBUG 0x0010 -#define ERL_START_VERBOSE 0x0020 -#define ERL_START_REMOTE 0x0040 - -/* error return values */ -#define ERL_S_TIMEOUT -51 /* a timeout occurred */ -#define ERL_BADARG -52 /* an argument contained an incorrect value */ -#define ERL_SYS_ERROR -99 /* a system error occurred (check errno) */ - -/* start an erlang system */ -int erl_start_sys(ei_cnode *ec, char *alive, Erl_IpAddr addr, int flags, - char *erl, char *add_args[]); - -#endif /* _ERL_START_H */ diff --git a/lib/erl_interface/test/erl_call_SUITE.erl b/lib/erl_interface/test/erl_call_SUITE.erl index 9cfc2ac25c..8f1d0c9f90 100644 --- a/lib/erl_interface/test/erl_call_SUITE.erl +++ b/lib/erl_interface/test/erl_call_SUITE.erl @@ -27,14 +27,16 @@ random_cnode_name/1, test_connect_to_host_port/1, unresolvable_hostname/1, - timeout/1]). + timeout/1, + test_fetch_stdout/1]). all() -> [smoke, random_cnode_name, test_connect_to_host_port, unresolvable_hostname, - timeout]. + timeout, + test_fetch_stdout]. smoke(Config) when is_list(Config) -> Name = atom_to_list(?MODULE) @@ -129,6 +131,113 @@ test_connect_to_host_port_do(Name) -> end, ok. +test_fetch_stdout(Config) when is_list(Config) -> + Name = atom_to_list(?MODULE) + ++ "-" + ++ "fetch_stdout" + ++ "-" + ++ integer_to_list(erlang:system_time(microsecond)), + try + test_fetch_stdout_do(Name) + after + halt_node(Name) + end, + ok. + + +test_fetch_stdout_do(Name) -> + Port = start_node_and_get_port(Name), + %% Test that the -fetch_stdout option works + "hejok" = get_erl_call_result(["-address", + erlang:integer_to_list(Port), + "-a", + "io format [[104,101,106]]", + "-fetch_stdout"]), + %% Test that the -fetch_stdout option works together with + %% -no_result_term + "hej" = get_erl_call_result(["-address", + erlang:integer_to_list(Port), + "-a", + "io format [[104,101,106]]", + "-fetch_stdout", + "-no_result_term"]), + %% Test that we can print several times + MultiPrintCodeStr = + "io:format(\"hej\"),io:format(\"hej\"),io:format(\"hej\").\n", + "hejhejhej" = get_erl_call_result(["-address", + erlang:integer_to_list(Port), + "-a", + erlang_eval_call_string(MultiPrintCodeStr), + "-fetch_stdout", + "-no_result_term"]), + %% Test that we can print from a sub-process + SubProcPrintStr = + "begin\n" + " P = self(),\n" + " Printer = fun() ->\n" + " io:format(\"subhej\"),\n" + " receive\n" + " hej -> P ! hej\n" + " end\n" + " end,\n" + " PrinterPid = spawn(Printer),\n" + " PrinterPid ! hej,\n" + " receive\n" + " hej -> ok\n" + " end\n" + "end.\n", + "subhej" = get_erl_call_result(["-address", + erlang:integer_to_list(Port), + "-a", + erlang_eval_call_string(SubProcPrintStr), + "-fetch_stdout", + "-no_result_term"]), + %% Test that the remote group leader supports the multi-requests + %% request + TriggerMultiRequestsRequestStr = + "begin\n" + " %% Create multi request\n" + " MultiReqCollectGL =\n" + " spawn(\n" + " fun() ->\n" + " (fun GL(ReqList) ->\n" + " receive\n" + " {io_request, From, ReplyAs, Req} ->\n" + " From ! {io_reply, ReplyAs, ok},\n" + " GL([Req | ReqList]);\n" + " {get_reqs, Pid} ->\n" + " Pid ! {multi_req,\n" + " {requests, lists:reverse(ReqList)}}\n" + " end\n" + " end)([])\n" + " end),\n" + " OldGL = erlang:group_leader(),\n" + " erlang:group_leader(MultiReqCollectGL, self()),\n" + " io:format(\"test1\"),\n" + " io:format(\"test2\"),\n" + " io:format(\"test3\"),\n" + " MultiReqCollectGL ! {get_reqs, self()},\n" + " MultiReqsRequest =\n" + " receive\n" + " {multi_req, R} -> R\n" + " end,\n" + " erlang:group_leader(OldGL, self()),\n" + " %% Send multi request\n" + " erlang:group_leader() ! {io_request,self(), self(), MultiReqsRequest},\n" + " Me = self(),\n" + " receive {io_reply, Me, ok} -> ok end,\n" + " %% Send normal request\n" + " io:format(\"test4\")\n" + "end.\n", + "test1test2test3test4" = + get_erl_call_result(["-address", + erlang:integer_to_list(Port), + "-a", + erlang_eval_call_string(TriggerMultiRequestsRequestStr), + "-fetch_stdout", + "-no_result_term"]), + ok. + %% OTP-16604: Tests that erl_call works even when the local hostname cannot be %% resolved. unresolvable_hostname(_Config) -> @@ -261,4 +370,6 @@ get_port_res(Port, Acc) when is_port(Port) -> {Port, eof} -> lists:flatten(Acc) end. - + +erlang_eval_call_string(CodeStr) -> + lists:flatten(io_lib:format("erl_eval eval_str [~w]",[CodeStr])). diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index 002a069a92..71f3765f21 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -95,6 +95,7 @@ start(Options) -> init(Options) -> XMLDir = proplists:get_value(dir, Options, ?XMLDIR), + ensure_xmldir(XMLDir), St = #state{verbose = proplists:get_bool(verbose, Options), xmldir = XMLDir, testsuites = []}, @@ -255,6 +256,19 @@ add_testcase_to_testsuite({error, Exception}, TestCaseTmp, TestSuite) -> testcases = [TestCase|TestSuite#testsuite.testcases] } end. +ensure_xmldir(XMLDir) -> + Steps = [ + fun filelib:ensure_dir/1, + fun file:make_dir/1], + lists:foldl(fun ensure_xmldir/2, XMLDir, Steps). + +ensure_xmldir(Fun, XMLDir) -> + case Fun(XMLDir) of + ok -> XMLDir; + {error, eexist} -> XMLDir; + {error, _Reason} = Error -> throw(Error) + end. + %% ---------------------------------------------------------------------------- %% Write a report to the XML directory. %% This function opens the report file, calls write_report_to/2 and closes the file. diff --git a/lib/eunit/test/eunit_SUITE.erl b/lib/eunit/test/eunit_SUITE.erl index e55091e8c1..df754e46c5 100644 --- a/lib/eunit/test/eunit_SUITE.erl +++ b/lib/eunit/test/eunit_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, app_test/1,appup_test/1,eunit_test/1,surefire_utf8_test/1,surefire_latin_test/1, - surefire_c0_test/1]). + surefire_c0_test/1, surefire_ensure_dir_test/1]). -include_lib("common_test/include/ct.hrl"). @@ -30,7 +30,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [app_test, appup_test, eunit_test, surefire_utf8_test, surefire_latin_test, - surefire_c0_test]. + surefire_c0_test, surefire_ensure_dir_test]. groups() -> []. @@ -76,6 +76,11 @@ surefire_c0_test(Config) when is_list(Config) -> true = lists:member($\t, Chars), ok. +surefire_ensure_dir_test(Config) when is_list(Config) -> + XMLDir = filename:join(proplists:get_value(priv_dir, Config), "c1"), + ok = eunit:test(tc0, [{report,{eunit_surefire,[{dir,XMLDir}]}}]), + ok = file:del_dir_r(XMLDir). + check_surefire(Module) -> File = "TEST-"++atom_to_list(Module)++".xml", file:delete(File), diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 562384f7be..479c9699b5 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -670,6 +670,22 @@ </section> +<section><title>Kernel 6.5.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>When running Xref in the <c>modules</c> mode, the + Debugger application would show up as a depency for the + Kernel applications.</p> + <p> + Own Id: OTP-17223 Aux Id: GH-4546, PR-4554 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.5.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/gen_tcp_socket.erl b/lib/kernel/src/gen_tcp_socket.erl index c90b68ecad..3b6315e3a3 100644 --- a/lib/kernel/src/gen_tcp_socket.erl +++ b/lib/kernel/src/gen_tcp_socket.erl @@ -848,9 +848,9 @@ server_read_opts() -> header => 0, deliver => term, start_opts => [], % Just to make it settable + line_delimiter => $\n, %% XXX not implemented yet - exit_on_close => true, - line_delimiter => $\n}, + exit_on_close => true}, server_read_write_opts()). -compile({inline, [server_write_opts/0]}). server_write_opts() -> @@ -1426,6 +1426,7 @@ handle_event( #recv{info = ?select_info(SelectRef)}, {#params{socket = Socket} = P, D}) -> %% + %% ?DBG([info, {socket, Socket}, {ref, SelectRef}]), handle_recv(P, D, []); %% handle_event( @@ -1520,6 +1521,7 @@ handle_closed(Type, Content, State, {P, _D}) -> handle_connect( #params{socket = Socket} = P, D, From, Addr, Timeout) -> %% + %% ?DBG([{d, D}, {addr, Addr}]), case socket:connect(Socket, Addr, nowait) of ok -> handle_connected( @@ -1571,6 +1573,7 @@ handle_connected(P, {D, ActionsR}) -> handle_connected(P, D, ActionsR). %% handle_connected(P, D, ActionsR) -> + %% ?DBG([{p, P}, {d, D}, {actions_r, ActionsR}]), case D of #{active := false} -> {next_state, 'connected', @@ -1585,6 +1588,7 @@ handle_recv_start( when Packet =:= raw, 0 < Length; Packet =:= 0, 0 < Length -> Size = iolist_size(Buffer), + %% ?DBG([{packet, Packet}, {length, Length}, {buf_sz, Size}]), if Length =< Size -> {Data, NewBuffer} = @@ -1602,12 +1606,13 @@ handle_recv_start( [{{timeout, recv}, Timeout, recv}]) end; handle_recv_start(P, D, From, _Length, Timeout) -> + %% ?DBG([{p, P}, {d, D}]), handle_recv( P, D#{recv_length => 0, recv_from => From}, [{{timeout, recv}, Timeout, recv}]). handle_recv(P, #{packet := Packet, recv_length := Length} = D, ActionsR) -> - %% ?DBG({recv_len, Length}), + %% ?DBG([{packet, Packet}, {recv_length, Length}]), if 0 < Length -> handle_recv_length(P, D, ActionsR, Length); @@ -1755,13 +1760,23 @@ handle_recv_length(P, D, ActionsR, _0, Buffer) -> handle_recv_deliver(P, D#{buffer := <<>>}, ActionsR, Data) end. -handle_recv_decode(P, #{packet_size := PacketSize} = D, ActionsR, Data) -> - case - erlang:decode_packet( - decode_packet(D), Data, - [{packet_size, PacketSize}, - {line_length, PacketSize}]) - of +handle_recv_decode(P, + #{packet := line, + line_delimiter := LineDelimiter, + packet_size := PacketSize} = D, + ActionsR, Data) -> + DecodeOpts = [{line_delimiter, LineDelimiter}, + {line_length, PacketSize}], + handle_recv_decode(P, D, + ActionsR, Data, DecodeOpts); +handle_recv_decode(P, D, ActionsR, Data) -> + handle_recv_decode(P, D, ActionsR, Data, []). + +handle_recv_decode(P, #{packet_size := PacketSize} = D, + ActionsR, Data, DecocdeOpts0) -> + %% ?DBG([{packet_sz, PacketSize}, {decode_opts0, DecocdeOpts0}]), + DecodeOpts = [{packet_size, PacketSize}|DecocdeOpts0], + case erlang:decode_packet(decode_packet(D), Data, DecodeOpts) of {ok, Decoded, Rest} -> %% is_list(Buffer) -> try to decode first %% is_binary(Buffer) -> get more data first @@ -1818,9 +1833,11 @@ handle_recv_error_decode( handle_recv_more(P, D, ActionsR, BufferedData) -> case socket_recv(P#params.socket, 0) of {ok, <<MoreData/binary>>} -> - Data = catbin(BufferedData, MoreData), + %% ?DBG([{more_data_sz, byte_size(MoreData)}]), + Data = catbin(BufferedData, MoreData), handle_recv_decode(P, D, ActionsR, Data); {select, SelectInfo} -> + %% ?DBG([{select_info, SelectInfo}]), {next_state, #recv{info = SelectInfo}, {P, D#{buffer := BufferedData}}, @@ -1969,6 +1986,9 @@ recv_data_deliver( packet := Packet} = D, ActionsR, Data) -> %% + %% ?DBG([{owner, Owner}, + %% {mode, Mode}, + %% {header, Header}, {deliver, Deliver}, {packet, Packet}]), DeliverData = deliver_data(Data, Mode, Header, Packet), case D of #{recv_from := From} -> @@ -2170,6 +2190,7 @@ state_setopts_server(P, D, State, Opts, {Tag, Value}) -> {{error, einval}, D} end; _ -> + %% ?DBG([{tag, Tag}, {value, Value}]), state_setopts(P, D#{Tag => Value}, State, Opts) end. diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index 74fa6744ee..07b7b0ef6e 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1805,7 +1805,7 @@ start_protos_listen(Name, Host, Node, [Proto | Ps], Ls, CleanHalt) -> catch error:undef -> proto_error(CleanHalt, Proto, "not supported"), start_protos_listen(Name, Host, Node, Ps, Ls, CleanHalt); - error:Reason -> + _:Reason -> register_error(CleanHalt, Proto, Reason), start_protos_listen(Name, Host, Node, Ps, Ls, CleanHalt) end; diff --git a/lib/kernel/src/rpc.erl b/lib/kernel/src/rpc.erl index 29b6d40592..6613028a0b 100644 --- a/lib/kernel/src/rpc.erl +++ b/lib/kernel/src/rpc.erl @@ -132,8 +132,22 @@ handle_call({call, Mod, Fun, Args, Gleader}, To, S) -> %% Spawn not to block the rex server. ExecCall = fun () -> set_group_leader(Gleader), + GleaderBeforeCall = group_leader(), Reply = execute_call(Mod, Fun, Args), - gen_server:reply(To, Reply) + case Gleader of + {send_stdout_to_caller, _} -> + %% The group leader sends the response + %% to make sure that the client gets + %% all stdout that it should get before + %% the response + Ref = erlang:make_ref(), + GleaderBeforeCall ! {stop, self(), Ref, To, Reply}, + receive + Ref -> ok + end; + _ -> + gen_server:reply(To, Reply) + end end, try {_,Mon} = spawn_monitor(ExecCall), @@ -199,9 +213,17 @@ handle_info({From, {send, Name, Msg}}, S) -> ok %% It's up to Name to respond !!!!! end, {noreply, S}; -handle_info({From, {call, _Mod, _Fun, _Args, _Gleader} = Request}, S) -> +handle_info({From, {call, Mod, Fun, Args, Gleader}}, S) -> %% Special for hidden C node's, uugh ... To = {From, ?NAME}, + NewGleader = + case Gleader of + send_stdout_to_caller -> + {send_stdout_to_caller, From}; + _ -> + Gleader + end, + Request = {call, Mod, Fun, Args, NewGleader}, case handle_call(Request, To, S) of {noreply, _NewS} = Return -> Return; @@ -249,6 +271,8 @@ execute_call(Mod, Fun, Args) -> set_group_leader(Gleader) when is_pid(Gleader) -> group_leader(Gleader, self()); +set_group_leader({send_stdout_to_caller, CallerPid}) -> + group_leader(cnode_call_group_leader_start(CallerPid), self()); set_group_leader(user) -> %% For example, hidden C nodes doesn't want any I/O. Gleader = case whereis(user) of @@ -1265,3 +1289,114 @@ pinfo(Pid, Item) when node(Pid) =:= node() -> process_info(Pid, Item); pinfo(Pid, Item) -> block_call(node(Pid), erlang, process_info, [Pid, Item]). + +%% The following functions with the cnode_call_group_leader_ prefix +%% are used for RPC requests with the group leader field set to +%% send_stdout_to_caller. The group leader that these functions +%% implement sends back data that are written to stdout during the +%% call. The group leader implementation is heavily inspired by the +%% example from the documentation of "The Erlang I/O Protocol". + + +%% A record is used for the state even though it consists of only one +%% pid to make future extension easier +-record(cnode_call_group_leader_state, + { + caller_pid :: pid() + }). + +-spec cnode_call_group_leader_loop(State :: #cnode_call_group_leader_state{}) -> ok | no_return(). + +cnode_call_group_leader_loop(State) -> + receive + {io_request, From, ReplyAs, Request} -> + {_, Reply, NewState} + = cnode_call_group_leader_request(Request, State), + From ! {io_reply, ReplyAs, Reply}, + cnode_call_group_leader_loop(NewState); + {stop, StopRequesterPid, Ref, To, Reply} -> + gen_server:reply(To, Reply), + StopRequesterPid ! Ref, + ok; + _Unknown -> + cnode_call_group_leader_loop(State) + end. + +-spec cnode_call_group_leader_request(Request, State) -> Result when + Request :: any(), + State :: #cnode_call_group_leader_state{}, + Result :: {ok | error, Reply, NewState}, + Reply :: term(), + NewState :: #cnode_call_group_leader_state{}. + +cnode_call_group_leader_request({put_chars, Encoding, Chars}, + State) -> + cnode_call_group_leader_put_chars(Chars, Encoding, State); +cnode_call_group_leader_request({put_chars, Encoding, Module, Function, Args}, + State) -> + try + cnode_call_group_leader_request({put_chars, + Encoding, + apply(Module, Function, Args)}, + State) + catch + _:_ -> + {error, {error, Function}, State} + end; +cnode_call_group_leader_request({requests, Reqs}, State) -> + cnode_call_group_leader_multi_request(Reqs, {ok, ok, State}); +cnode_call_group_leader_request({get_until, _, _, _, _, _}, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request({get_chars, _, _, _}, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request({get_line, _, _}, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request({get_geometry,_}, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request({setopts, _Opts}, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request(getopts, State) -> + {error, {error,enotsup}, State}; +cnode_call_group_leader_request(_Other, State) -> + {error, {error,request}, State}. + +-spec cnode_call_group_leader_multi_request(Requests, PrevResponse) -> Result when + Requests :: list(), + PrevResponse :: {ok | error, Reply, State :: #cnode_call_group_leader_state{}}, + Result :: {ok | error, Reply, NewState :: #cnode_call_group_leader_state{}}, + Reply :: term(). + +cnode_call_group_leader_multi_request([R|Rs], {ok, _Res, State}) -> + cnode_call_group_leader_multi_request(Rs, cnode_call_group_leader_request(R, State)); +cnode_call_group_leader_multi_request([_|_], Error) -> + Error; +cnode_call_group_leader_multi_request([], Result) -> + Result. + +-spec cnode_call_group_leader_put_chars(Chars, Encoding, State) -> Result when + Chars :: unicode:latin1_chardata() | unicode:chardata() | unicode:external_chardata(), + Encoding :: unicode:encoding(), + State :: #cnode_call_group_leader_state{}, + Result :: {ok | error, term(), NewState}, + NewState :: #cnode_call_group_leader_state{}. + +cnode_call_group_leader_put_chars(Chars, Encoding, State) -> + CNodePid = State#cnode_call_group_leader_state.caller_pid, + case unicode:characters_to_binary(Chars,Encoding,utf8) of + Data when is_binary(Data) -> + CNodePid ! {rex_stdout, Data}, + {ok, ok, State}; + Error -> + {error, {error, Error}, state} + end. + +-spec cnode_call_group_leader_init(CallerPid :: pid()) -> ok | no_return(). + +cnode_call_group_leader_init(CallerPid) -> + State = #cnode_call_group_leader_state{caller_pid = CallerPid}, + cnode_call_group_leader_loop(State). + +-spec cnode_call_group_leader_start(CallerPid :: pid()) -> pid(). + +cnode_call_group_leader_start(CallerPid) -> + spawn_link(fun() -> cnode_call_group_leader_init(CallerPid) end). diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index c216886546..5d305868a1 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -186,6 +186,9 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " @@ -354,15 +357,29 @@ do_recv_delim(Config) -> [{active,false}, {packet,line}, {line_delimiter,$X}], {ok, Client} = gen_tcp:connect(localhost, Port, Opts), {ok, A} = gen_tcp:accept(L), + ?P("send the data"), ok = gen_tcp:send(A, "abcXefgX"), + %% Why do we need a timeout? %% Sure, normally there would be no delay, %% but this testcase has nothing to do with timeouts? ?P("read the first chunk"), - {ok, "abcX"} = gen_tcp:recv(Client, 0), %, 200), + {ok, "abcX"} = gen_tcp:recv(Client, 0), % 200), ?P("read the second chunk"), - {ok, "efgX"} = gen_tcp:recv(Client, 0), %, 200), + {ok, "efgX"} = gen_tcp:recv(Client, 0), % 200), + + ?P("set active = 2"), + ok = inet:setopts(Client, [{active,2}]), + + ?P("send the data again"), + ok = gen_tcp:send(A, "abcXefgX"), + + ?P("await the first chunk"), + receive {tcp, Client, "abcX"} -> ?P("received first chunck") end, + ?P("await the second chunk"), + receive {tcp, Client, "efgX"} -> ?P("received second chunck") end, + ?P("cleanup"), ok = gen_tcp:close(Client), ok = gen_tcp:close(A), diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index cb4a7aa995..83c96e6c46 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -206,6 +206,9 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " @@ -4397,7 +4400,8 @@ several_accepts_in_one_go(Config) when is_list(Config) -> do_several_accepts_in_one_go(Config) -> ?P("create listen socket"), - LS = case ?LISTEN(Config, 0,[]) of + NumActors = 8, + LS = case ?LISTEN(Config, 0, [{backlog, NumActors}]) of {ok, LSock} -> LSock; {error, eaddrnotavail = Reason} -> @@ -4405,13 +4409,15 @@ do_several_accepts_in_one_go(Config) -> end, Parent = self(), {ok, PortNo} = inet:port(LS), - F1 = fun() -> ?P("acceptor starting"), - Parent ! {accepted,self(),gen_tcp:accept(LS)} + F1 = fun() -> + ?P("acceptor starting"), + Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - F2 = fun() -> ?P("connector starting"), - Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} + F2 = fun() -> + ?P("connector starting"), + Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} end, - Ns = lists:seq(1,8), + Ns = lists:seq(1, NumActors), ?P("start acceptors"), _ = [spawn(F1) || _ <- Ns], ?P("await accept timeouts"), @@ -4419,7 +4425,7 @@ do_several_accepts_in_one_go(Config) -> ?P("start connectors"), _ = [spawn(F2) || _ <- Ns], ?P("await accepts"), - ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],8,15000), + ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],NumActors,15000), ?P("await connects"), ok = ?EXPECT_CONNECTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],1000), ?P("done"), diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index a52f70933e..5e78ab6f58 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2020. All Rights Reserved. +%% Copyright Ericsson AB 1998-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -100,6 +100,9 @@ init_per_suite(Config0) -> ?P("init_per_suite -> end when " "~n Config: ~p", [Config1]), + %% We need a monitor on this node also + kernel_test_sys_monitor:start(), + Config1 end. @@ -109,10 +112,13 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " - "~n Nodes: ~p", [erlang:nodes()]), + "~n Nodes: ~p", [erlang:nodes()]), Config1. @@ -133,15 +139,49 @@ end_per_group(_GroupName, Config) -> Config. -init_per_testcase(read_packets, Config) -> +init_per_testcase(Case, Config0) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config0, erlang:nodes(), pi(links), pi(monitors)]), + + kernel_test_global_sys_monitor:reset_events(), + + Config1 = init_per_testcase2(Case, Config0), + + ?P("init_per_testcase -> done when" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), + Config1. + +init_per_testcase2(read_packets, Config) -> ct:timetrap({minutes, 2}), Config; -init_per_testcase(_Case, Config) -> +init_per_testcase2(_Case, Config) -> Config. -end_per_testcase(_Case, _Config) -> + +end_per_testcase(_Case, Config) -> + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + ?P("system events during test: " + "~n ~p", [kernel_test_global_sys_monitor:events()]), + + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), ok. + %%------------------------------------------------------------- %% Send two packets to a closed port (on some systems this causes the socket %% to be closed). @@ -793,7 +833,7 @@ sendtos_ok({unix,_}, _) -> true; sendtos_ok(_, _) -> false. %% Using the option returns einval, so it is not implemented. -sendttl_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); +sendttl_ok({unix,darwin}, _OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); sendttl_ok({unix,linux}, OSVer) -> not semver_lt(OSVer, {4,0,0}); %% Using the option returns enoprotoopt, so it is not implemented. sendttl_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); @@ -1279,6 +1319,13 @@ get_localaddr([Localhost|Ls]) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +pi(Item) -> + {Item, Val} = process_info(self(), Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% %% Utils %% diff --git a/lib/kernel/test/global_SUITE.erl b/lib/kernel/test/global_SUITE.erl index 22db756d97..689ea09792 100644 --- a/lib/kernel/test/global_SUITE.erl +++ b/lib/kernel/test/global_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2019. All Rights Reserved. +%% Copyright Ericsson AB 1997-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,8 +19,11 @@ %% -module(global_SUITE). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, +-export([all/0, suite/0, groups/0, init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, + names/1, names_hidden/1, locks/1, locks_hidden/1, bad_input/1, names_and_locks/1, lock_die/1, name_die/1, basic_partition/1, basic_name_partition/1, @@ -38,18 +41,77 @@ both_known_1/1, lost_unregister/1, mass_death/1, - garbage_messages/1]). - --export([global_load/3, lock_global/2, lock_global2/2]). - --export([]). --export([init_mass_spawn/1]). + garbage_messages/1, + + ring_line/1 + ]). + +%% Not used +-export([simple_dis/4, + config_dc/4, + w/2, + check_same/2, + check_same/1, + stop/0, + start_nodes_serially/3]). + +-export([lock_global/2, lock_global2/2]). + +-export([ + %% Called via ?RES + resolve_none/3, + resolve_first/3, + resolve_second/3, + bad_resolver/3, + badrpc_resolver/3, + lock_resolver/3, + exit_resolver/3, + disconnect_first/3, + halt_second/3, + init_mass_spawn/1, + + %% Called via rpc_cast + part_2_2/4, + part1/4, + part1_5/4, + part2/10, + part3/10, + crash/1, + single_node/3, + part_ext/4, + isolated_node/4, + mk_part_node/3, + alone/2, + global_load/3, + + %% Called via rpc:call + halt_node/1, + start_proc/0, start_proc/1, + start_proc2/1, + start_proc3/1, + start_proc4/1, + start_proc_basic/1, + start_resolver/2, + + %% Called as a fun (fun ?MODULE:function/x) + fix_basic_name/3, + + %% Called via spawn + init_proc_basic/2, + init_2/0, + p_init/1, p_init/2, + p_init2/2 + + ]). -export([start_tracer/0, stop_tracer/0, get_trace/0]). --compile(export_all). +%% Exports for error_logger handler +-export([init/1, handle_event/2, handle_info/2, handle_call/2, terminate/2]). + -include_lib("common_test/include/ct.hrl"). +-include("kernel_test_lib.hrl"). -define(NODES, [node()|nodes()]). @@ -66,7 +128,8 @@ all() -> case init:get_argument(ring_line) of {ok, _} -> [ring_line]; _ -> - [names, names_hidden, locks, locks_hidden, bad_input, + [ + names, names_hidden, locks, locks_hidden, bad_input, names_and_locks, lock_die, name_die, basic_partition, advanced_partition, basic_name_partition, stress_partition, simple_ring, simple_line, ring, line, @@ -75,7 +138,8 @@ all() -> simple_resolve2, simple_resolve3, leftover_name, re_register_name, name_exit, external_nodes, many_nodes, sync_0, global_groups_change, register_1, both_known_1, - lost_unregister, mass_death, garbage_messages] + lost_unregister, mass_death, garbage_messages + ] end. groups() -> @@ -102,29 +166,59 @@ end_per_suite(_Config) -> -define(nodes_tag, '$global_nodes'). -define(registered, proplists:get_value(registered, Config)). -init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - ok = gen_server:call(global_name_server, high_level_trace_start,infinity), +init_per_testcase(Case, Config0) when is_atom(Case) andalso is_list(Config0) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config0, erlang:nodes(), pi(links), pi(monitors)]), + + ok = gen_server:call(global_name_server, + high_level_trace_start, + infinity), %% Make sure that everything is dead and done. Otherwise there are problems %% on platforms on which it takes a long time to shut down a node. stop_nodes(nodes()), timer:sleep(1000), - [{?TESTCASE, Case}, {registered, registered()} | Config]. + Config1 = [{?TESTCASE, Case}, {registered, registered()} | Config0], + + ?P("init_per_testcase -> done when" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [Config1, erlang:nodes(), pi(links), pi(monitors)]), + + Config1. end_per_testcase(_Case, Config) -> - ct:log("Calling end_per_testcase!",[]), + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + write_high_level_trace(Config), - _ = - gen_server:call(global_name_server, high_level_trace_stop, infinity), + _ = gen_server:call(global_name_server, high_level_trace_stop, infinity), [global:unregister_name(N) || N <- global:registered_names()], InitRegistered = ?registered, Registered = registered(), - [io:format("~s local names: ~p~n", [What, N]) || + + [?P("end_per_testcase -> " + "~n ~s local names: ~p", [What, N]) || {What, N} <- [{"Added", Registered -- InitRegistered}, {"Removed", InitRegistered -- Registered}], N =/= []], + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [erlang:nodes(), pi(links), pi(monitors)]), + ok. %%% General comments: @@ -385,7 +479,7 @@ write_high_level_trace(Config) -> end. write_high_level_trace(Nodes, Config) -> - When = now(), + When = erlang:timestamp(), %% 'info' returns more than the trace, which is nice. Data = [{Node, {info, rpc:call(Node, global, info, [])}} || Node <- Nodes], @@ -419,22 +513,44 @@ lock_global2(Id, Parent) -> %% Kill Pid2 and check that 'test' isn't registered. names(Config) when is_list(Config) -> + ?TC_TRY(names, fun() -> do_names(Config) end). + +do_names(Config) -> + ?P("names -> begin when" + "~n Nodes: ~p" + "~n Names: ~p", + [nodes(), + case net_adm:names() of + {ok, N} -> + N; + _ -> + "-" + end]), Timeout = 30, ct:timetrap({seconds,Timeout}), + ?P("names -> init high level trace"), init_high_level_trace(Timeout), + ?P("names -> init condition"), init_condition(Config), + ?P("names -> get registered names"), OrigNames = global:registered_names(), + ?P("names -> start node cp1"), {ok, Cp1} = start_node(cp1, Config), + ?P("names -> start node cp2"), {ok, Cp2} = start_node(cp2, Config), + ?P("names -> start node cp3"), {ok, Cp3} = start_node(cp3, Config), + ?P("names -> wait for ready net"), wait_for_ready_net(Config), %% start a proc and register it + ?P("names -> start and register process 'test'"), {Pid, yes} = start_proc(test), %% test that it is registered at all nodes + ?P("names -> verify process has been registered on all nodes"), ?UNTIL(begin (Pid =:= global:whereis_name(test)) and (Pid =:= rpc:call(Cp1, global, whereis_name, [test])) and @@ -444,25 +560,32 @@ names(Config) when is_list(Config) -> end), %% try to register the same name + ?P("names -> try register (locally) another 'test' (and expect rejection)"), no = global:register_name(test, self()), + ?P("names -> try register (on cp1) another 'test' (and expect rejection)"), no = rpc:call(Cp1, global, register_name, [test, self()]), %% let process exit, check that it is unregistered automatically + ?P("names -> terminate the 'test' process"), exit_p(Pid), + ?P("names -> verify 'test' process has been automatically unregistered"), ?UNTIL((undefined =:= global:whereis_name(test)) and (undefined =:= rpc:call(Cp1, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp2, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp3, global, whereis_name, [test]))), %% test re_register + ?P("names -> start and register another process 'test'"), {Pid2, yes} = start_proc(test), + ?P("names -> verify process 'test' has been registered"), ?UNTIL(Pid2 =:= rpc:call(Cp3, global, whereis_name, [test])), Pid3 = rpc:call(Cp3, ?MODULE, start_proc2, [test]), ?UNTIL(Pid3 =:= rpc:call(Cp3, global, whereis_name, [test])), Pid3 = global:whereis_name(test), %% test sending + ?P("names -> test sending (from local)"), global:send(test, {ping, self()}), receive {pong, Cp3} -> ok @@ -470,6 +593,7 @@ names(Config) when is_list(Config) -> 2000 -> ct:fail(timeout1) end, + ?P("names -> test sending (from cp1)"), rpc:call(Cp1, global, send, [test, {ping, self()}]), receive {pong, Cp3} -> ok @@ -477,31 +601,43 @@ names(Config) when is_list(Config) -> 2000 -> ct:fail(timeout2) end, + ?P("names -> unregister 'test' process"), _ = global:unregister_name(test), ?UNTIL((undefined =:= global:whereis_name(test)) and (undefined =:= rpc:call(Cp1, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp2, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp3, global, whereis_name, [test]))), + ?P("names -> terminate process 'test'"), exit_p(Pid3), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), %% register a proc + ?P("names -> register a process"), {_Pid6, yes} = rpc:call(Cp3, ?MODULE, start_proc, [test]), + ?P("names -> write high level trace"), write_high_level_trace(Config), %% stop the nodes, and make sure names are released. + ?P("names -> stop node cp1"), stop_node(Cp1), + ?P("names -> stop node cp2"), stop_node(Cp2), + ?P("names -> stop node cp3"), stop_node(Cp3), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), exit_p(Pid2), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), init_condition(Config), + + ?P("names -> done"), ok. %% Tests that names on a hidden node doesn't interfere with names on @@ -1164,7 +1300,7 @@ advanced_partition(Config) when is_list(Config) -> %% make cp3-cp6 connected, partitioned from us and cp0-cp2 rpc_cast(Cp3, ?MODULE, part2, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5,Cp6]), + [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), ?UNTIL(is_ready_partition(Config)), %% start different processes in this partition @@ -1266,7 +1402,7 @@ stress_partition(Config) when is_list(Config) -> %% make cp3-cp5 connected, partitioned from us and cp0-cp2 %% cp6 is alone (single node). cp6 pings cp0 and cp3 in 12 secs... rpc_cast(Cp3, ?MODULE, part3, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5,Cp6]), + [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), ?UNTIL(is_ready_partition(Config)), %% start different processes in this partition @@ -1903,9 +2039,9 @@ otp_5737(Config) when is_list(Config) -> {'EXIT', _} = (catch global:set_lock(LockId, Nodes, -1)), {'EXIT', _} = (catch global:set_lock(LockId, Nodes, a)), true = global:set_lock(LockId, Nodes, 0), - Time1 = now(), + Time1 = erlang:timestamp(), false = global:set_lock({?MODULE,not_me}, Nodes, 0), - true = timer:now_diff(now(), Time1) < 5000, + true = timer:now_diff(erlang:timestamp(), Time1) < 5000, _ = global:del_lock(LockId, Nodes), Fun = fun() -> ok end, @@ -2880,7 +3016,7 @@ sync_until() -> sync_until(no_log_file). sync_until(LogFile) -> - Time = ?UNTIL_LOOP - (msec(now()) rem ?UNTIL_LOOP), + Time = ?UNTIL_LOOP - (msec(erlang:timestamp()) rem ?UNTIL_LOOP), catch append_to_file(LogFile, {sync_until, Time}), timer:sleep(Time). @@ -3437,11 +3573,11 @@ is_node_in_part(File, MyPart) -> Rs -> erlang:display({is_node_in_part, resolvers, Rs}), trace_message({node(), is_node_in_part, Rs}), - append_to_file(File, {now(), Known, Nodes, Rs}), + append_to_file(File, {erlang:timestamp(), Known, Nodes, Rs}), false end; _ -> - append_to_file(File, {now(), Known, Nodes}), + append_to_file(File, {erlang:timestamp(), Known, Nodes}), false end. @@ -3497,7 +3633,7 @@ collect_resolves() -> cr(0). cr(Res) -> receive {resolve_called, Name, Node} -> - io:format("resolve called: ~w ~w~n", [Name, Node]), + ?P("resolve called: ~w ~w", [Name, Node]), cr(Res+1) after 0 -> Res @@ -3580,7 +3716,7 @@ loop_2() -> end. msec() -> - msec(now()). + msec(erlang:timestamp()). msec(T) -> element(1,T)*1000000000 + element(2,T)*1000 + element(3,T) div 1000. @@ -3792,7 +3928,7 @@ node_names(Names, Config) -> %% simple_resolve assumes that the node name comes first. node_name(Name, Config) -> U = "_", - {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(now()), + {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(erlang:timestamp()), Date = io_lib:format("~4w_~2..0w_~2..0w__~2..0w_~2..0w_~2..0w", [Y,M,D, H,Min,S]), L = lists:flatten(Date), @@ -3913,13 +4049,13 @@ mass_death(Config) when is_list(Config) -> {H,M,S} = time(), io:format("Started probing: ~.4.0w-~.2.0w-~.2.0w ~.2.0w:~.2.0w:~.2.0w~n", [YYYY,MM,DD,H,M,S]), - wait_mass_death(Nodes, OrigNames, erlang:now(), Config). + wait_mass_death(Nodes, OrigNames, erlang:timestamp(), Config). wait_mass_death(Nodes, OrigNames, Then, Config) -> Names = global:registered_names(), case Names--OrigNames of [] -> - T = now_diff(erlang:now(), Then) div 1000, + T = now_diff(erlang:timestamp(), Then) div 1000, lists:foreach( fun (Node) -> stop_node(Node) @@ -4128,17 +4264,68 @@ garbage_messages(Config) when is_list(Config) -> ok. wait_for_ready_net(Config) -> - wait_for_ready_net(?NODES, Config). + {Pid, MRef} = spawn_monitor(fun() -> + wait_for_ready_net(?NODES, Config) + end), + wait_for_ready_net_loop(Pid, MRef). + +wait_for_ready_net_loop(Pid, MRef) -> + receive + {'DOWN', MRef, process, Pid, Info} -> + ?P("wait-for-ready-net process terminated: " + "~n ~p", [Info]), + ok; + + {'EXIT', ParentPid, {timetrap_timeout, _Timeout, _Stack}} -> + ?P("wait-for-ready-net -> received timetrap timeout:" + "~n Regarding: ~p" + "~n Waiter: ~p" + "~n Current Location: ~p" + "~n Mesages: ~p", + [ParentPid, Pid, pi(Pid, current_location), pi(Pid, messages)]), + ct:fail("Timeout waiting for ready network") + + end. wait_for_ready_net(Nodes0, Config) -> Nodes = lists:sort(Nodes0), - io:format("wait_for_ready_net ~p~n", [Nodes]), + ?P("wait_for_ready_net ->" + "~n Nodes: ~p", [Nodes]), ?UNTIL(begin - lists:all(fun(N) -> Nodes =:= get_known(N) end, Nodes) and + lists:all(fun(N) -> + ?P("wait_for_ready_net -> " + "get known (by global) for ~p", [N]), + GNs = get_known(N), + ?P("wait_for_ready_net -> verify same for ~p:" + "~n Global Known: ~p" + "~n Nodes: ~p", [N, GNs, Nodes]), + GRes = Nodes =:= GNs, + ?P("wait_for_ready_net => ~p", [GRes]), + GRes + end, + Nodes) and lists:all(fun(N) -> - LNs = rpc:call(N, erlang, nodes, []), - Nodes =:= lists:sort([N | LNs]) - end, Nodes) + ?P("wait_for_ready_net -> " + "get erlang nodes for ~p", [N]), + case rpc:call(N, erlang, nodes, []) of + RNs0 when is_list(RNs0) -> + RNs = lists:sort([N | RNs0]), + ?P("wait_for_ready_net -> " + "verify same for ~p: " + "~n Remote nodes: ~p" + "~n (Local) Nodes: ~p", + [N, RNs, Nodes]), + ERes = Nodes =:= RNs, + ?P("wait_for_ready_net => ~p", + [ERes]), + ERes; + BadRes -> + ?P("failed get remote nodes: " + "~n ~p", [BadRes]), + false + end + end, + Nodes) end). get_known(Node) -> @@ -4180,7 +4367,7 @@ rpc_cast(Node, Module, Function, Args, File) -> pong -> rpc:cast(Node, Module, Function, Args); Else -> - append_to_file(File, {now(), {rpc_cast, Node, Module, Function, + append_to_file(File, {erlang:timestamp(), {rpc_cast, Node, Module, Function, Args, Else}}) %% Maybe we should crash, but it probably doesn't matter. end. @@ -4218,14 +4405,14 @@ start_tracer() -> tracer(L) -> receive %% {save, Term} -> - %% tracer([{now(),Term} | L]); + %% tracer([{erlang:timestamp(),Term} | L]); {get, From} -> From ! {trace, lists:reverse(L)}, tracer([]); stop -> exit(normal); Term -> - tracer([{now(),Term} | L]) + tracer([{erlang:timestamp(),Term} | L]) end. stop_tracer() -> @@ -4256,6 +4443,17 @@ trace_message(M) -> ok end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pi(Item) -> + pi(self(), Item). +pi(Pid, Item) -> + {Item, Val} = process_info(Pid, Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%----------------------------------------------------------------- %% The error_logger handler used for OTP-6931. %%----------------------------------------------------------------- diff --git a/lib/kernel/test/socket_SUITE.erl b/lib/kernel/test/socket_SUITE.erl index 139a886aa5..2ec9227a84 100644 --- a/lib/kernel/test/socket_SUITE.erl +++ b/lib/kernel/test/socket_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2018-2020. All Rights Reserved. +%% Copyright Ericsson AB 2018-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19696,7 +19696,10 @@ api_opt_ip_recvtos_udp4(doc) -> api_opt_ip_recvtos_udp4(_Config) when is_list(_Config) -> ?TT(?SECS(5)), tc_try(api_opt_ip_recvtos_udp4, - fun() -> has_support_ip_recvtos() end, + fun() -> + has_support_ip_recvtos(), + has_support_ip_tos() % Used in the test + end, fun() -> Set = fun(Sock, Value) -> socket:setopt(Sock, ip, recvtos, Value) @@ -19706,15 +19709,15 @@ api_opt_ip_recvtos_udp4(_Config) when is_list(_Config) -> end, Send = fun(Sock, Data, Dest, default) -> Msg = #{addr => Dest, - iov => [Data]}, + iov => [Data]}, socket:sendmsg(Sock, Msg); (Sock, Data, Dest, TOS) -> CMsg = #{level => ip, - type => tos, - data => TOS}, + type => tos, + data => TOS}, Msg = #{addr => Dest, - ctrl => [CMsg], - iov => [Data]}, + ctrl => [CMsg], + iov => [Data]}, socket:sendmsg(Sock, Msg) end, Recv = fun(Sock) -> @@ -19856,23 +19859,23 @@ api_opt_ip_recvtos_udp(InitState) -> end end}, - #{desc => "set tos = mincost on src sock", + #{desc => "set tos = reliability on src sock", cmd => fun(#{sock_src := Sock}) -> - ok = socket:setopt(Sock, ip, tos, mincost) + ok = socket:setopt(Sock, ip, tos, reliability) end}, - #{desc => "send req (to dst) (w tos = mincost)", + #{desc => "send req (to dst) (w tos = reliability)", cmd => fun(#{sock_src := Sock, sa_dst := Dst, send := Send}) -> Send(Sock, ?BASIC_REQ, Dst, default) end}, - %% #{desc => "send req (to dst) (w explicit tos = mincost)", + %% #{desc => "send req (to dst) (w explicit tos = reliability)", %% cmd => fun(#{sock_src := Sock, %% sa_dst := Dst, %% send := Send}) -> %% socket:setopt(Sock, otp, debug, true), - %% case Send(Sock, ?BASIC_REQ, Dst, mincost) of + %% case Send(Sock, ?BASIC_REQ, Dst, reliability) of %% ok -> %% socket:setopt(Sock, otp, debug, false), %% ok; @@ -19975,21 +19978,21 @@ api_opt_ip_recvtos_udp(InitState) -> end end}, - #{desc => "set tos = mincost on src sock", + #{desc => "set tos = reliability on src sock", cmd => fun(#{sock_src := Sock}) -> - ok = socket:setopt(Sock, ip, tos, mincost) + ok = socket:setopt(Sock, ip, tos, reliability) end}, #{desc => "send req (to dst) (w tos = mincost)", cmd => fun(#{sock_src := Sock, sa_dst := Dst, send := Send}) -> Send(Sock, ?BASIC_REQ, Dst, default) end}, - #{desc => "recv req (from src) - w tos = mincost", + #{desc => "recv req (from src) - w tos = reliability", cmd => fun(#{sock_dst := Sock, sa_src := Src, recv := Recv}) -> case Recv(Sock) of {ok, {Src, [#{level := ip, type := TOS, - value := mincost = TOSData}], + value := reliability = TOSData}], ?BASIC_REQ}} when ((TOS =:= tos) orelse (TOS =:= recvtos)) -> ?SEV_IPRINT("got expected TOS (~w) = ~w " diff --git a/lib/kernel/test/zlib_SUITE.erl b/lib/kernel/test/zlib_SUITE.erl index 52ae1b3ae6..215c91ef76 100644 --- a/lib/kernel/test/zlib_SUITE.erl +++ b/lib/kernel/test/zlib_SUITE.erl @@ -395,6 +395,7 @@ api_inflateReset(Config) when is_list(Config) -> api_inflate2(Config) when is_list(Config) -> Data = [<<1,2,2,3,3,3,4,4,4,4>>], Compressed = zlib:compress(Data), + Z1 = zlib:open(), ?m(ok, zlib:inflateInit(Z1)), ?m([], zlib:inflate(Z1, <<>>)), @@ -408,7 +409,20 @@ api_inflate2(Config) when is_list(Config) -> ?m(ok, zlib:inflateEnd(Z1)), ?m(ok, zlib:inflateInit(Z1)), ?m(?EXIT(data_error), zlib:inflate(Z1, <<2,1,2,1,2>>)), - ?m(ok, zlib:close(Z1)). + ?m(ok, zlib:close(Z1)), + + %% OTP-17299: we failed to fully flush the zlib state if we ran out of + %% input and filled the internal output buffer at the same time. + EdgeCaseData = <<"gurka", 0:16384/integer-unit:8>>, + EdgeCaseZipped = zlib:zip(EdgeCaseData), + Z2 = zlib:open(), + ?m(ok, zlib:inflateInit(Z2, -15)), + Unzipped = iolist_to_binary(zlib:inflate(Z2, EdgeCaseZipped)), + ?m(EdgeCaseData, Unzipped), + ?m(ok, zlib:inflateEnd(Z2)), + ?m(ok, zlib:close(Z2)), + + ok. %% Test inflate/3; same as inflate/2 but with the default options inverted. api_inflate3(Config) when is_list(Config) -> diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index c0da0674f0..e98a250aa2 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -1385,7 +1385,7 @@ do_add_backend_type(Name, Module) -> ModuleRegistered = lists:keymember(Module, 2, Types), do_write_table_property(schema, {mnesia_backend_types, [{Name, Module}|Types]}), - ModuleRegistered. + not ModuleRegistered. delete_backend_type(Name) -> schema_transaction(fun() -> do_delete_backend_type(Name) end). diff --git a/lib/mnesia/test/ext_test.erl b/lib/mnesia/test/ext_test.erl index ad32245a11..cd39f4a6f1 100644 --- a/lib/mnesia/test/ext_test.erl +++ b/lib/mnesia/test/ext_test.erl @@ -70,10 +70,26 @@ semantics(_Alias, _) -> init_backend() -> ?DBG(init_backend), + ct:log("init_backend ~p", [?MODULE]), + %% cheat and stuff a marker in mnesia_gvar + K = backend_init_marker(), + case try mnesia_lib:val(K) catch _:_ -> error end of + error -> + ct:log("BACKEND marker will be inserted (~p)", [?MODULE]), + mnesia_lib:set(K, true); + Other -> + ct:log("BACKEND marker already present (~p)", [?MODULE]), + error({backend_already_initialized, {?MODULE, Other}}) + end, ok. +backend_init_marker() -> + {test, ?MODULE, backend_init}. + add_aliases(_As) -> ?DBG(_As), + ct:log("add_aliases(~p)", [_As]), + true = mnesia_lib:val(backend_init_marker()), ok. remove_aliases(_) -> diff --git a/lib/mnesia/test/mnesia_config_test.erl b/lib/mnesia/test/mnesia_config_test.erl index 45da909264..08fa9ea613 100644 --- a/lib/mnesia/test/mnesia_config_test.erl +++ b/lib/mnesia/test/mnesia_config_test.erl @@ -38,6 +38,7 @@ dump_log_update_in_place/1, event_module/1, + backend_plugin_registration/1, inconsistent_database/1, max_wait_for_decision/1, send_compressed/1, @@ -104,7 +105,7 @@ all() -> [access_module, auto_repair, backup_module, debug, dir, dump_log_load_regulation, {group, dump_log_thresholds}, dump_log_update_in_place, - event_module, + event_module, backend_plugin_registration, inconsistent_database, max_wait_for_decision, send_compressed, app_test, {group, schema_config}, unknown_config]. @@ -721,6 +722,21 @@ event_module(Config) when is_list(Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +backend_plugin_registration(doc) -> + ["Ensure that backend plugins can be registered at runtime,", + "that the init_backend is called exactly once when registering", + "a plugin, and not again when registering a new alias"]; +backend_plugin_registration(Config) when is_list(Config) -> + Nodes = ?acquire_schema(1, [{default_properties, []} | Config]), + ?match(ok, mnesia:start()), + ?match({atomic,ok}, mnesia:add_backend_type(ext_ets, ext_test)), + ?match({atomic,ok}, mnesia:add_backend_type(ext_dets, ext_test)), + ?verify_mnesia(Nodes, []), + ?cleanup(1, Config). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% start_one_disc_full_then_one_disc_less(doc)-> ["Start a disk node and then a disk less one. Distribute some", diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 1cdac3cde6..848f023aad 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -672,7 +672,7 @@ do_prepare([delete_schema | Actions], Selected, All, Config, File, Line) -> end, do_prepare(Actions, Selected, All, Config, File, Line); do_prepare([create_schema | Actions], Selected, All, Config, File, Line) -> - Ext = ?BACKEND, + Ext = proplists:get_value(default_properties, Config, ?BACKEND), case diskless(Config) of true -> rpc:multicall(Selected, application, set_env, [mnesia, schema, Ext]), diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index ba1562bf15..773e752ad4 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2018. All Rights Reserved. +%% Copyright Ericsson AB 2009-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -796,12 +796,8 @@ do_spec_rel_files(#rel{name = RelName} = Rel, Sys) -> PathFlag = true, {ok, Script} = do_gen_script(Rel, Sys, MergedApps, PathFlag, Variables), {ok, BootBin} = gen_boot(Script), - Date = date(), - Time = time(), - RelIoList = io_lib:format("%% rel generated at ~w ~w\n~tp.\n\n", - [Date, Time, GenRel]), - ScriptIoList = io_lib:format("%% script generated at ~w ~w\n~tp.\n\n", - [Date, Time, Script]), + RelIoList = io_lib:format("~tp.\n\n", [GenRel]), + ScriptIoList = io_lib:format("~tp.\n\n", [Script]), [ {write_file, RelFile, to_utf8_bin_with_enc_comment(RelIoList)}, {write_file, ScriptFile, to_utf8_bin_with_enc_comment(ScriptIoList)}, @@ -1199,8 +1195,7 @@ spec_app_file(#app{name = Name, Info#app_info.modules)], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", - [date(), time(), Contents]), + AppIoList = io_lib:format("~tp.\n\n", [Contents]), [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}]; all -> %% Include all included modules @@ -1208,8 +1203,7 @@ spec_app_file(#app{name = Name, ModNames = [M#mod.name || M <- Mods, M#mod.is_included], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", - [date(), time(), Contents]), + AppIoList = io_lib:format("~tp.\n\n", [Contents]), [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}] end. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 7b86bf58d0..ca4fb4300a 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2020. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% -module(systools_make). @@ -1198,8 +1198,8 @@ generate_script(Output, Release, Appls, Flags) -> ScriptFile = Output ++ ".script", case file:open(ScriptFile, [write,{encoding,utf8}]) of {ok, Fd} -> - io:format(Fd, "%% ~s\n%% script generated at ~w ~w\n~tp.\n", - [epp:encoding_to_string(utf8), date(), time(), Script]), + io:format(Fd, "%% ~s\n~tp.\n", + [epp:encoding_to_string(utf8), Script]), case file:close(Fd) of ok -> BootFile = Output ++ ".boot", diff --git a/lib/snmp/src/agent/snmpa.erl b/lib/snmp/src/agent/snmpa.erl index 729789d487..995e6d627d 100644 --- a/lib/snmp/src/agent/snmpa.erl +++ b/lib/snmp/src/agent/snmpa.erl @@ -39,7 +39,7 @@ int_to_enum/2, int_to_enum/3, enum_to_int/2, enum_to_int/3, - info/0, info/1, old_info_format/1, + info/0, info/1, load_mib/1, load_mib/2, load_mibs/1, load_mibs/2, load_mibs/3, unload_mib/1, unload_mib/2, @@ -125,9 +125,6 @@ mib_storage_options/0 ]). --deprecated([{old_info_format, 1, "use \"new\" format instead"}]). - - -include("snmpa_atl.hrl"). -include("snmpa_internal.hrl"). -include_lib("snmp/include/snmp_types.hrl"). % type of me needed. @@ -280,16 +277,6 @@ get_next(Agent, Vars, Context) -> snmpa_agent:get_next(Agent, Vars, Context). info() -> info(snmp_master_agent). info(Agent) -> snmpa_agent:info(Agent). -old_info_format(Info) when is_list(Info) -> - {value, Vsns} = lists:keysearch(vsns, 1, Info), - {value, {_, MibInfo}} = lists:keysearch(mib_server, 1, Info), - {value, SAa} = lists:keysearch(subagents, 1, MibInfo), - {value, LoadedMibs} = lists:keysearch(loaded_mibs, 1, MibInfo), - {value, TreeSz} = lists:keysearch(tree_size_bytes, 1, MibInfo), - {value, ProcMem} = lists:keysearch(process_memory, 1, MibInfo), - {value, DbMem} = lists:keysearch(db_memory, 1, MibInfo), - [Vsns, SAa, LoadedMibs, TreeSz, ProcMem, DbMem]. - %% - diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 4a9fc3b19d..bd5bff57ab 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,23 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.11</title> <section><title>Improvements and New Features</title> @@ -504,6 +521,23 @@ </section> +<section><title>Ssh 4.9.1.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.9.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -841,6 +875,23 @@ </section> +<section><title>Ssh 4.7.6.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.7.6.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/prebuild.skip b/lib/ssh/prebuild.skip new file mode 100644 index 0000000000..ff54b7fafb --- /dev/null +++ b/lib/ssh/prebuild.skip @@ -0,0 +1 @@ +src/deps/ssh.d diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 4f4b55f5c5..29d77fa6f5 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -126,6 +126,17 @@ INTERNAL_HRL_FILES = \ ssh_transport.hrl \ ssh_xfer.hrl +DEPDIR=$(ERL_TOP)/lib/ssh/src/deps +DEP_FILE=$(DEPDIR)/ssh.d +$(shell mkdir -p $(dir $(DEP_FILE)) >/dev/null) + +ifeq ($(TARGET), win32) + # Native path without C: ignore driveletter case + ERL_TOP_NATIVE = $(shell w32_path.sh -m $(ERL_TOP) | sed "s@[a-zA-Z]:@:@") +else + ERL_TOP_NATIVE = $(ERL_TOP) +endif + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- @@ -143,11 +154,22 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/kernel/src \ $(TARGET_FILES): $(BEHAVIOUR_TARGET_FILES_2) $(BEHAVIOUR_TARGET_FILES_2): $(BEHAVIOUR_TARGET_FILES_1) -debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) +$(DEP_FILE): $(ERL_FILES) + @echo SED $(TARGET) $(ERL_TOP_NATIVE) + $(gen_verbose)erlc -M $(ERL_FILES) \ + | sed "s@[a-zA-Z]\?$(ERL_TOP_NATIVE)@../../..@g" \ + | sed "s/\.$(EMULATOR)/\.$$\(EMULATOR\)/" \ + | sed 's@^ssh_@$$(EBIN)/ssh_@' \ + | sed 's@^sshc_@$$(EBIN)/sshc_@' \ + | sed 's@^sshd_@$$(EBIN)/sshd_@' \ + > $(DEP_FILE) + +debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(DEP_FILE) clean: rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(BEHAVIOUR_TARGET_FILES) rm -f errs core *~ + rm -rf $(DEPDIR) $(APP_TARGET): $(APP_SRC) ../vsn.mk $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ @@ -174,87 +196,7 @@ release_spec: opt release_docs_spec: - -deps: - erlc -M $(ERL_FILES) \ - | sed 's@$(ERL_TOP)/lib@../..@g' \ - | sed 's/\.$(EMULATOR)/\.$$\(EMULATOR\)/' \ - | sed 's@^ssh_@$$(EBIN)/ssh_@' - -ssh.$(EMULATOR): ssh.erl ssh.hrl ssh_connect.hrl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/file.hrl -$(EBIN)/ssh_sup.$(EMULATOR): ssh_sup.erl -sshc_sup.$(EMULATOR): sshc_sup.erl -sshd_sup.$(EMULATOR): sshd_sup.erl ssh.hrl -$(EBIN)/ssh_connection_sup.$(EMULATOR): ssh_connection_sup.erl -$(EBIN)/ssh_connection.$(EMULATOR): ssh_connection.erl ssh.hrl ssh_connect.hrl \ - ssh_transport.hrl -$(EBIN)/ssh_connection_handler.$(EMULATOR): ssh_connection_handler.erl ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_kexint.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_userauth_client.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_userauth_server.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_shell.$(EMULATOR): ssh_shell.erl ssh_connect.hrl -$(EBIN)/ssh_system_sup.$(EMULATOR): ssh_system_sup.erl ssh.hrl -$(EBIN)/ssh_subsystem_sup.$(EMULATOR): ssh_subsystem_sup.erl -$(EBIN)/ssh_channel_sup.$(EMULATOR): ssh_channel_sup.erl ssh.hrl -$(EBIN)/ssh_acceptor_sup.$(EMULATOR): ssh_acceptor_sup.erl ssh.hrl -$(EBIN)/ssh_acceptor.$(EMULATOR): ssh_acceptor.erl ssh.hrl -$(EBIN)/ssh_agent.$(EMULATOR): ssh_agent.erl ssh.hrl ssh_agent.hrl -$(EBIN)/ssh_app.$(EMULATOR): ssh_app.erl -$(EBIN)/ssh_auth.$(EMULATOR): ssh_auth.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl ssh_auth.hrl ssh_transport.hrl -$(EBIN)/ssh_bits.$(EMULATOR): ssh_bits.erl ssh.hrl -$(EBIN)/ssh_cli.$(EMULATOR): ssh_cli.erl ssh.hrl ssh_connect.hrl -$(EBIN)/ssh_file.$(EMULATOR): ssh_file.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/file.hrl ssh.hrl -$(EBIN)/ssh_io.$(EMULATOR): ssh_io.erl ssh.hrl -$(EBIN)/ssh_info.$(EMULATOR): ssh_info.erl -$(EBIN)/ssh_message.$(EMULATOR): ssh_message.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl ssh_connect.hrl ssh_auth.hrl ssh_transport.hrl -$(EBIN)/ssh_no_io.$(EMULATOR): ssh_no_io.erl ssh_transport.hrl -$(EBIN)/ssh_sftp.$(EMULATOR): ssh_sftp.erl \ - ../../kernel/include/file.hrl ssh.hrl \ - ssh_xfer.hrl -$(EBIN)/ssh_sftpd.$(EMULATOR): ssh_sftpd.erl \ - ../../kernel/include/file.hrl ssh.hrl \ - ssh_xfer.hrl -$(EBIN)/ssh_sftpd_file.$(EMULATOR): ssh_sftpd_file.erl -$(EBIN)/ssh_transport.$(EMULATOR): ssh_transport.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/inet.hrl \ - ssh_transport.hrl ssh.hrl -$(EBIN)/ssh_xfer.$(EMULATOR): ssh_xfer.erl ssh.hrl ssh_xfer.hrl -$(EBIN)/ssh_sftpd_file_api.$(EMULATOR): ssh_sftpd_file_api.erl -$(EBIN)/ssh_client_channel.$(EMULATOR): ssh_client_channel.erl ssh_connect.hrl -$(EBIN)/ssh_channel.$(EMULATOR): ssh_channel.erl ssh_connect.hrl -$(EBIN)/ssh_daemon_channel.$(EMULATOR): ssh_daemon_channel.erl -$(EBIN)/ssh_server_channel.$(EMULATOR): ssh_server_channel.erl -$(EBIN)/ssh_client_key_api.$(EMULATOR): ssh_client_key_api.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl -$(EBIN)/ssh_server_key_api.$(EMULATOR): ssh_server_key_api.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl - +# ---------------------------------------------------- +# Dependencies +# ---------------------------------------------------- +-include $(DEP_FILE) diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 04627052e5..a3db492f0f 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -26,6 +26,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- MODULES= \ + ssh_cth \ ssh_algorithms_SUITE \ ssh_options_SUITE \ ssh_basic_SUITE \ diff --git a/lib/ssh/test/ssh_cth.erl b/lib/ssh/test/ssh_cth.erl new file mode 100644 index 0000000000..09d863cc51 --- /dev/null +++ b/lib/ssh/test/ssh_cth.erl @@ -0,0 +1,64 @@ +-module(ssh_cth). + +-export([id/1, + init/2, + pre_init_per_suite/3, + pre_end_per_suite/3, + pre_init_per_group/4, + pre_end_per_group/4, + post_end_per_group/5, + pre_init_per_testcase/3, + pre_end_per_testcase/4 + ]). + +-record(c, { + suite, % suite name + groups = [], % Group path in reversed order + n % test case number + }). + +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +init(_Id, _Opts) -> + {ok, #c{n=1}}. + +pre_init_per_suite(Suite, Config, CTHState) -> + {Config, CTHState#c{suite=Suite}}. + +pre_end_per_suite(Suite, Config, State) -> + ct:pal("BEGIN ~p:end_per_suite(...)", [Suite]), + {Config, State}. + + +pre_init_per_group(_Suite, Group, Config ,State) -> + {Config, State#c{groups = (State#c.groups ++ [Group])}}. + +pre_end_per_group(Suite, Group, Config, State) -> + ct:pal("BEGIN ~p:end_per_group(~p,...)", [Suite,Group]), + {Config, State}. + +post_end_per_group(_Suite, Group, _Config, Return, State) -> + {Return, State#c{groups = lists:reverse(lists:reverse(State#c.groups)--[Group])}}. + + +pre_init_per_testcase(SuiteName, Config, CTHState) -> + ct:pal("########## ~p ~p ~s ~p~n", [CTHState#c.suite, CTHState#c.n, groups(Config), SuiteName]), + {Config, CTHState#c{n = CTHState#c.n + 1}}. + +pre_end_per_testcase(Suite,TC,Config,State) -> + ct:pal("BEGIN ~p:end_per_testcase(~p,...)", [Suite,TC]), + {Config, State}. + + +groups(Config) -> + F = fun(X) -> io_lib:format("~w",[X]) end, + io_lib:format("~s", [lists:join("/", lists:map(F,get_groups(Config)))]). + +get_groups(Config) -> + P = proplists:get_value(tc_group_path, Config, []) ++ + [proplists:get_value(tc_group_properties, Config, [])], + [Name || L <- P, + is_list(L), + {name,Name} <- L]. + diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index dff8da55c7..2a4487a813 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.11 +SSH_VSN = 4.11.1 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 7ee4b308e3..db0852c5b6 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -94,6 +94,22 @@ </section> +<section><title>SSL 10.2.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix handling of emulated socket options, the previous + patch was incomplete,</p> + <p> + Own Id: OTP-17305</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 10.2.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index 424bf05791..0b4211e1c3 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -86,8 +86,7 @@ trusted_cert_and_paths(Chain, CertDbHandle, CertDbRef, PartialChainHandler) -> Result -> Result end - end, Paths). - + end, Paths). %%-------------------------------------------------------------------- -spec certificate_chain(undefined | binary() | #'OTPCertificate'{} , db_handle(), certdb_ref() | {extracted, list()}) -> {error, no_cert} | {ok, der_cert() | undefined, [der_cert()]}. @@ -533,7 +532,12 @@ handle_incomplete_chain([PeerCert| _] = Chain0, PartialChainHandler, Default, Ce %% See if we have the certificates to rebuild it. case certificate_chain(PeerCert, CertDbHandle, CertDbRef) of {ok, _, [PeerCert | _] = Chain} when Chain =/= Chain0 -> %% Chain candidate found - handle_partial_chain(lists:reverse(Chain), PartialChainHandler, CertDbHandle, CertDbRef); + case lists:prefix(Chain0, Chain) of + true -> + handle_partial_chain(lists:reverse(Chain), PartialChainHandler, CertDbHandle, CertDbRef); + false -> + Default + end; _ -> Default end. diff --git a/lib/ssl/test/openssl_cipher_suite_SUITE.erl b/lib/ssl/test/openssl_cipher_suite_SUITE.erl index fb1f28aa4a..ad0ecec8b7 100644 --- a/lib/ssl/test/openssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/openssl_cipher_suite_SUITE.erl @@ -401,8 +401,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; TestCase == ecdhe_rsa_rc4_128; TestCase == ecdhe_ecdsa_rc4_128; TestCase == dh_anon_rc4_128 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(rc4, SupCiphers) of + case supported_cipher(rc4, "RC4") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -413,8 +412,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; TestCase == psk_aes_128_ccm_8; TestCase == dhe_psk_aes_128_ccm_8; TestCase == ecdhe_psk_aes_128_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_128_ccm, SupCiphers) of + case supported_cipher(aes_128_ccm, "AES128-CCM8") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -425,8 +423,8 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; TestCase == psk_aes_256_ccm_8; TestCase == dhe_psk_aes_256_ccm_8; TestCase == ecdhe_psk_aes_256_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_256_ccm, SupCiphers) of + + case supported_cipher(aes_256_ccm, "AES128-CCM8") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -434,10 +432,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; {skip, "Missing AES_256_CCM crypto support"} end; init_per_testcase(aes_256_gcm_sha384, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_256_gcm, SupCiphers)) andalso - (lists:member(sha384, SupHashs)) of + case supported_cipher(aes_256_gcm, "AES256_GCM", sha384) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -445,10 +440,7 @@ init_per_testcase(aes_256_gcm_sha384, Config) -> {skip, "Missing AES_256_GCM crypto support"} end; init_per_testcase(aes_128_gcm_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case lists:member(aes_128_gcm, SupCiphers) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_gcm, "AES128_GCM", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -457,10 +449,7 @@ init_per_testcase(aes_128_gcm_sha256, Config) -> end; init_per_testcase(chacha20_poly1305_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(chacha20_poly1305, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(chacha20_poly1305_sha256, "CHACHA", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -468,10 +457,7 @@ init_per_testcase(chacha20_poly1305_sha256, Config) -> {skip, "Missing CHACHA20_POLY1305 crypto support"} end; init_per_testcase(aes_128_ccm_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_128_ccm, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_ccm, "AES128_CCM", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -480,10 +466,7 @@ init_per_testcase(aes_128_ccm_sha256, Config) -> end; init_per_testcase(aes_128_ccm_8_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_128_ccm, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_ccm, "AES128_CCM8", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -493,8 +476,7 @@ init_per_testcase(aes_128_ccm_8_sha256, Config) -> init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_ccm; TestCase == ecdhe_ecdsa_with_aes_128_ccm_8-> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_128_ccm, SupCiphers) of + case supported_cipher(aes_128_ccm, "AES128_CCM") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -504,8 +486,8 @@ init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_cc init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_256_ccm; TestCase == ecdhe_ecdsa_with_aes_256_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_256_ccm, SupCiphers) of + + case supported_cipher(aes_256_ccm, "AES256_CCM") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -983,3 +965,13 @@ test_ciphers(Kex, Cipher, Version) -> openssl_suitestr_to_map(OpenSSLSuiteStrs) -> [ssl_cipher_format:suite_openssl_str_to_map(SuiteStr) || SuiteStr <- OpenSSLSuiteStrs]. + + +supported_cipher(Cipher, CipherStr) -> + SupCrypto = proplists:get_value(ciphers, crypto:supports()), + SupOpenssl = [OCipher || OCipher <- ssl_test_lib:openssl_ciphers(), string:find(OCipher, CipherStr) =/= nomatch], + lists:member(Cipher, SupCrypto) andalso SupOpenssl =/= []. + +supported_cipher(Cipher, CipherStr, Hash) -> + Hashes = proplists:get_value(hashs, crypto:supports()), + supported_cipher(Cipher, CipherStr) andalso lists:member(Hash, Hashes). diff --git a/lib/ssl/test/openssl_session_SUITE.erl b/lib/ssl/test/openssl_session_SUITE.erl index 08369733dc..2dcec86579 100644 --- a/lib/ssl/test/openssl_session_SUITE.erl +++ b/lib/ssl/test/openssl_session_SUITE.erl @@ -168,7 +168,6 @@ reuse_session_erlang_server(Config) when is_list(Config) -> {_, ServerNode, _} = ssl_test_lib:run_where(Config), - Ciphers = common_ciphers(Version), Data = "From openssl to erlang", @@ -176,14 +175,13 @@ reuse_session_erlang_server(Config) when is_list(Config) -> {from, self()}, {mfa, {ssl_test_lib, active_recv, [length(Data)]}}, {reconnect_times, 5}, - {options, [{ciphers, Ciphers}, - {versions, [Version]}| ServerOpts]}]), + {options, [{versions, [Version]}| ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), {_Client, OpenSSLPort} = ssl_test_lib:start_client(openssl, [{port, Port}, {reconnect, true}, - {options, [{ciphers, Ciphers} | ClientOpts]}, + {options, ClientOpts}, return_port], Config), true = port_command(OpenSSLPort, Data), @@ -200,9 +198,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> Version = ssl_test_lib:protocol_version(Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - Ciphers = common_ciphers(Version), Server = ssl_test_lib:start_server(openssl, [], - [{server_opts, [{ciphers, Ciphers} | ServerOpts]} | Config]), + [{server_opts, ServerOpts} | Config]), Port = ssl_test_lib:inet_port(Server), Client0 = @@ -212,7 +209,6 @@ reuse_session_erlang_client(Config) when is_list(Config) -> {from, self()}, {options, [{reuse_sessions, save}, {verify, verify_peer}, - {ciphers, Ciphers}, {versions, [Version]} | ClientOpts]}]), SID = receive @@ -226,9 +222,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [ {ciphers, Ciphers}, - {versions, [Version]}, - {reuse_session, SID} | ClientOpts]}]), + {from, self()}, {options, [{versions, [Version]}, + {reuse_session, SID} | ClientOpts]}]), receive {Client1, SID} -> ok @@ -245,8 +240,7 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [{ciphers, Ciphers}, - {versions, [Version]} | ClientOpts]}]), + {from, self()}, {options, [{versions, [Version]} | ClientOpts]}]), receive {Client2, ID} -> case ID of @@ -258,14 +252,3 @@ reuse_session_erlang_client(Config) when is_list(Config) -> end, ssl_test_lib:close(Client2). - -%%-------------------------------------------------------------------- -%% Internal functions ------------------------------------------------ -%%-------------------------------------------------------------------- - -common_ciphers(Version) -> - OpenSSLCiphers = ssl_test_lib:openssl_ciphers(), - ErlOpenSSLCiphers = [ssl:str_to_suite(C) || C <- OpenSSLCiphers], - ErlCiphers = ssl:cipher_suites(all, Version), - [Suite || Suite <- ErlOpenSSLCiphers, lists:member(Suite, ErlCiphers)]. - diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index b737c41ba2..a82b682526 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -73,7 +73,9 @@ fake_root_no_intermediate_legacy/0, fake_root_no_intermediate_legacy/1, fake_intermediate_cert/0, - fake_intermediate_cert/1 + fake_intermediate_cert/1, + incompleat_chain_length/0, + incompleat_chain_length/1 ]). %% Apply export @@ -124,7 +126,8 @@ basic_tests() -> fake_root_no_intermediate, fake_root_legacy, fake_root_no_intermediate_legacy, - fake_intermediate_cert + fake_intermediate_cert, + incompleat_chain_length ]. options_tests() -> @@ -730,6 +733,74 @@ fake_intermediate_cert(Config) when is_list(Config) -> ssl_test_lib:check_client_alert(Client1, bad_certificate). +incompleat_chain_length() -> + [{doc,"Test that attempts to reconstruct incomplete chains does not make shorter incomplete chains"}]. +incompleat_chain_length(Config) when is_list(Config)-> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + OtherROOT = public_key:pkix_test_root_cert("OTHER SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(3)}, + {extensions, Ext}]), + + + #{client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := ServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => OtherROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}], + [{key, ssl_test_lib:hardcode_rsa_key(3)}] + ], + peer => [{key, ssl_test_lib:hardcode_rsa_key(1)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + + VerifyFun = {fun(_,{bad_cert, unknown_ca}, UserState) -> + %% accept this error to provoke the + %% building of an shorter incomplete chain + %% than the one recived + {valid, UserState}; + (_,{extension, _} = Extension, #{ext := N} = UserState) -> + ct:pal("~p", [Extension]), + {unknown, UserState#{ext => N +1}}; + (_, valid, #{intermediates := N} = UserState) -> + {valid, UserState#{intermediates => N +1}}; + (_, valid_peer, #{intermediates := 2, + ext := 1} = UserState) -> + {valid, UserState}; + (_, valid_peer, UserState) -> + ct:pal("~p", [UserState]), + {error, {bad_cert, too_short_path}} + end, #{intermediates => 0, + ext => 0}}, + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerConf} + ]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{verify, verify_peer}, {verify_fun, VerifyFun} | ClientConf]}]), + ssl_test_lib:check_result(Client, ok, Server, ok). + %%-------------------------------------------------------------------- %% callback functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 2123ac5def..67be91196f 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -828,14 +828,17 @@ localhost_ip(InetVer) -> localhost_ipstr(InetVer) -> {ok, Addr} = inet:getaddr(net_adm:localhost(), InetVer), - case InetVer of - inet -> - lists:flatten(io_lib:format("'{~p,~p,~p,~p}'", - erlang:tuple_to_list(Addr))); - inet6 -> - lists:flatten(io_lib:format("'{~p,~p,~p,~p,~p,~p,~p,~p}'", - erlang:tuple_to_list(Addr))) - end. + Str = case InetVer of + inet -> + io_lib:format("{~p,~p,~p,~p}", erlang:tuple_to_list(Addr)); + inet6 -> + io_lib:format("{~p,~p,~p,~p,~p,~p,~p,~p}", erlang:tuple_to_list(Addr)) + end, + Qouted = case os:type() of + {win32, _} -> Str; + _ -> [$',Str,$'] + end, + lists:flatten(Qouted). inet_ver() -> inet. diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index d90d566c14..e596d60aa3 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -73,7 +73,7 @@ </func> <func> - <name name="filtermap" arity="2" since=""/> + <name name="filtermap" arity="2" since="OTP 24.0"/> <fsummary>Filter and map a map.</fsummary> <desc> <p>Returns a map <c><anno>Map</anno></c> that is the result of calling diff --git a/lib/stdlib/doc/src/proplists.xml b/lib/stdlib/doc/src/proplists.xml index 6aed2d45e1..003b3875e2 100644 --- a/lib/stdlib/doc/src/proplists.xml +++ b/lib/stdlib/doc/src/proplists.xml @@ -138,7 +138,7 @@ expand([{{foo, true}, [bar, baz]}], [{foo, false}, fie, foo, fum])</code> </func> <func> - <name name="from_map" arity="1" since=""/> + <name name="from_map" arity="1" since="OTP 24.0"/> <fsummary></fsummary> <desc> <p>Converts the map <c><anno>Map</anno></c> to a property list.</p> @@ -373,7 +373,7 @@ split([{c, 2}, {e, 1}, a, {c, 3, 4}, d, {b, 5}, b], [a, b, c])</code> </func> <func> - <name name="to_map" arity="1" since=""/> + <name name="to_map" arity="1" since="OTP 24.0"/> <fsummary></fsummary> <desc> <p>Converts the property list <c><anno>List</anno></c> to a map.</p> @@ -397,7 +397,7 @@ to_map([a, {b, 1}, {c, 2}, {c, 3}])</code> </func> <func> - <name name="to_map" arity="2" since=""/> + <name name="to_map" arity="2" since="OTP 24.0"/> <fsummary></fsummary> <desc> <p>Converts the property list <c><anno>List</anno></c> to a map after diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 08320f4ec4..a4335604bc 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -122,7 +122,7 @@ <title>Original API</title> </fsdescription> <func> - <name name="all" arity="2" since=""/> + <name name="all" arity="2" since="OTP 24.0"/> <fsummary>Return <c>true</c> if all items in a queue satisfy <c>Pred</c>.</fsummary> <desc> @@ -133,7 +133,7 @@ </func> <func> - <name name="any" arity="2" since=""/> + <name name="any" arity="2" since="OTP 24.0"/> <fsummary>Return <c>true</c> if any of the items in a queue satisfy <c>Pred</c>.</fsummary> <desc> @@ -144,7 +144,7 @@ </func> <func> - <name name="delete" arity="2" since=""/> + <name name="delete" arity="2" since="OTP 24.0"/> <fsummary>Delete an item from the front of a queue.</fsummary> <desc> <p>Returns a copy of <c><anno>Q1</anno></c> where the first item @@ -154,7 +154,7 @@ </func> <func> - <name name="delete_r" arity="2" since=""/> + <name name="delete_r" arity="2" since="OTP 24.0"/> <fsummary>Delete an item from the rear of a queue.</fsummary> <desc> <p>Returns a copy of <c><anno>Q1</anno></c> where the last item @@ -164,7 +164,7 @@ </func> <func> - <name name="delete_with" arity="2" since=""/> + <name name="delete_with" arity="2" since="OTP 24.0"/> <fsummary>Delete an item matching a predicate from the front of a queue.</fsummary> <desc> @@ -175,7 +175,7 @@ </func> <func> - <name name="delete_with_r" arity="2" since=""/> + <name name="delete_with_r" arity="2" since="OTP 24.0"/> <fsummary>Delete an item matching a predicate from the rear of a queue.</fsummary> <desc> @@ -207,7 +207,7 @@ </func> <func> - <name name="filtermap" arity="2" since=""/> + <name name="filtermap" arity="2" since="OTP 24.0"/> <fsummary>Filter and map a queue.</fsummary> <desc> <p>Returns a queue <c><anno>Q2</anno></c> that is the result of calling @@ -222,7 +222,7 @@ </func> <func> - <name name="fold" arity="3" since=""/> + <name name="fold" arity="3" since="OTP 24.0"/> <fsummary>Fold a function over a queue.</fsummary> <desc> <p>Calls <c><anno>Fun</anno>(<anno>Item</anno>, <anno>AccIn</anno>)</c> diff --git a/lib/stdlib/examples/erl_id_trans.erl b/lib/stdlib/examples/erl_id_trans.erl index 264148eaf4..292c06506a 100644 --- a/lib/stdlib/examples/erl_id_trans.erl +++ b/lib/stdlib/examples/erl_id_trans.erl @@ -31,7 +31,7 @@ parse_transform(Forms, _Options) -> forms(Forms). parse_transform_info() -> - #{columns => true}. + #{error_location => column}. %% forms(Fs) -> lists:map(fun (F) -> form(F) end, Fs). diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 3ab281dbad..1e34728d0d 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -89,8 +89,6 @@ obsolete(net, sleep, 1) -> {deprecated, "use 'receive after T -> ok end' instead"}; obsolete(queue, lait, 1) -> {deprecated, "use queue:liat/1 instead"}; -obsolete(snmpa, old_info_format, 1) -> - {deprecated, "use \"new\" format instead", "OTP 24"}; obsolete(snmpm, async_get, 3) -> {deprecated, "use snmpm:async_get2/3 instead.", "OTP 25"}; obsolete(snmpm, async_get, 4) -> diff --git a/lib/stdlib/test/maps_SUITE.erl b/lib/stdlib/test/maps_SUITE.erl index 4159ea6c3b..4860e39b70 100644 --- a/lib/stdlib/test/maps_SUITE.erl +++ b/lib/stdlib/test/maps_SUITE.erl @@ -495,8 +495,12 @@ iter_kv(I) -> t_put_opt(Config) when is_list(Config) -> Value = id(#{complex => map}), - Map = id(#{a => Value}), - true = erts_debug:same(maps:put(a, Value, Map), Map), + Small = id(#{a => Value}), + true = erts_debug:same(maps:put(a, Value, Small), Small), + + LargeBase = maps:from_list([{I,I}||I<-lists:seq(1,200)]), + Large = LargeBase#{a => Value}, + true = erts_debug:same(maps:put(a, Value, Large), Large), ok. t_merge_opt(Config) when is_list(Config) -> diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index d15ffb9b40..354b65c9a3 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -5185,7 +5185,7 @@ function_type(Type) -> %% Arguments = [erl_parse()] %% Type = erl_parse() --spec function_type('any_arity' | syntaxTree(), syntaxTree()) -> syntaxTree(). +-spec function_type('any_arity' | [syntaxTree()], syntaxTree()) -> syntaxTree(). function_type(Arguments, Return) -> tree(function_type, diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 47373ca1fe..358bcf5043 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -141,6 +141,22 @@ </section> +<section><title>Tools 3.3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p><c>cover</c> would crash when compiling a module + having an exported function named <c>clauses</c>.</p> + <p> + Own Id: OTP-17162 Aux Id: GH-4549, PR-2997, PR-4555, + elixir-lang/elixir#10666 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/otp_versions.table b/otp_versions.table index 9a8bd80298..4bfa73a5f9 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,4 +1,6 @@ +OTP-23.3.1 : ssh-4.11.1 # asn1-5.0.14 common_test-1.20 compiler-7.6.7 crypto-4.9 debugger-5.0 dialyzer-4.3.1 diameter-2.2.3 edoc-0.12 eldap-1.2.9 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11.1 kernel-7.3 megaco-3.19.5 mnesia-4.19 observer-2.9.5 odbc-2.13.3 os_mon-2.6.1 parsetools-2.2 public_key-1.10 reltool-0.8 runtime_tools-1.16 sasl-4.0.2 snmp-5.8 ssl-10.3 stdlib-3.14.1 syntax_tools-2.5 tftp-1.0.2 tools-3.4.4 wx-1.9.3 xmerl-1.3.26 : OTP-23.3 : common_test-1.20 compiler-7.6.7 crypto-4.9 dialyzer-4.3.1 eldap-1.2.9 erts-11.2 jinterface-1.11.1 kernel-7.3 mnesia-4.19 odbc-2.13.3 public_key-1.10 runtime_tools-1.16 sasl-4.0.2 snmp-5.8 ssh-4.11 ssl-10.3 stdlib-3.14.1 syntax_tools-2.5 tools-3.4.4 wx-1.9.3 # asn1-5.0.14 debugger-5.0 diameter-2.2.3 edoc-0.12 erl_docgen-1.0.2 erl_interface-4.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 megaco-3.19.5 observer-2.9.5 os_mon-2.6.1 parsetools-2.2 reltool-0.8 tftp-1.0.2 xmerl-1.3.26 : +OTP-23.2.7.2 : ssl-10.2.4.2 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11 kernel-7.2.1 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.8 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.7.1 : ssl-10.2.4.1 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11 kernel-7.2.1 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.8 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.7 : kernel-7.2.1 ssl-10.2.4 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.8 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.6 : inets-7.3.2 ssh-4.10.8 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssl-10.2.3 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : @@ -19,6 +21,7 @@ OTP-23.0.3 : compiler-7.6.2 erts-11.0.3 # asn1-5.0.13 common_test-1.19 crypto-4. OTP-23.0.2 : erts-11.0.2 megaco-3.19.1 # asn1-5.0.13 common_test-1.19 compiler-7.6.1 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0.1 : compiler-7.6.1 erts-11.0.1 # asn1-5.0.13 common_test-1.19 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0 : asn1-5.0.13 common_test-1.19 compiler-7.6 crypto-4.7 debugger-5.0 dialyzer-4.2 edoc-0.12 erl_docgen-1.0 erl_interface-4.0 erts-11.0 eunit-2.5 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tools-3.4 wx-1.9.1 xmerl-1.3.25 # diameter-2.2.3 eldap-1.2.8 et-1.6.4 ftp-1.0.4 reltool-0.8 tftp-1.0.2 : +OTP-22.3.4.17 : erts-10.7.2.9 kernel-6.5.2.2 ssh-4.9.1.3 tools-3.3.1.1 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 crypto-4.6.5.2 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.16 : erts-10.7.2.8 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 crypto-4.6.5.2 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.15 : crypto-4.6.5.2 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 erts-10.7.2.7 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.14 : compiler-7.5.4.3 erts-10.7.2.7 # asn1-5.0.12 common_test-1.18.2 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : @@ -67,6 +70,7 @@ OTP-22.0.3 : compiler-7.4.2 dialyzer-4.0.1 erts-10.4.2 ssl-9.3.2 stdlib-3.9.2 # OTP-22.0.2 : compiler-7.4.1 crypto-4.5.1 erts-10.4.1 stdlib-3.9.1 # asn1-5.0.9 common_test-1.17.3 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3.1 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.1 : ssl-9.3.1 # asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 stdlib-3.9 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0 : asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3 stdlib-3.9 syntax_tools-2.2 tools-3.2 wx-1.8.8 xmerl-1.3.21 # diameter-2.2.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2 parsetools-2.1.8 tftp-1.0.1 : +OTP-21.3.8.22 : erts-10.3.5.17 ssh-4.7.6.6 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.21 : erts-10.3.5.16 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.20 : erl_interface-3.11.3.1 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erts-10.3.5.15 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.19 : crypto-4.4.2.3 erts-10.3.5.15 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : diff --git a/system/doc/general_info/deprecations_24.inc b/system/doc/general_info/deprecations_24.inc index 0eb859789a..d508aca7d3 100644 --- a/system/doc/general_info/deprecations_24.inc +++ b/system/doc/general_info/deprecations_24.inc @@ -28,3 +28,10 @@ </p> </section> + <section> + <title>?NO_APP macro</title> + <p> + The ?NO_APP macro in the edoc include file <c>edoc_doclet.hrl</c> has + been deprecated. + </p> + </section> |