diff options
219 files changed, 11553 insertions, 5267 deletions
diff --git a/.gitignore b/.gitignore index 91e0c76314..78f7f5b7b4 100644 --- a/.gitignore +++ b/.gitignore @@ -36,9 +36,9 @@ aarch64-unknown-linux-android arm-unknown-linux-androideabi armv7l-unknown-linux-gnueabihf i686-pc-linux-gnu -x86_64-unknown-linux-gnu +x86_64-pc-linux-gnu i386-apple-darwin[0-9]*.[0-9]*.[0-9]* -arm-apple-darwin[0-9]*.[0-9]*.[0-9]* +aarch64-apple-darwin[0-9]*.[0-9]*.[0-9]* x86_64-apple-darwin[0-9]*.[0-9]*.[0-9]* x86_64-unknown-freebsd[0-9]*.[0-9]* sparc-sun-solaris[0-9]*.[0-9]* diff --git a/HOWTO/INSTALL-WIN32-OLD.md b/HOWTO/INSTALL-WIN32-OLD.md index 0f86d467ec..98718d0cc1 100644 --- a/HOWTO/INSTALL-WIN32-OLD.md +++ b/HOWTO/INSTALL-WIN32-OLD.md @@ -44,7 +44,7 @@ Erlang, for example, is `otp_win64_%OTP-REL%.exe`. If you feel comfortable with the environment and build system, and have all the necessary tools, you have a great opportunity to make the Erlang/OTP distribution for Windows better. Please submit -any suggestions to our [JIRA] [2] and patches to our [git project] [3] to let +any suggestions or patches to our [git project] [2] to let them find their way into the next version of Erlang. If making changes to the build system (like makefiles etc) please bear in mind that the same makefiles are used on Unix/VxWorks, so that your changes @@ -892,7 +892,6 @@ cannot be built as MsysGIT/MSYS does not handle symbolic links. [1]: http://www.erlang.org/static/doc/mailinglist.html - [2]: https://github.com/erlang/otp/issues - [3]: https://github.com/erlang/otp + [2]: https://github.com/erlang/otp [?TOC]: true diff --git a/HOWTO/INSTALL-WIN32.md b/HOWTO/INSTALL-WIN32.md index 3de5ee428e..571a2f023d 100644 --- a/HOWTO/INSTALL-WIN32.md +++ b/HOWTO/INSTALL-WIN32.md @@ -28,7 +28,7 @@ version of Erlang, for example, is `otp_win64_%OTP-REL%.exe`. If you feel comfortable with the environment and build system, and have all the necessary tools, you have a great opportunity to make the Erlang/OTP distribution for Windows better. Please submit -any suggestions to our [JIRA] [1] and patches to our [git project] [2] to let +any suggestions or patches to our [git project] [1] to let them find their way into the next version of Erlang. If making changes to the build system (like makefiles etc) please bear in mind that the same makefiles are used on Unix/VxWorks, so that your changes @@ -482,7 +482,6 @@ Frequently Asked Questions NSIS 3.05, Win32 OpenSSL 1.1.1d and wxWidgets-3.1.3. - [1]: https://github.com/erlang/otp/issues - [2]: https://github.com/erlang/otp + [1]: https://github.com/erlang/otp [?TOC]: true diff --git a/OTP_VERSION b/OTP_VERSION index f7635638ff..7a5aa45517 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -24.0-rc1 +24.0-rc2 diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex 9e81b610a5..80b2e770fa 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/configure.src b/configure.src index 0ced1500ac..c0a195b147 100644 --- a/configure.src +++ b/configure.src @@ -148,7 +148,14 @@ while test $# != 0; do case $1 in --without-*) skip_app=`expr "$1" : '--without-\(.*\)'` - if test -d "lib/$skip_app"; then + if [ "$skip_app" = "stdlib" ] || + [ "$skip_app" = "kernel" ] || + [ "$skip_app" = "sasl" ] || + [ "$skip_app" = "compiler" ] || + [ "$skip_app" = "erl_interface" ]; then + echo "ERROR: $skip_app is a mandatory application" 1>&2 + exit 1 + elif test -d "lib/$skip_app"; then skip_applications="$skip_applications $skip_app" fi;; *) diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 325fd5c7f6..9b7a502d2d 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -364,19 +364,10 @@ dnl dnl Try to find perl version 5. If found set PERL to the absolute path dnl of the program, if not found set PERL to false. dnl -dnl On some systems /usr/bin/perl is perl 4 and e.g. -dnl /usr/local/bin/perl is perl 5. We try to handle this case by -dnl putting a couple of -dnl Tries to handle the case that there are two programs called perl -dnl in the path and one of them is perl 5 and the other isn't. -dnl AC_DEFUN(LM_PROG_PERL5, [AC_PATH_PROGS(PERL, perl5 perl, false, /usr/local/bin:/opt/local/bin:/usr/local/gnu/bin:${PATH}) -changequote(, )dnl -dnl[ That bracket is needed to balance the right bracket below -if test "$PERL" = "false" || $PERL -e 'exit ($] >= 5)'; then -changequote([, ])dnl +if test "$PERL" = "false"; then ac_cv_path_PERL=false PERL=false dnl AC_MSG_WARN(perl version 5 not found) diff --git a/erts/autoconf/config.guess b/erts/autoconf/config.guess index f7eb141e75..1972fda8eb 100755 --- a/erts/autoconf/config.guess +++ b/erts/autoconf/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2015 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2015-03-04' +timestamp='2021-01-25' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ timestamp='2015-03-04' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a @@ -27,19 +27,19 @@ timestamp='2015-03-04' # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess # # Please send patches to <config-patches@gnu.org>. -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2015 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -84,8 +84,6 @@ if test $# != 0; then exit 1 fi -trap 'exit 1' 1 2 15 - # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a @@ -96,66 +94,89 @@ trap 'exit 1' 1 2 15 # Portable tmp directory creation inspired by the Autoconf team. -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; - for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ; set_cc_for_build= ;' +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 + +set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then +if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi -UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown -UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown -UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown +UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown +UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown +UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown -case "${UNAME_SYSTEM}" in +case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - LIBC=gnu + LIBC=unknown - eval $set_cc_for_build - cat <<-EOF > $dummy.c + set_cc_for_build + cat <<-EOF > "$dummy.c" #include <features.h> #if defined(__UCLIBC__) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc - #else + #elif defined(__GLIBC__) LIBC=gnu + #else + #include <stdarg.h> + /* First heuristic to detect musl libc. */ + #ifdef __DEFINED_va_list + LIBC=musl + #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')" + + # Second heuristic to detect musl libc. + if [ "$LIBC" = unknown ] && + command -v ldd >/dev/null && + ldd --version 2>&1 | grep -q ^musl; then + LIBC=musl + fi + + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + if [ "$LIBC" = unknown ]; then + LIBC=gnu + fi ;; esac # Note: order is significant - the case branches are not exclusive. -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -167,29 +188,32 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ - /sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || \ - echo unknown)` - case "${UNAME_MACHINE_ARCH}" in + UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ + /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + echo unknown)) + case "$UNAME_MACHINE_ARCH" in + aarch64eb) machine=aarch64_be-unknown ;; armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) - arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` - endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` - machine=${arch}${endian}-unknown + arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,') + endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p') + machine="${arch}${endian}"-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. - case "${UNAME_MACHINE_ARCH}" in - arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build + # to ELF recently (or will in the future) and ABI. + case "$UNAME_MACHINE_ARCH" in + earm*) + os=netbsdelf + ;; + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -205,10 +229,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ;; esac # Determine ABI tags. - case "${UNAME_MACHINE_ARCH}" in + case "$UNAME_MACHINE_ARCH" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr") ;; esac # The OS release @@ -216,112 +240,125 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in + case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2) ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}${abi}" + echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//') + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//') + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//') + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" + exit ;; + *:MidnightBSD:*:*) + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) - echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" + exit ;; + *:OS108:*:*) + echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd${UNAME_RELEASE} + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:Sortix:*:*) + echo "$UNAME_MACHINE"-unknown-sortix + exit ;; + *:Twizzler:*:*) + echo "$UNAME_MACHINE"-unknown-twizzler + exit ;; + *:Redox:*:*) + echo "$UNAME_MACHINE"-unknown-redox + exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}') ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}') ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on # OSF/1 and Tru64 systems produced since 1995. I hope that # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1) case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; - Alpha\ *:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # Should we change UNAME_MACHINE based on the output of uname instead - # of the specific Alpha model? - echo alpha-pc-interix - exit ;; - 21064:Windows_NT:50:3) - echo alpha-dec-winnt3.5 - exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos + echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos + echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition @@ -333,7 +370,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} + echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos @@ -343,7 +380,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "`(/bin/universe) 2>/dev/null`" = att ; then + if test "$( (/bin/universe) 2>/dev/null)" = att ; then echo pyramid-pyramid-sysv3 else echo pyramid-pyramid-bsd @@ -356,69 +393,69 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo sparc-icl-nx6 exit ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case `/usr/bin/uname -p` in + case $(/usr/bin/uname -p) in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux${UNAME_RELEASE} + echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval $set_cc_for_build - SUN_ARCH="i386" + set_cc_for_build + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi - echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:*:*) - case "`/usr/bin/arch -k`" in + case "$(/usr/bin/arch -k)" in Series*|S4*) - UNAME_RELEASE=`uname -v` + UNAME_RELEASE=$(uname -v) ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')" exit ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) - UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 - case "`/bin/arch`" in + UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null) + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 + case "$(/bin/arch)" in sun3) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) - echo sparc-sun-sunos${UNAME_RELEASE} + echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} + echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not @@ -429,44 +466,44 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} + echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} + echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} + echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} + echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} + echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} + echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} + echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} + echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include <stdio.h> /* for printf() prototype */ int main (int argc, char *argv[]) { @@ -475,23 +512,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && - dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`$dummy $dummyarg` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') && + SYSTEM_NAME=$("$dummy" "$dummyarg") && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos${UNAME_RELEASE} + echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax @@ -516,18 +553,18 @@ EOF exit ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + UNAME_PROCESSOR=$(/usr/bin/uname -p) + if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] + if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ + test "$TARGET_BINARY_INTERFACE"x = x then - echo m88k-dg-dgux${UNAME_RELEASE} + echo m88k-dg-dgux"$UNAME_RELEASE" else - echo m88k-dg-dguxbcs${UNAME_RELEASE} + echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else - echo i586-dg-dgux${UNAME_RELEASE} + echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) @@ -544,26 +581,26 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix exit ;; ia64:AIX:*:*) - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if test -x /usr/bin/oslevel ; then + IBM_REV=$(/usr/bin/oslevel) else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <sys/systemcfg.h> main() @@ -574,7 +611,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") then echo "$SYSTEM_NAME" else @@ -587,28 +624,28 @@ EOF fi exit ;; *:AIX:*:[4567]) - IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }') + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/lslpp ] ; then - IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` + if test -x /usr/bin/lslpp ; then + IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/) else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; - ibmrt:4.4BSD:*|romp-ibm:BSD:*) + ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx @@ -623,28 +660,28 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in - 9000/31? ) HP_ARCH=m68000 ;; - 9000/[34]?? ) HP_ARCH=m68k ;; + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') + case "$UNAME_MACHINE" in + 9000/31?) HP_ARCH=m68000 ;; + 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) - if [ -x /usr/bin/getconf ]; then - sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + if test -x /usr/bin/getconf; then + sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null) + sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null) + case "$sc_cpu_version" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + case "$sc_kernel_bits" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + if test "$HP_ARCH" = ""; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include <stdlib.h> @@ -677,13 +714,13 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy") test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if test "$HP_ARCH" = hppa2.0w then - eval $set_cc_for_build + set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -694,23 +731,23 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') + echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <unistd.h> int main () @@ -735,11 +772,11 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) @@ -748,17 +785,17 @@ EOF *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf exit ;; i*86:OSF1:*:*) - if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk + if test -x /usr/sbin/sysversion ; then + echo "$UNAME_MACHINE"-unknown-osf1mk else - echo ${UNAME_MACHINE}-unknown-osf1 + echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) @@ -783,130 +820,123 @@ EOF echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz) + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/') echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/') echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} + echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" + exit ;; + arm:FreeBSD:*:*) + UNAME_PROCESSOR=$(uname -p) + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf + fi exit ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` - case ${UNAME_PROCESSOR} in + UNAME_PROCESSOR=$(/usr/bin/uname -p) + case "$UNAME_PROCESSOR" in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin + echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) - echo ${UNAME_MACHINE}-pc-mingw64 + echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 + echo "$UNAME_MACHINE"-pc-mingw32 exit ;; *:MSYS*:*) - echo ${UNAME_MACHINE}-pc-msys - exit ;; - i*:windows32*:*) - # uname -m includes "-pc" on this system. - echo ${UNAME_MACHINE}-mingw32 + echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 + echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) - case ${UNAME_MACHINE} in + case "$UNAME_MACHINE" in x86) - echo i586-pc-interix${UNAME_RELEASE} + echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix${UNAME_RELEASE} + echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) - echo ia64-unknown-interix${UNAME_RELEASE} + echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; - [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) - echo i${UNAME_MACHINE}-pc-mks - exit ;; - 8664:Windows_NT:*) - echo x86_64-pc-mks - exit ;; - i*:Windows_NT*:* | Pentium*:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we - # UNAME_MACHINE based on the output of uname instead of i386? - echo i586-pc-interix - exit ;; i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin + echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-unknown-cygwin - exit ;; - p*:CYGWIN*:*) - echo powerpcle-unknown-cygwin + echo x86_64-pc-cygwin exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC" exit ;; - i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -916,132 +946,182 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arc:Linux:*:* | arceb:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval $set_cc_for_build + set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; e2k:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) - echo ${UNAME_MACHINE}-pc-linux-${LIBC} + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + k1om:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + IS_GLIBC=0 + test x"${LIBC}" = xgnu && IS_GLIBC=1 + sed 's/^ //' << EOF > "$dummy.c" #undef CPU - #undef ${UNAME_MACHINE} - #undef ${UNAME_MACHINE}el + #undef mips + #undef mipsel + #undef mips64 + #undef mips64el + #if ${IS_GLIBC} && defined(_ABI64) + LIBCABI=gnuabi64 + #else + #if ${IS_GLIBC} && defined(_ABIN32) + LIBCABI=gnuabin32 + #else + LIBCABI=${LIBC} + #endif + #endif + + #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa64r6 + #else + #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa32r6 + #else + #if defined(__mips64) + CPU=mips64 + #else + CPU=mips + #endif + #endif + #endif + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=${UNAME_MACHINE}el + MIPS_ENDIAN=el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=${UNAME_MACHINE} + MIPS_ENDIAN= #else - CPU= + MIPS_ENDIAN= #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')" + test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; + mips64el:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; openrisc*:Linux:*:*) - echo or1k-unknown-linux-${LIBC} + echo or1k-unknown-linux-"$LIBC" exit ;; or32:Linux:*:* | or1k*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-${LIBC} + echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-${LIBC} + echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level - case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; - PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; - *) echo hppa-unknown-linux-${LIBC} ;; + case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-${LIBC} + echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-${LIBC} + echo powerpc-unknown-linux-"$LIBC" exit ;; ppc64le:Linux:*:*) - echo powerpc64le-unknown-linux-${LIBC} + echo powerpc64le-unknown-linux-"$LIBC" exit ;; ppcle:Linux:*:*) - echo powerpcle-unknown-linux-${LIBC} + echo powerpcle-unknown-linux-"$LIBC" + exit ;; + riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-${LIBC} + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + set_cc_for_build + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_X32 >/dev/null + then + LIBCABI="$LIBC"x32 + fi + fi + echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1055,51 +1135,51 @@ EOF # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx + echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop + echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos + echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable + echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} + echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp + echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; - i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + i*86:*:4.*:*) + UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//') if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. - case `/bin/uname -X | grep "^Machine"` in + case $(/bin/uname -X | grep "^Machine") in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then - UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name` - echo ${UNAME_MACHINE}-pc-isc$UNAME_REL + UNAME_REL=$(sed -n 's/.*Version //p' </usr/options/cb.name) + echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL" elif /bin/uname -X 2>/dev/null >/dev/null ; then - UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //')) (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ && UNAME_MACHINE=i586 @@ -1107,9 +1187,9 @@ EOF && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv32 + echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) @@ -1117,7 +1197,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1129,9 +1209,9 @@ EOF exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) @@ -1149,41 +1229,41 @@ EOF 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} + echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} + echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} + echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} + echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} + echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 @@ -1193,8 +1273,8 @@ EOF exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 + UNAME_MACHINE=$( (uname -p) 2>/dev/null) + echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi @@ -1214,23 +1294,23 @@ EOF exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo ${UNAME_MACHINE}-stratus-vos + echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} + echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + if test -d /usr/nec; then + echo mips-nec-sysv"$UNAME_RELEASE" else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. @@ -1249,77 +1329,97 @@ EOF echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} + echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} + echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} + echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux${UNAME_RELEASE} + echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux${UNAME_RELEASE} + echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux${UNAME_RELEASE} + echo sx8r-nec-superux"$UNAME_RELEASE" + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} + echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" + exit ;; + arm64:Darwin:*:*) + echo aarch64-apple-darwin"$UNAME_RELEASE" exit ;; *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - eval $set_cc_for_build - if test "$UNAME_PROCESSOR" = unknown ; then - UNAME_PROCESSOR=powerpc + UNAME_PROCESSOR=$(uname -p) + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + if command -v xcode-select > /dev/null 2> /dev/null && \ + ! xcode-select --print-path > /dev/null 2> /dev/null ; then + # Avoid executing cc if there is no toolchain installed as + # cc will be a stub that puts up a graphical alert + # prompting the user to install developer tools. + CC_FOR_BUILD=no_compiler_found + else + set_cc_for_build fi - if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc fi elif test "$UNAME_PROCESSOR" = i386 ; then - # Avoid executing cc on OS X 10.9, as it ships with a stub - # that puts up a graphical alert prompting to install - # developer tools. Any system running Mac OS X 10.7 or - # later (Darwin 11 and later) is required to have a 64-bit - # processor. This is not true of the ARM version of Darwin - # that Apple uses in portable devices. - UNAME_PROCESSOR=x86_64 + # uname -m returns i386 or x86_64 + UNAME_PROCESSOR=$UNAME_MACHINE fi - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + UNAME_PROCESSOR=$(uname -p) + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk${UNAME_RELEASE} + NEO-*:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk${UNAME_RELEASE} + echo nse-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSR-*:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk"$UNAME_RELEASE" exit ;; - NSR-?:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux @@ -1328,18 +1428,19 @@ EOF echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + # shellcheck disable=SC2154 + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi - echo ${UNAME_MACHINE}-unknown-plan9 + echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 @@ -1360,14 +1461,14 @@ EOF echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; *:*VMS:*:*) - UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "${UNAME_MACHINE}" in + UNAME_MACHINE=$( (uname -p) 2>/dev/null) + case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; @@ -1376,62 +1477,223 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')" exit ;; i*86:rdos:*:*) - echo ${UNAME_MACHINE}-pc-rdos + echo "$UNAME_MACHINE"-pc-rdos exit ;; - i*86:AROS:*:*) - echo ${UNAME_MACHINE}-pc-aros + *:AROS:*:*) + echo "$UNAME_MACHINE"-unknown-aros exit ;; x86_64:VMkernel:*:*) - echo ${UNAME_MACHINE}-unknown-esx + echo "$UNAME_MACHINE"-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs exit ;; + *:Unleashed:*:*) + echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" + exit ;; +esac + +# No uname command or uname output not recognized. +set_cc_for_build +cat > "$dummy.c" <<EOF +#ifdef _SEQUENT_ +#include <sys/types.h> +#include <sys/utsname.h> +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#include <signal.h> +#if defined(_SIZE_T_) || defined(SIGLOST) +#include <sys/utsname.h> +#endif +#endif +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include <sys/param.h> + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null); + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); +#endif + +#if defined (vax) +#if !defined (ultrix) +#include <sys/param.h> +#if defined (BSD) +#if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +#else +#if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#endif +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#else +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname un; + uname (&un); + printf ("vax-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("vax-dec-ultrix\n"); exit (0); +#endif +#endif +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname *un; + uname (&un); + printf ("mips-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("mips-dec-ultrix\n"); exit (0); +#endif +#endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. +test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } + +echo "$0: unable to guess system type" >&2 + +case "$UNAME_MACHINE:$UNAME_SYSTEM" in + mips:Linux | mips64:Linux) + # If we got here on MIPS GNU/Linux, output extra information. + cat >&2 <<EOF + +NOTE: MIPS GNU/Linux systems require a C compiler to fully recognize +the system type. Please install a C compiler and try again. +EOF + ;; esac cat >&2 <<EOF -$0: unable to guess system type -This script, last modified $timestamp, has failed to recognize -the operating system you are using. It is advised that you -download the most up to date version of the config scripts from +This script (version $timestamp), has failed to recognize the +operating system you are using. If your script is old, overwrite *all* +copies of config.guess and config.sub with the latest versions from: - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD + https://git.savannah.gnu.org/cgit/config.git/plain/config.guess and - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + https://git.savannah.gnu.org/cgit/config.git/plain/config.sub +EOF -If the version you run ($0) is already up to date, please -send the following data and any information you think might be -pertinent to <config-patches@gnu.org> in order to provide the needed -information to handle your system. +year=$(echo $timestamp | sed 's,-.*,,') +# shellcheck disable=SC2003 +if test "$(expr "$(date +%Y)" - "$year")" -lt 3 ; then + cat >&2 <<EOF + +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp -uname -m = `(uname -m) 2>/dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` +uname -m = $( (uname -m) 2>/dev/null || echo unknown) +uname -r = $( (uname -r) 2>/dev/null || echo unknown) +uname -s = $( (uname -s) 2>/dev/null || echo unknown) +uname -v = $( (uname -v) 2>/dev/null || echo unknown) -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null` +/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null) +/bin/uname -X = $( (/bin/uname -X) 2>/dev/null) -hostinfo = `(hostinfo) 2>/dev/null` -/bin/universe = `(/bin/universe) 2>/dev/null` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` -/bin/arch = `(/bin/arch) 2>/dev/null` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` +hostinfo = $( (hostinfo) 2>/dev/null) +/bin/universe = $( (/bin/universe) 2>/dev/null) +/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null) +/bin/arch = $( (/bin/arch) 2>/dev/null) +/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null) +/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null) -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" EOF +fi exit 1 # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/erts/autoconf/config.sub b/erts/autoconf/config.sub index 8f1229c6f7..63c1f1c8b5 100755 --- a/erts/autoconf/config.sub +++ b/erts/autoconf/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2015 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2015-03-08' +timestamp='2021-01-08' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ timestamp='2015-03-08' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a @@ -33,7 +33,7 @@ timestamp='2015-03-08' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -50,15 +50,14 @@ timestamp='2015-03-08' # CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM # It is wrong to echo any other type of specification. -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -68,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2015 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -90,12 +89,12 @@ while test $# -gt 0 ; do - ) # Use stdin as input. break ;; -* ) - echo "$me: invalid option $1$help" + echo "$me: invalid option $1$help" >&2 exit 1 ;; *local*) # First pass through any local machine types. - echo $1 + echo "$1" exit ;; * ) @@ -111,1231 +110,1169 @@ case $# in exit 1;; esac -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ - kopensolaris*-gnu* | \ - storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - android-linux) - os=-linux-android - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown - ;; - *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` - else os=; fi - ;; -esac +# Split fields of configuration type +# shellcheck disable=SC2162 +IFS="-" read field1 field2 field3 field4 <<EOF +$1 +EOF -### Let's recognize common machines as not being operating systems so -### that things like config.sub decstation-3100 work. We also -### recognize some manufacturers as not being operating systems, so we -### can provide default operating systems below. -case $os in - -sun*os*) - # Prevent following clause from handling this invalid input. - ;; - -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ - -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ - -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ - -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ - -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ - -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis | -knuth | -cray | -microblaze*) - os= - basic_machine=$1 - ;; - -bluegene*) - os=-cnk - ;; - -sim | -cisco | -oki | -wec | -winbond) - os= - basic_machine=$1 - ;; - -scout) - ;; - -wrs) - os=-vxworks - basic_machine=$1 - ;; - -chorusos*) - os=-chorusos - basic_machine=$1 - ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 - ;; - -hiux*) - os=-hiuxwe2 - ;; - -sco6) - os=-sco5v6 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco5v6*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -isc) - os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -clix*) - basic_machine=clipper-intergraph - ;; - -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -lynx*178) - os=-lynxos178 - ;; - -lynx*5) - os=-lynxos5 - ;; - -lynx*) - os=-lynxos +# Separate into logical components for further validation +case $1 in + *-*-*-*-*) + echo Invalid configuration \`"$1"\': more than four components >&2 + exit 1 ;; - -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + *-*-*-*) + basic_machine=$field1-$field2 + basic_os=$field3-$field4 ;; - -windowsnt*) - os=`echo $os | sed -e 's/windowsnt/winnt/'` + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + basic_os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + basic_os=linux-android + ;; + *) + basic_machine=$field1-$field2 + basic_os=$field3 + ;; + esac ;; - -psos*) - os=-psos + *-*) + # A lone config we happen to match not fitting any pattern + case $field1-$field2 in + decstation-3100) + basic_machine=mips-dec + basic_os= + ;; + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + basic_os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* \ + | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ + | ultra | tti* | harris | dolphin | highlevel | gould \ + | cbm | ns | masscomp | apple | axis | knuth | cray \ + | microblaze* | sim | cisco \ + | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + basic_os= + ;; + *) + basic_machine=$field1 + basic_os=$field2 + ;; + esac + ;; + esac ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + basic_os=bsd + ;; + a29khif) + basic_machine=a29k-amd + basic_os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + basic_os=scout + ;; + alliant) + basic_machine=fx80-alliant + basic_os= + ;; + altos | altos3068) + basic_machine=m68k-altos + basic_os= + ;; + am29k) + basic_machine=a29k-none + basic_os=bsd + ;; + amdahl) + basic_machine=580-amdahl + basic_os=sysv + ;; + amiga) + basic_machine=m68k-unknown + basic_os= + ;; + amigaos | amigados) + basic_machine=m68k-unknown + basic_os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + basic_os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + basic_os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + basic_os=bsd + ;; + aros) + basic_machine=i386-pc + basic_os=aros + ;; + aux) + basic_machine=m68k-apple + basic_os=aux + ;; + balance) + basic_machine=ns32k-sequent + basic_os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + basic_os=linux + ;; + cegcc) + basic_machine=arm-unknown + basic_os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + basic_os=bsd + ;; + convex-c2) + basic_machine=c2-convex + basic_os=bsd + ;; + convex-c32) + basic_machine=c32-convex + basic_os=bsd + ;; + convex-c34) + basic_machine=c34-convex + basic_os=bsd + ;; + convex-c38) + basic_machine=c38-convex + basic_os=bsd + ;; + cray) + basic_machine=j90-cray + basic_os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + basic_os= + ;; + da30) + basic_machine=m68k-da30 + basic_os= + ;; + decstation | pmax | pmin | dec3100 | decstatn) + basic_machine=mips-dec + basic_os= + ;; + delta88) + basic_machine=m88k-motorola + basic_os=sysv3 + ;; + dicos) + basic_machine=i686-pc + basic_os=dicos + ;; + djgpp) + basic_machine=i586-pc + basic_os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + basic_os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + basic_os=ose + ;; + gmicro) + basic_machine=tron-gmicro + basic_os=sysv + ;; + go32) + basic_machine=i386-pc + basic_os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + basic_os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + basic_os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + basic_os=hms + ;; + harris) + basic_machine=m88k-harris + basic_os=sysv3 + ;; + hp300 | hp300hpux) + basic_machine=m68k-hp + basic_os=hpux + ;; + hp300bsd) + basic_machine=m68k-hp + basic_os=bsd + ;; + hppaosf) + basic_machine=hppa1.1-hp + basic_os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + basic_os=proelf + ;; + i386mach) + basic_machine=i386-mach + basic_os=mach + ;; + isi68 | isi) + basic_machine=m68k-isi + basic_os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + basic_os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + basic_os=sysv + ;; + merlin) + basic_machine=ns32k-utek + basic_os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + basic_os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + basic_os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + basic_os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + basic_os=coff + ;; + morphos) + basic_machine=powerpc-unknown + basic_os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + basic_os=moxiebox + ;; + msdos) + basic_machine=i386-pc + basic_os=msdos + ;; + msys) + basic_machine=i686-pc + basic_os=msys + ;; + mvs) + basic_machine=i370-ibm + basic_os=mvs + ;; + nacl) + basic_machine=le32-unknown + basic_os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + basic_os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + basic_os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + basic_os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + basic_os=newsos + ;; + news1000) + basic_machine=m68030-sony + basic_os=newsos + ;; + necv70) + basic_machine=v70-nec + basic_os=sysv + ;; + nh3000) + basic_machine=m68k-harris + basic_os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + basic_os=cxux + ;; + nindy960) + basic_machine=i960-intel + basic_os=nindy + ;; + mon960) + basic_machine=i960-intel + basic_os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + basic_os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + basic_os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + basic_os=ose + ;; + os68k) + basic_machine=m68k-none + basic_os=os68k + ;; + paragon) + basic_machine=i860-intel + basic_os=osf + ;; + parisc) + basic_machine=hppa-unknown + basic_os=linux + ;; + psp) + basic_machine=mipsallegrexel-sony + basic_os=psp + ;; + pw32) + basic_machine=i586-unknown + basic_os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + basic_os=rdos + ;; + rdos32) + basic_machine=i386-pc + basic_os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + basic_os=coff + ;; + sa29200) + basic_machine=a29k-amd + basic_os=udi + ;; + sei) + basic_machine=mips-sei + basic_os=seiux + ;; + sequent) + basic_machine=i386-sequent + basic_os= + ;; + sps7) + basic_machine=m68k-bull + basic_os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + basic_os= + ;; + stratus) + basic_machine=i860-stratus + basic_os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + basic_os= + ;; + sun2os3) + basic_machine=m68000-sun + basic_os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + basic_os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + basic_os= + ;; + sun3os3) + basic_machine=m68k-sun + basic_os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + basic_os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + basic_os= + ;; + sun4os3) + basic_machine=sparc-sun + basic_os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + basic_os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + basic_os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + basic_os= + ;; + sv1) + basic_machine=sv1-cray + basic_os=unicos + ;; + symmetry) + basic_machine=i386-sequent + basic_os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + basic_os=unicos + ;; + t90) + basic_machine=t90-cray + basic_os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + basic_os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + basic_os=tpf + ;; + udi29k) + basic_machine=a29k-amd + basic_os=udi + ;; + ultra3) + basic_machine=a29k-nyu + basic_os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + basic_os=none + ;; + vaxv) + basic_machine=vax-dec + basic_os=sysv + ;; + vms) + basic_machine=vax-dec + basic_os=vms + ;; + vsta) + basic_machine=i386-pc + basic_os=vsta + ;; + vxworks960) + basic_machine=i960-wrs + basic_os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + basic_os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + basic_os=vxworks + ;; + xbox) + basic_machine=i686-pc + basic_os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + basic_os=unicos + ;; + *) + basic_machine=$1 + basic_os= + ;; + esac ;; esac -# Decode aliases for certain CPU-COMPANY combinations. +# Decode 1-component or ad-hoc basic machines case $basic_machine in - # Recognize the basic CPU types without company name. - # Some are omitted here because they have special meanings below. - 1750a | 580 \ - | a29k \ - | aarch64 | aarch64_be \ - | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ - | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ - | am33_2.0 \ - | arc | arceb \ - | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ - | avr | avr32 \ - | be32 | be64 \ - | bfin \ - | c4x | c8051 | clipper \ - | d10v | d30v | dlx | dsp16xx \ - | e2k | epiphany \ - | fido | fr30 | frv | ft32 \ - | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ - | hexagon \ - | i370 | i860 | i960 | ia64 \ - | ip2k | iq2000 \ - | k1om \ - | le32 | le64 \ - | lm32 \ - | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ - | mn10200 | mn10300 \ - | moxie \ - | mt \ - | msp430 \ - | nds32 | nds32le | nds32be \ - | nios | nios2 | nios2eb | nios2el \ - | ns16k | ns32k \ - | open8 | or1k | or1knd | or32 \ - | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle \ - | pyramid \ - | riscv32 | riscv64 \ - | rl78 | rx \ - | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ - | sh64 | sh64le \ - | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ - | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ - | spu \ - | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ - | ubicom32 \ - | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ - | visium \ - | we32k \ - | x86 | xc16x | xstormy16 | xtensa \ - | z8k | z80) - basic_machine=$basic_machine-unknown - ;; - c54x) - basic_machine=tic54x-unknown - ;; - c55x) - basic_machine=tic55x-unknown - ;; - c6x) - basic_machine=tic6x-unknown - ;; - leon|leon[3-9]) - basic_machine=sparc-$basic_machine - ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) - basic_machine=$basic_machine-unknown - os=-none + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + w89k) + cpu=hppa1.1 + vendor=winbond ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + op50n) + cpu=hppa1.1 + vendor=oki ;; - ms1) - basic_machine=mt-unknown + op60c) + cpu=hppa1.1 + vendor=oki ;; - - strongarm | thumb | xscale) - basic_machine=arm-unknown + ibm*) + cpu=i370 + vendor=ibm ;; - xgate) - basic_machine=$basic_machine-unknown - os=-none + orion105) + cpu=clipper + vendor=highlevel ;; - xscaleeb) - basic_machine=armeb-unknown + mac | mpw | mac-mpw) + cpu=m68k + vendor=apple ;; - - xscaleel) - basic_machine=armel-unknown + pmac | pmac-mpw) + cpu=powerpc + vendor=apple ;; - # We use `pc' rather than `unknown' - # because (1) that's what they normally are, and - # (2) the word "unknown" tends to confuse beginning users. - i*86 | x86_64) - basic_machine=$basic_machine-pc - ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 - ;; - # Recognize the basic CPU types with company name. - 580-* \ - | a29k-* \ - | aarch64-* | aarch64_be-* \ - | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ - | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* | avr32-* \ - | be32-* | be64-* \ - | bfin-* | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | c8051-* | clipper-* | craynv-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | e2k-* | elxsi-* \ - | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ - | h8300-* | h8500-* \ - | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ - | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ - | ip2k-* | iq2000-* \ - | k1om-* \ - | le32-* | le64-* \ - | lm32-* \ - | m32c-* | m32r-* | m32rle-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ - | microblaze-* | microblazeel-* \ - | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ - | mips16-* \ - | mips64-* | mips64el-* \ - | mips64octeon-* | mips64octeonel-* \ - | mips64orion-* | mips64orionel-* \ - | mips64r5900-* | mips64r5900el-* \ - | mips64vr-* | mips64vrel-* \ - | mips64vr4100-* | mips64vr4100el-* \ - | mips64vr4300-* | mips64vr4300el-* \ - | mips64vr5000-* | mips64vr5000el-* \ - | mips64vr5900-* | mips64vr5900el-* \ - | mipsisa32-* | mipsisa32el-* \ - | mipsisa32r2-* | mipsisa32r2el-* \ - | mipsisa32r6-* | mipsisa32r6el-* \ - | mipsisa64-* | mipsisa64el-* \ - | mipsisa64r2-* | mipsisa64r2el-* \ - | mipsisa64r6-* | mipsisa64r6el-* \ - | mipsisa64sb1-* | mipsisa64sb1el-* \ - | mipsisa64sr71k-* | mipsisa64sr71kel-* \ - | mipsr5900-* | mipsr5900el-* \ - | mipstx39-* | mipstx39el-* \ - | mmix-* \ - | mt-* \ - | msp430-* \ - | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* | nios2eb-* | nios2el-* \ - | none-* | np1-* | ns16k-* | ns32k-* \ - | open8-* \ - | or1k*-* \ - | orion-* \ - | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ - | pyramid-* \ - | rl78-* | romp-* | rs6000-* | rx-* \ - | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ - | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ - | tahoe-* \ - | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tile*-* \ - | tron-* \ - | ubicom32-* \ - | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ - | vax-* \ - | visium-* \ - | we32k-* \ - | x86-* | x86_64-* | xc16x-* | xps100-* \ - | xstormy16-* | xtensa*-* \ - | ymp-* \ - | z8k-* | z80-*) - ;; - # Recognize the basic CPU types without company name, with glob match. - xtensa*) - basic_machine=$basic_machine-unknown - ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-unknown - os=-bsd - ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - basic_machine=m68000-att + cpu=m68000 + vendor=att ;; 3b*) - basic_machine=we32k-att - ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; - amd64) - basic_machine=x86_64-pc - ;; - amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv - ;; - amiga | amiga-*) - basic_machine=m68k-unknown - ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aros) - basic_machine=i386-pc - os=-aros - ;; - asmjs) - basic_machine=asmjs-unknown - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - blackfin) - basic_machine=bfin-unknown - os=-linux - ;; - blackfin-*) - basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + cpu=we32k + vendor=att ;; bluegene*) - basic_machine=powerpc-ibm - os=-cnk - ;; - c54x-*) - basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c55x-*) - basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c6x-*) - basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c90) - basic_machine=c90-cray - os=-unicos - ;; - cegcc) - basic_machine=arm-unknown - os=-cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - craynv) - basic_machine=craynv-cray - os=-unicosmp - ;; - cr16 | cr16-*) - basic_machine=cr16-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds - ;; - crisv32 | crisv32-* | etraxfs*) - basic_machine=crisv32-axis - ;; - cris | cris-* | etrax*) - basic_machine=cris-axis - ;; - crx) - basic_machine=crx-unknown - os=-elf - ;; - da30 | da30-*) - basic_machine=m68k-da30 - ;; - decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) - basic_machine=mips-dec + cpu=powerpc + vendor=ibm + basic_os=cnk ;; decsystem10* | dec10*) - basic_machine=pdp10-dec - os=-tops10 + cpu=pdp10 + vendor=dec + basic_os=tops10 ;; decsystem20* | dec20*) - basic_machine=pdp10-dec - os=-tops20 + cpu=pdp10 + vendor=dec + basic_os=tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) - basic_machine=m68k-motorola - ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dicos) - basic_machine=i686-pc - os=-dicos + cpu=m68k + vendor=motorola ;; - djgpp) - basic_machine=i586-pc - os=-msdosdjgpp - ;; - dpx20 | dpx20-*) - basic_machine=rs6000-bull - os=-bosx - ;; - dpx2* | dpx2*-bull) - basic_machine=m68k-bull - os=-sysv3 - ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon - ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd + dpx2*) + cpu=m68k + vendor=bull + basic_os=sysv3 ;; encore | umax | mmax) - basic_machine=ns32k-encore + cpu=ns32k + vendor=encore ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose + elxsi) + cpu=elxsi + vendor=elxsi + basic_os=${basic_os:-bsd} ;; fx2800) - basic_machine=i860-alliant + cpu=i860 + vendor=alliant ;; genix) - basic_machine=ns32k-ns - ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 + cpu=ns32k + vendor=ns ;; h3050r* | hiux*) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 - ;; - hp300-*) - basic_machine=m68k-hp - ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux + cpu=hppa1.1 + vendor=hitachi + basic_os=hiuxwe2 ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k2[0-9][0-9] | hp9k31[0-9]) - basic_machine=m68000-hp + cpu=m68000 + vendor=hp ;; hp9k3[2-9][0-9]) - basic_machine=m68k-hp + cpu=m68k + vendor=hp ;; hp9k6[0-9][0-9] | hp6[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k7[0-79][0-9] | hp7[0-79][0-9]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k78[0-9] | hp78[0-9]) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][13679] | hp8[0-9][13679]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][0-9] | hp8[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hppa-next) - os=-nextstep3 - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm + cpu=hppa1.0 + vendor=hp ;; i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv32 + cpu=$(echo "$1" | sed -e 's/86.*/86/') + vendor=pc + basic_os=sysv32 ;; i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv4 + cpu=$(echo "$1" | sed -e 's/86.*/86/') + vendor=pc + basic_os=sysv4 ;; i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv + cpu=$(echo "$1" | sed -e 's/86.*/86/') + vendor=pc + basic_os=sysv ;; i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-solaris2 - ;; - i386mach) - basic_machine=i386-mach - os=-mach + cpu=$(echo "$1" | sed -e 's/86.*/86/') + vendor=pc + basic_os=solaris2 ;; - i386-vsta | vsta) - basic_machine=i386-unknown - os=-vsta + j90 | j90-cray) + cpu=j90 + vendor=cray + basic_os=${basic_os:-unicos} ;; iris | iris4d) - basic_machine=mips-sgi - case $os in - -irix*) + cpu=mips + vendor=sgi + case $basic_os in + irix*) ;; *) - os=-irix4 + basic_os=irix4 ;; esac ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - leon-*|leon[3-9]-*) - basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` - ;; - m68knommu) - basic_machine=m68k-unknown - os=-linux - ;; - m68knommu-*) - basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux - ;; - m88k-omron*) - basic_machine=m88k-omron - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv - ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - microblaze*) - basic_machine=microblaze-xilinx - ;; - mingw64) - basic_machine=x86_64-pc - os=-mingw64 - ;; - mingw32) - basic_machine=i686-pc - os=-mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - os=-mingw32ce - ;; miniframe) - basic_machine=m68000-convergent - ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) - basic_machine=m68k-atari - os=-mint + cpu=m68000 + vendor=convergent ;; - mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` - ;; - mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - moxiebox) - basic_machine=moxie-unknown - os=-moxiebox - ;; - msdos) - basic_machine=i386-pc - os=-msdos - ;; - ms1-*) - basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` - ;; - msys) - basic_machine=i686-pc - os=-msys - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - nacl) - basic_machine=le32-unknown - os=-nacl - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 - ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) + cpu=m68k + vendor=atari + basic_os=mint ;; news-3600 | risc-news) - basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv - ;; - next | m*-next ) - basic_machine=m68k-next - case $os in - -nextstep* ) + cpu=mips + vendor=sony + basic_os=newsos + ;; + next | m*-next) + cpu=m68k + vendor=next + case $basic_os in + openstep*) + ;; + nextstep*) ;; - -ns2*) - os=-nextstep2 + ns2*) + basic_os=nextstep2 ;; *) - os=-nextstep3 + basic_os=nextstep3 ;; esac ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; np1) - basic_machine=np1-gould - ;; - neo-tandem) - basic_machine=neo-tandem - ;; - nse-tandem) - basic_machine=nse-tandem - ;; - nsr-tandem) - basic_machine=nsr-tandem + cpu=np1 + vendor=gould ;; op50n-* | op60c-*) - basic_machine=hppa1.1-oki - os=-proelf - ;; - openrisc | openrisc-*) - basic_machine=or32-unknown - ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k + cpu=hppa1.1 + vendor=oki + basic_os=proelf ;; pa-hitachi) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - parisc) - basic_machine=hppa-unknown - os=-linux - ;; - parisc-*) - basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + cpu=hppa1.1 + vendor=hitachi + basic_os=hiuxwe2 ;; pbd) - basic_machine=sparc-tti + cpu=sparc + vendor=tti ;; pbb) - basic_machine=m68k-tti - ;; - pc532 | pc532-*) - basic_machine=ns32k-pc532 - ;; - pc98) - basic_machine=i386-pc - ;; - pc98-*) - basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + cpu=m68k + vendor=tti ;; - pentium | p5 | k5 | k6 | nexgen | viac3) - basic_machine=i586-pc - ;; - pentiumpro | p6 | 6x86 | athlon | athlon_*) - basic_machine=i686-pc - ;; - pentiumii | pentium2 | pentiumiii | pentium3) - basic_machine=i686-pc - ;; - pentium4) - basic_machine=i786-pc - ;; - pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + pc532) + cpu=ns32k + vendor=pc532 ;; pn) - basic_machine=pn-gould - ;; - power) basic_machine=power-ibm - ;; - ppc | ppcbe) basic_machine=powerpc-unknown - ;; - ppc-* | ppcbe-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + cpu=pn + vendor=gould ;; - ppcle | powerpclittle | ppc-le | powerpc-little) - basic_machine=powerpcle-unknown - ;; - ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64) basic_machine=powerpc64-unknown - ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) - basic_machine=powerpc64le-unknown - ;; - ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + power) + cpu=power + vendor=ibm ;; ps2) - basic_machine=i386-ibm - ;; - pw32) - basic_machine=i586-unknown - os=-pw32 - ;; - rdos | rdos64) - basic_machine=x86_64-pc - os=-rdos - ;; - rdos32) - basic_machine=i386-pc - os=-rdos - ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff + cpu=i386 + vendor=ibm ;; rm[46]00) - basic_machine=mips-siemens + cpu=mips + vendor=siemens ;; rtpc | rtpc-*) - basic_machine=romp-ibm - ;; - s390 | s390-*) - basic_machine=s390-ibm - ;; - s390x | s390x-*) - basic_machine=s390x-ibm - ;; - sa29200) - basic_machine=a29k-amd - os=-udi + cpu=romp + vendor=ibm ;; - sb1) - basic_machine=mipsisa64sb1-unknown + sde) + cpu=mipsisa32 + vendor=sde + basic_os=${basic_os:-elf} ;; - sb1el) - basic_machine=mipsisa64sb1el-unknown + simso-wrs) + cpu=sparclite + vendor=wrs + basic_os=vxworks ;; - sde) - basic_machine=mipsisa32-sde - os=-elf + tower | tower-32) + cpu=m68k + vendor=ncr ;; - sei) - basic_machine=mips-sei - os=-seiux + vpp*|vx|vx-*) + cpu=f301 + vendor=fujitsu ;; - sequent) - basic_machine=i386-sequent + w65) + cpu=w65 + vendor=wdc ;; - sh) - basic_machine=sh-hitachi - os=-hms + w89k-*) + cpu=hppa1.1 + vendor=winbond + basic_os=proelf ;; - sh5el) - basic_machine=sh5le-unknown + none) + cpu=none + vendor=none ;; - sh64) - basic_machine=sh64-unknown + leon|leon[3-9]) + cpu=sparc + vendor=$basic_machine ;; - sparclite-wrs | simso-wrs) - basic_machine=sparclite-wrs - os=-vxworks + leon-*|leon[3-9]-*) + cpu=sparc + vendor=$(echo "$basic_machine" | sed 's/-.*//') ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 + + *-*) + # shellcheck disable=SC2162 + IFS="-" read cpu vendor <<EOF +$basic_machine +EOF ;; - spur) - basic_machine=spur-unknown + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + cpu=$basic_machine + vendor=pc ;; - st2000) - basic_machine=m68k-tandem + # These rules are duplicated from below for sake of the special case above; + # i.e. things that normalized to x86 arches should also default to "pc" + pc98) + cpu=i386 + vendor=pc ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 + x64 | amd64) + cpu=x86_64 + vendor=pc ;; - strongarm-* | thumb-*) - basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + # Recognize the basic CPU types without company name. + *) + cpu=$basic_machine + vendor=unknown ;; - sun2) - basic_machine=m68000-sun +esac + +unset -v basic_machine + +# Decode basic machines in the full and proper CPU-Company form. +case $cpu-$vendor in + # Here we handle the default manufacturer of certain CPU types in canonical form. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv-unknown) + vendor=cray + basic_os=${basic_os:-unicosmp} ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 + c90-unknown | c90-cray) + vendor=cray + basic_os=${Basic_os:-unicos} ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 + fx80-unknown) + vendor=alliant ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 + romp-unknown) + vendor=ibm ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 + mmix-unknown) + vendor=knuth ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 + microblaze-unknown | microblazeel-unknown) + vendor=xilinx ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 + rs6000-unknown) + vendor=ibm ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 + vax-unknown) + vendor=dec ;; - sun3 | sun3-*) - basic_machine=m68k-sun + pdp11-unknown) + vendor=dec ;; - sun4) - basic_machine=sparc-sun + we32k-unknown) + vendor=att ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun + cydra-unknown) + vendor=cydrome ;; - sv1) - basic_machine=sv1-cray - os=-unicos + i370-ibm*) + vendor=ibm ;; - symmetry) - basic_machine=i386-sequent - os=-dynix + orion-unknown) + vendor=highlevel ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos + xps-unknown | xps100-unknown) + cpu=xps100 + vendor=honeywell ;; - t90) - basic_machine=t90-cray - os=-unicos + + # Here we normalize CPU types with a missing or matching vendor + dpx20-unknown | dpx20-bull) + cpu=rs6000 + vendor=bull + basic_os=${basic_os:-bosx} ;; - tile*) - basic_machine=$basic_machine-unknown - os=-linux-gnu + + # Here we normalize CPU types irrespective of the vendor + amd64-*) + cpu=x86_64 ;; - tx39) - basic_machine=mipstx39-unknown + blackfin-*) + cpu=bfin + basic_os=linux ;; - tx39el) - basic_machine=mipstx39el-unknown + c54x-*) + cpu=tic54x ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 + c55x-*) + cpu=tic55x ;; - tower | tower-32) - basic_machine=m68k-ncr + c6x-*) + cpu=tic6x ;; - tpf) - basic_machine=s390x-ibm - os=-tpf + e500v[12]-*) + cpu=powerpc + basic_os=${basic_os}"spe" ;; - udi29k) - basic_machine=a29k-amd - os=-udi + mips3*-*) + cpu=mips64 ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 + ms1-*) + cpu=mt ;; - v810 | necv810) - basic_machine=v810-nec - os=-none + m68knommu-*) + cpu=m68k + basic_os=linux ;; - vaxv) - basic_machine=vax-dec - os=-sysv + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + cpu=s12z ;; - vms) - basic_machine=vax-dec - os=-vms + openrisc-*) + cpu=or32 ;; - vpp*|vx|vx-*) - basic_machine=f301-fujitsu + parisc-*) + cpu=hppa + basic_os=linux ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + cpu=i586 ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks + pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + cpu=i686 ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + cpu=i686 ;; - w65*) - basic_machine=w65-wdc - os=-none + pentium4-*) + cpu=i786 ;; - w89k-*) - basic_machine=hppa1.1-winbond - os=-proelf + pc98-*) + cpu=i386 ;; - xbox) - basic_machine=i686-pc - os=-mingw32 + ppc-* | ppcbe-*) + cpu=powerpc ;; - xps | xps100) - basic_machine=xps100-honeywell + ppcle-* | powerpclittle-*) + cpu=powerpcle ;; - xscale-* | xscalee[bl]-*) - basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ppc64-*) + cpu=powerpc64 ;; - ymp) - basic_machine=ymp-cray - os=-unicos + ppc64le-* | powerpc64little-*) + cpu=powerpc64le ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim + sb1-*) + cpu=mipsisa64sb1 ;; - z80-*-coff) - basic_machine=z80-unknown - os=-sim + sb1el-*) + cpu=mipsisa64sb1el ;; - none) - basic_machine=none-none - os=-none + sh5e[lb]-*) + cpu=$(echo "$cpu" | sed 's/^\(sh.\)e\(.\)$/\1\2e/') ;; - -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond + spur-*) + cpu=spur ;; - op50n) - basic_machine=hppa1.1-oki + strongarm-* | thumb-*) + cpu=arm ;; - op60c) - basic_machine=hppa1.1-oki + tx39-*) + cpu=mipstx39 ;; - romp) - basic_machine=romp-ibm + tx39el-*) + cpu=mipstx39el ;; - mmix) - basic_machine=mmix-knuth + x64-*) + cpu=x86_64 ;; - rs6000) - basic_machine=rs6000-ibm + xscale-* | xscalee[bl]-*) + cpu=$(echo "$cpu" | sed 's/^xscale/arm/') ;; - vax) - basic_machine=vax-dec + arm64-*) + cpu=aarch64 ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown + + # Recognize the canonical CPU Types that limit and/or modify the + # company names they are paired with. + cr16-*) + basic_os=${basic_os:-elf} ;; - pdp11) - basic_machine=pdp11-dec + crisv32-* | etraxfs*-*) + cpu=crisv32 + vendor=axis ;; - we32k) - basic_machine=we32k-att + cris-* | etrax*-*) + cpu=cris + vendor=axis ;; - sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown + crx-*) + basic_os=${basic_os:-elf} ;; - sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) - basic_machine=sparc-sun + neo-tandem) + cpu=neo + vendor=tandem ;; - cydra) - basic_machine=cydra-cydrome + nse-tandem) + cpu=nse + vendor=tandem ;; - orion) - basic_machine=orion-highlevel + nsr-tandem) + cpu=nsr + vendor=tandem ;; - orion105) - basic_machine=clipper-highlevel + nsv-tandem) + cpu=nsv + vendor=tandem ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple + nsx-tandem) + cpu=nsx + vendor=tandem ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple + mipsallegrexel-sony) + cpu=mipsallegrexel + vendor=sony ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. + tile*-*) + basic_os=${basic_os:-linux-gnu} ;; + *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 + # Recognize the canonical CPU types that are allowed with any + # company name. + case $cpu in + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ + | alphapca5[67] | alpha64pca5[67] \ + | am33_2.0 \ + | amdgcn \ + | arc | arceb \ + | arm | arm[lb]e | arme[lb] | armv* \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin | bpf | bs2000 \ + | c[123]* | c30 | [cjt]90 | c4x \ + | c8051 | clipper | craynv | csky | cydra \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | elxsi | epiphany \ + | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | h8300 | h8500 \ + | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i*86 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | loongarch32 | loongarch64 | loongarchx32 \ + | m32c | m32r | m32rle \ + | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ + | m88110 | m88k | maxq | mb | mcore | mep | metag \ + | microblaze | microblazeel \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64eb | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mmix \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | none | np1 | ns16k | ns32k | nvptx \ + | open8 \ + | or1k* \ + | or32 \ + | orion \ + | picochip \ + | pdp10 | pdp11 | pj | pjl | pn | power \ + | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv32be | riscv64 | riscv64be \ + | rl78 | romp | rs6000 | rx \ + | s390 | s390x \ + | score \ + | sh | shl \ + | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ + | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | spu \ + | tahoe \ + | thumbv7* \ + | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tron \ + | ubicom32 \ + | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | vax \ + | visium \ + | w65 \ + | wasm32 | wasm64 \ + | we32k \ + | x86 | x86_64 | xc16x | xgate | xps100 \ + | xstormy16 | xtensa* \ + | ymp \ + | z8k | z80) + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + exit 1 + ;; + esac ;; esac # Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` +case $vendor in + digital*) + vendor=dec ;; - *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + commodore*) + vendor=cbm ;; *) ;; @@ -1343,200 +1280,213 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x"$os" != x"" ] +if test x$basic_os != x then + +# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# set os. +case $basic_os in + gnu/linux*) + kernel=linux + os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|') + ;; + os2-emx) + kernel=os2 + os=$(echo $basic_os | sed -e 's|os2-emx|emx|') + ;; + nto-qnx*) + kernel=nto + os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|') + ;; + *-*) + # shellcheck disable=SC2162 + IFS="-" read kernel os <<EOF +$basic_os +EOF + ;; + # Default OS when just kernel was specified + nto*) + kernel=nto + os=$(echo $basic_os | sed -e 's|nto|qnx|') + ;; + linux*) + kernel=linux + os=$(echo $basic_os | sed -e 's|linux|gnu|') + ;; + *) + kernel= + os=$basic_os + ;; +esac + +# Now, normalize the OS (knowing we just have one component, it's not a kernel, +# etc.) case $os in - # First match some system type aliases - # that might get confused with valid system types. - # -solaris* is a basic system type, with this one exception. - -auroraux) - os=-auroraux + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux ;; - -solaris1 | -solaris1.*) - os=`echo $os | sed -e 's|solaris1|sunos4|'` + bluegene*) + os=cnk ;; - -solaris) - os=-solaris2 + solaris1 | solaris1.*) + os=$(echo $os | sed -e 's|solaris1|sunos4|') ;; - -svr4*) - os=-sysv4 + solaris) + os=solaris2 ;; - -unixware*) - os=-sysv4.2uw + unixware*) + os=sysv4.2uw ;; - -gnu/linux*) - os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose ;; - # First accept the basic system types. - # The portable systems comes first. - # Each alternative MUST END IN A *, to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* | -plan9* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* | -cloudabi* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ - | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*) - # Remember, each alternative MUST END IN *, to match a version number. - ;; - -qnx*) - case $basic_machine in - x86-* | i*86-*) - ;; - *) - os=-nto$os - ;; - esac + # Some version numbers need modification + chorusos*) + os=chorusos ;; - -nto-qnx*) + isc) + os=isc2.2 ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` + sco6) + os=sco5v6 ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + sco5) + os=sco3.2v5 ;; - -mac*) - os=`echo $os | sed -e 's|mac|macos|'` + sco4) + os=sco3.2v4 ;; - -linux-dietlibc) - os=-linux-dietlibc + sco3.2.[4-9]*) + os=$(echo $os | sed -e 's/sco3.2./sco3.2v/') ;; - -linux*) - os=`echo $os | sed -e 's|linux|linux-gnu|'` + sco*v* | scout) + # Don't match below ;; - -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` + sco*) + os=sco3.2v2 ;; - -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` + psos*) + os=psos ;; - -opened*) - os=-openedition + qnx*) + os=qnx ;; - -os400*) - os=-os400 + hiux*) + os=hiuxwe2 ;; - -wince*) - os=-wince + lynx*178) + os=lynxos178 ;; - -osfrose*) - os=-osfrose + lynx*5) + os=lynxos5 ;; - -osf*) - os=-osf + lynxos*) + # don't get caught up in next wildcard ;; - -utek*) - os=-bsd + lynx*) + os=lynxos ;; - -dynix*) - os=-bsd + mac[0-9]*) + os=$(echo "$os" | sed -e 's|mac|macos|') ;; - -acis*) - os=-aos + opened*) + os=openedition ;; - -atheos*) - os=-atheos + os400*) + os=os400 ;; - -syllable*) - os=-syllable + sunos5*) + os=$(echo "$os" | sed -e 's|sunos5|solaris2|') ;; - -386bsd) - os=-bsd + sunos6*) + os=$(echo "$os" | sed -e 's|sunos6|solaris3|') ;; - -ctix* | -uts*) - os=-sysv + wince*) + os=wince ;; - -nova*) - os=-rtmk-nova + utek*) + os=bsd ;; - -ns2 ) - os=-nextstep2 + dynix*) + os=bsd ;; - -nsk*) - os=-nsk + acis*) + os=aos ;; - # Preserve the version number of sinix5. - -sinix5.*) - os=`echo $os | sed -e 's|sinix|sysv|'` + atheos*) + os=atheos ;; - -sinix*) - os=-sysv4 + syllable*) + os=syllable ;; - -tpf*) - os=-tpf + 386bsd) + os=bsd ;; - -triton*) - os=-sysv3 + ctix* | uts*) + os=sysv ;; - -oss*) - os=-sysv3 + nova*) + os=rtmk-nova ;; - -svr4) - os=-sysv4 + ns2) + os=nextstep2 ;; - -svr3) - os=-sysv3 + # Preserve the version number of sinix5. + sinix5.*) + os=$(echo $os | sed -e 's|sinix|sysv|') ;; - -sysvr4) - os=-sysv4 + sinix*) + os=sysv4 ;; - # This must come after -sysvr4. - -sysv*) + tpf*) + os=tpf ;; - -ose*) - os=-ose + triton*) + os=sysv3 ;; - -es1800*) - os=-ose + oss*) + os=sysv3 ;; - -xenix) - os=-xenix + svr4*) + os=sysv4 ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint + svr3) + os=sysv3 ;; - -aros*) - os=-aros + sysvr4) + os=sysv4 ;; - -zvmoe) - os=-zvmoe + ose*) + os=ose ;; - -dicos*) - os=-dicos + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint ;; - -nacl*) + dicos*) + os=dicos ;; - -none) + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $cpu in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac ;; *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 - exit 1 + # No normalization, but not necessarily accepted, that comes below. ;; esac + else # Here we handle the default operating systems that come with various machines. @@ -1549,261 +1499,361 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. -case $basic_machine in +kernel= +case $cpu-$vendor in score-*) - os=-elf + os=elf ;; spu-*) - os=-elf + os=elf ;; *-acorn) - os=-riscix1.2 + os=riscix1.2 ;; arm*-rebel) - os=-linux + kernel=linux + os=gnu ;; arm*-semi) - os=-aout + os=aout ;; c4x-* | tic4x-*) - os=-coff + os=coff ;; c8051-*) - os=-elf + os=elf + ;; + clipper-intergraph) + os=clix ;; hexagon-*) - os=-elf + os=elf ;; tic54x-*) - os=-coff + os=coff ;; tic55x-*) - os=-coff + os=coff ;; tic6x-*) - os=-coff + os=coff ;; # This must come before the *-dec entry. pdp10-*) - os=-tops20 + os=tops20 ;; pdp11-*) - os=-none + os=none ;; *-dec | vax-*) - os=-ultrix4.2 + os=ultrix4.2 ;; m68*-apollo) - os=-domain + os=domain ;; i386-sun) - os=-sunos4.0.2 + os=sunos4.0.2 ;; m68000-sun) - os=-sunos3 + os=sunos3 ;; m68*-cisco) - os=-aout + os=aout ;; mep-*) - os=-elf + os=elf ;; mips*-cisco) - os=-elf + os=elf ;; mips*-*) - os=-elf + os=elf ;; or32-*) - os=-coff + os=coff ;; *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 + os=sysv3 ;; sparc-* | *-sun) - os=-sunos4.1.1 + os=sunos4.1.1 ;; - *-be) - os=-beos + pru-*) + os=elf ;; - *-haiku) - os=-haiku + *-be) + os=beos ;; *-ibm) - os=-aix + os=aix ;; *-knuth) - os=-mmixware + os=mmixware ;; *-wec) - os=-proelf + os=proelf ;; *-winbond) - os=-proelf + os=proelf ;; *-oki) - os=-proelf + os=proelf ;; *-hp) - os=-hpux + os=hpux ;; *-hitachi) - os=-hiux + os=hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv + os=sysv ;; *-cbm) - os=-amigaos + os=amigaos ;; *-dg) - os=-dgux + os=dgux ;; *-dolphin) - os=-sysv3 + os=sysv3 ;; m68k-ccur) - os=-rtu + os=rtu ;; m88k-omron*) - os=-luna + os=luna ;; - *-next ) - os=-nextstep + *-next) + os=nextstep ;; *-sequent) - os=-ptx + os=ptx ;; *-crds) - os=-unos + os=unos ;; *-ns) - os=-genix + os=genix ;; i370-*) - os=-mvs - ;; - *-next) - os=-nextstep3 + os=mvs ;; *-gould) - os=-sysv + os=sysv ;; *-highlevel) - os=-bsd + os=bsd ;; *-encore) - os=-bsd + os=bsd ;; *-sgi) - os=-irix + os=irix ;; *-siemens) - os=-sysv4 + os=sysv4 ;; *-masscomp) - os=-rtu + os=rtu ;; f30[01]-fujitsu | f700-fujitsu) - os=-uxpv + os=uxpv ;; *-rom68k) - os=-coff + os=coff ;; *-*bug) - os=-coff + os=coff ;; *-apple) - os=-macos + os=macos ;; *-atari*) - os=-mint + os=mint + ;; + *-wrs) + os=vxworks ;; *) - os=-none + os=none ;; esac + fi +# Now, validate our (potentially fixed-up) OS. +case $os in + # Sometimes we do "kernel-libc", so those need to count as OSes. + musl* | newlib* | uclibc*) + ;; + # Likewise for "kernel-abi" + eabi* | gnueabi*) + ;; + # VxWorks passes extra cpu info in the 4th filed. + simlinux | simwindows | spe) + ;; + # Now accept the basic system types. + # The portable systems comes first. + # Each alternative MUST end in a * to match a version number. + gnu* | android* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]* \ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ + | hiux* | abug | nacl* | netware* | windows* \ + | os9* | macos* | osx* | ios* \ + | mpw* | magic* | mmixware* | mon960* | lnews* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* | twizzler* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | mirbsd* | netbsd* | dicos* | openedition* | ose* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \ + | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | mint* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* | morphos* \ + | scout* | superux* | sysv* | rtmk* | tpf* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx*) + ;; + # This one is extra strict with allowed versions + sco3.2v2 | sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + none) + ;; + *) + echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + exit 1 + ;; +esac + +# As a final step for OS-related things, validate the OS-kernel combination +# (given a valid OS), if there is a kernel. +case $kernel-$os in + linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) + ;; + uclinux-uclibc* ) + ;; + -dietlibc* | -newlib* | -musl* | -uclibc* ) + # These are just libc implementations, not actual OSes, and thus + # require a kernel. + echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + exit 1 + ;; + kfreebsd*-gnu* | kopensolaris*-gnu*) + ;; + vxworks-simlinux | vxworks-simwindows | vxworks-spe) + ;; + nto-qnx*) + ;; + os2-emx) + ;; + *-eabi* | *-gnueabi*) + ;; + -*) + # Blank kernel with real OS is always fine. + ;; + *-*) + echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + exit 1 + ;; +esac + # Here we handle the case where we know the os, and the CPU type, but not the # manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) - case $os in - -riscix*) +case $vendor in + unknown) + case $cpu-$os in + *-riscix*) vendor=acorn ;; - -sunos*) + *-sunos*) vendor=sun ;; - -cnk*|-aix*) + *-cnk* | *-aix*) vendor=ibm ;; - -beos*) + *-beos*) vendor=be ;; - -hpux*) + *-hpux*) vendor=hp ;; - -mpeix*) + *-mpeix*) vendor=hp ;; - -hiux*) + *-hiux*) vendor=hitachi ;; - -unos*) + *-unos*) vendor=crds ;; - -dgux*) + *-dgux*) vendor=dg ;; - -luna*) + *-luna*) vendor=omron ;; - -genix*) + *-genix*) vendor=ns ;; - -mvs* | -opened*) + *-clix*) + vendor=intergraph + ;; + *-mvs* | *-opened*) + vendor=ibm + ;; + *-os400*) vendor=ibm ;; - -os400*) + s390-* | s390x-*) vendor=ibm ;; - -ptx*) + *-ptx*) vendor=sequent ;; - -tpf*) + *-tpf*) vendor=ibm ;; - -vxsim* | -vxworks* | -windiss*) + *-vxsim* | *-vxworks* | *-windiss*) vendor=wrs ;; - -aux*) + *-aux*) vendor=apple ;; - -hms*) + *-hms*) vendor=hitachi ;; - -mpw* | -macos*) + *-mpw* | *-macos*) vendor=apple ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) vendor=atari ;; - -vos*) + *-vos*) vendor=stratus ;; esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` ;; esac -echo $basic_machine$os +echo "$cpu-$vendor-${kernel:+$kernel-}$os" exit # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/erts/autoconf/install-sh b/erts/autoconf/install-sh index a5897de6ea..ec298b5374 100755 --- a/erts/autoconf/install-sh +++ b/erts/autoconf/install-sh @@ -1,7 +1,7 @@ #!/bin/sh # install - install a program, script, or datafile -scriptversion=2006-12-25.00 +scriptversion=2020-11-14.01; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the @@ -35,25 +35,21 @@ scriptversion=2006-12-25.00 # FSF changes to this file are in the public domain. # # Calling this script install-sh is preferred over install.sh, to prevent -# `make' implicit rules from creating a file called install from it +# 'make' implicit rules from creating a file called install from it # when there is no Makefile. # # This script is compatible with the BSD install script, but was written # from scratch. +tab=' ' nl=' ' -IFS=" "" $nl" +IFS=" $tab$nl" -# set DOITPROG to echo to test this script +# Set DOITPROG to "echo" to test this script. -# Don't use :- since 4.3BSD and earlier shells don't like it. doit=${DOITPROG-} -if test -z "$doit"; then - doit_exec=exec -else - doit_exec=$doit -fi +doit_exec=${doit:-exec} # Put in absolute file names if you don't have them in your path; # or use environment vars. @@ -68,22 +64,16 @@ mvprog=${MVPROG-mv} rmprog=${RMPROG-rm} stripprog=${STRIPPROG-strip} -posix_glob='?' -initialize_posix_glob=' - test "$posix_glob" != "?" || { - if (set -f) 2>/dev/null; then - posix_glob= - else - posix_glob=: - fi - } -' - posix_mkdir= # Desired mode of installed file. mode=0755 +# Create dirs (including intermediate dirs) using mode 755. +# This is like GNU 'install' as of coreutils 8.32 (2020). +mkdir_umask=22 + +backupsuffix= chgrpcmd= chmodcmd=$chmodprog chowncmd= @@ -97,7 +87,7 @@ dir_arg= dst_arg= copy_on_change=false -no_target_directory= +is_target_a_directory=possibly usage="\ Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE @@ -114,18 +104,28 @@ Options: --version display version info and exit. -c (ignored) - -C install only if different (preserve the last data modification time) + -C install only if different (preserve data modification time) -d create directories instead of installing files. -g GROUP $chgrpprog installed files to GROUP. -m MODE $chmodprog installed files to MODE. -o USER $chownprog installed files to USER. + -p pass -p to $cpprog. -s $stripprog installed files. + -S SUFFIX attempt to back up existing files, with suffix SUFFIX. -t DIRECTORY install into DIRECTORY. -T report an error if DSTFILE is a directory. Environment variables override the default commands: CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG + +By default, rm is invoked with -f; when overridden with RMPROG, +it's up to you to specify -f if you want it. + +If -S is not specified, no backups are attempted. + +Email bug reports to bug-automake@gnu.org. +Automake home page: https://www.gnu.org/software/automake/ " while test $# -ne 0; do @@ -137,42 +137,62 @@ while test $# -ne 0; do -d) dir_arg=true;; -g) chgrpcmd="$chgrpprog $2" - shift;; + shift;; --help) echo "$usage"; exit $?;; -m) mode=$2 - case $mode in - *' '* | *' '* | *' -'* | *'*'* | *'?'* | *'['*) - echo "$0: invalid mode: $mode" >&2 - exit 1;; - esac - shift;; + case $mode in + *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; -o) chowncmd="$chownprog $2" - shift;; + shift;; + + -p) cpprog="$cpprog -p";; -s) stripcmd=$stripprog;; - -t) dst_arg=$2 - shift;; + -S) backupsuffix="$2" + shift;; + + -t) + is_target_a_directory=always + dst_arg=$2 + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + shift;; - -T) no_target_directory=true;; + -T) is_target_a_directory=never;; --version) echo "$0 $scriptversion"; exit $?;; - --) shift - break;; + --) shift + break;; - -*) echo "$0: invalid option: $1" >&2 - exit 1;; + -*) echo "$0: invalid option: $1" >&2 + exit 1;; *) break;; esac shift done +# We allow the use of options -d and -T together, by making -d +# take the precedence; this is for compatibility with GNU install. + +if test -n "$dir_arg"; then + if test -n "$dst_arg"; then + echo "$0: target directory not allowed when installing a directory." >&2 + exit 1 + fi +fi + if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then # When -d is used, all remaining arguments are directories to create. # When -t is used, the destination is already specified. @@ -186,6 +206,10 @@ if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then fi shift # arg dst_arg=$arg + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac done fi @@ -194,13 +218,26 @@ if test $# -eq 0; then echo "$0: no input file specified." >&2 exit 1 fi - # It's OK to call `install-sh -d' without argument. + # It's OK to call 'install-sh -d' without argument. # This can happen when creating conditional directories. exit 0 fi if test -z "$dir_arg"; then - trap '(exit $?); exit' 1 2 13 15 + if test $# -gt 1 || test "$is_target_a_directory" = always; then + if test ! -d "$dst_arg"; then + echo "$0: $dst_arg: Is not a directory." >&2 + exit 1 + fi + fi +fi + +if test -z "$dir_arg"; then + do_exit='(exit $ret); exit $ret' + trap "ret=129; $do_exit" 1 + trap "ret=130; $do_exit" 2 + trap "ret=141; $do_exit" 13 + trap "ret=143; $do_exit" 15 # Set umask so as not to create temps with too-generous modes. # However, 'strip' requires both read and write access to temps. @@ -211,16 +248,16 @@ if test -z "$dir_arg"; then *[0-7]) if test -z "$stripcmd"; then - u_plus_rw= + u_plus_rw= else - u_plus_rw='% 200' + u_plus_rw='% 200' fi cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; *) if test -z "$stripcmd"; then - u_plus_rw= + u_plus_rw= else - u_plus_rw=,u+rw + u_plus_rw=,u+rw fi cp_umask=$mode$u_plus_rw;; esac @@ -228,9 +265,9 @@ fi for src do - # Protect names starting with `-'. + # Protect names problematic for 'test' and other utilities. case $src in - -*) src=./$src;; + -* | [=\(\)!]) src=./$src;; esac if test -n "$dir_arg"; then @@ -238,6 +275,10 @@ do dstdir=$dst test -d "$dstdir" dstdir_status=$? + # Don't chown directories that already exist. + if test $dstdir_status = 0; then + chowncmd="" + fi else # Waiting for this to be detected by the "$cpprog $src $dsttmp" command @@ -252,185 +293,150 @@ do echo "$0: no destination specified." >&2 exit 1 fi - dst=$dst_arg - # Protect names starting with `-'. - case $dst in - -*) dst=./$dst;; - esac - # If destination is a directory, append the input filename; won't work - # if double slashes aren't ignored. + # If destination is a directory, append the input filename. if test -d "$dst"; then - if test -n "$no_target_directory"; then - echo "$0: $dst_arg: Is a directory" >&2 - exit 1 + if test "$is_target_a_directory" = never; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 fi dstdir=$dst - dst=$dstdir/`basename "$src"` + dstbase=`basename "$src"` + case $dst in + */) dst=$dst$dstbase;; + *) dst=$dst/$dstbase;; + esac dstdir_status=0 else - # Prefer dirname, but fall back on a substitute if dirname fails. - dstdir=` - (dirname "$dst") 2>/dev/null || - expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$dst" : 'X\(//\)[^/]' \| \ - X"$dst" : 'X\(//\)$' \| \ - X"$dst" : 'X\(/\)' \| . 2>/dev/null || - echo X"$dst" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q' - ` - + dstdir=`dirname "$dst"` test -d "$dstdir" dstdir_status=$? fi fi + case $dstdir in + */) dstdirslash=$dstdir;; + *) dstdirslash=$dstdir/;; + esac + obsolete_mkdir_used=false if test $dstdir_status != 0; then case $posix_mkdir in '') - # Create intermediate dirs using mode 755 as modified by the umask. - # This is like FreeBSD 'install' as of 1997-10-28. - umask=`umask` - case $stripcmd.$umask in - # Optimize common cases. - *[2367][2367]) mkdir_umask=$umask;; - .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; - - *[0-7]) - mkdir_umask=`expr $umask + 22 \ - - $umask % 100 % 40 + $umask % 20 \ - - $umask % 10 % 4 + $umask % 2 - `;; - *) mkdir_umask=$umask,go-w;; - esac - - # With -d, create the new directory with the user-specified mode. - # Otherwise, rely on $mkdir_umask. - if test -n "$dir_arg"; then - mkdir_mode=-m$mode + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + # The $RANDOM variable is not portable (e.g., dash). Use it + # here however when possible just to lower collision chance. + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + + trap ' + ret=$? + rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null + exit $ret + ' 0 + + # Because "mkdir -p" follows existing symlinks and we likely work + # directly in world-writeable /tmp, make sure that the '$tmpdir' + # directory is successfully created first before we actually test + # 'mkdir -p'. + if (umask $mkdir_umask && + $mkdirprog $mkdir_mode "$tmpdir" && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + test_tmpdir="$tmpdir/a" + ls_ld_tmpdir=`ls -ld "$test_tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" else - mkdir_mode= + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null fi - - posix_mkdir=false - case $umask in - *[123567][0-7][0-7]) - # POSIX mkdir -p sets u+wx bits regardless of umask, which - # is incompatible with FreeBSD 'install' when (umask & 300) != 0. - ;; - *) - tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ - trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 - - if (umask $mkdir_umask && - exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 - then - if test -z "$dir_arg" || { - # Check for POSIX incompatibilities with -m. - # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or - # other-writeable bit of parent directory when it shouldn't. - # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. - ls_ld_tmpdir=`ls -ld "$tmpdir"` - case $ls_ld_tmpdir in - d????-?r-*) different_mode=700;; - d????-?--*) different_mode=755;; - *) false;; - esac && - $mkdirprog -m$different_mode -p -- "$tmpdir" && { - ls_ld_tmpdir_1=`ls -ld "$tmpdir"` - test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" - } - } - then posix_mkdir=: - fi - rmdir "$tmpdir/d" "$tmpdir" - else - # Remove any dirs left behind by ancient mkdir implementations. - rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null - fi - trap '' 0;; - esac;; + trap '' 0;; esac if $posix_mkdir && ( - umask $mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" ) then : else - # The umask is ridiculous, or mkdir does not conform to POSIX, + # mkdir does not conform to POSIX, # or it failed possibly due to a race condition. Create the # directory the slow way, step by step, checking for races as we go. case $dstdir in - /*) prefix='/';; - -*) prefix='./';; - *) prefix='';; + /*) prefix='/';; + [-=\(\)!]*) prefix='./';; + *) prefix='';; esac - eval "$initialize_posix_glob" - oIFS=$IFS IFS=/ - $posix_glob set -f + set -f set fnord $dstdir shift - $posix_glob set +f + set +f IFS=$oIFS prefixes= for d do - test -z "$d" && continue - - prefix=$prefix$d - if test -d "$prefix"; then - prefixes= - else - if $posix_mkdir; then - (umask=$mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break - # Don't fail if two instances are running concurrently. - test -d "$prefix" || exit 1 - else - case $prefix in - *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; - *) qprefix=$prefix;; - esac - prefixes="$prefixes '$qprefix'" - fi - fi - prefix=$prefix/ + test X"$d" = X && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ done if test -n "$prefixes"; then - # Don't fail if two instances are running concurrently. - (umask $mkdir_umask && - eval "\$doit_exec \$mkdirprog $prefixes") || - test -d "$dstdir" || exit 1 - obsolete_mkdir_used=true + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true fi fi fi @@ -443,14 +449,25 @@ do else # Make a couple of temp file names in the proper directory. - dsttmp=$dstdir/_inst.$$_ - rmtmp=$dstdir/_rm.$$_ + dsttmp=${dstdirslash}_inst.$$_ + rmtmp=${dstdirslash}_rm.$$_ # Trap to clean up those temp files at exit. trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 # Copy the file name to the temp name. - (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + (umask $cp_umask && + { test -z "$stripcmd" || { + # Create $dsttmp read-write so that cp doesn't create it read-only, + # which would cause strip to fail. + if test -z "$doit"; then + : >"$dsttmp" # No need to fork-exec 'touch'. + else + $doit touch "$dsttmp" + fi + } + } && + $doit_exec $cpprog "$src" "$dsttmp") && # and set any options; do chmod last to preserve setuid bits. # @@ -465,20 +482,24 @@ do # If -C, don't bother to copy if it wouldn't change the file. if $copy_on_change && - old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && - new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && - - eval "$initialize_posix_glob" && - $posix_glob set -f && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + set -f && set X $old && old=:$2:$4:$5:$6 && set X $new && new=:$2:$4:$5:$6 && - $posix_glob set +f && - + set +f && test "$old" = "$new" && $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 then rm -f "$dsttmp" else + # If $backupsuffix is set, and the file being installed + # already exists, attempt a backup. Don't worry if it fails, + # e.g., if mv doesn't support -f. + if test -n "$backupsuffix" && test -f "$dst"; then + $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null + fi + # Rename the file to the real destination. $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || @@ -486,24 +507,24 @@ do # to itself, or perhaps because mv is so ancient that it does not # support -f. { - # Now remove or move aside any old file at destination location. - # We try this two ways since rm can't unlink itself on some - # systems and the destination file might be busy for other - # reasons. In this case, the final cleanup might fail but the new - # file should still install successfully. - { - test ! -f "$dst" || - $doit $rmcmd -f "$dst" 2>/dev/null || - { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && - { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } - } || - { echo "$0: cannot unlink or rename $dst" >&2 - (exit 1); exit 1 - } - } && - - # Now rename the file to the real destination. - $doit $mvcmd "$dsttmp" "$dst" + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" } fi || exit 1 @@ -512,8 +533,9 @@ do done # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-end: "$" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" # End: diff --git a/erts/configure.in b/erts/configure.in index 0a059be46a..418a111922 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2159,14 +2159,33 @@ AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2]) AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \ pread pwrite memmove strerror strerror_r strncasecmp \ - gethrtime localtime_r gmtime_r inet_pton mprotect madvise posix_madvise \ + gethrtime localtime_r gmtime_r mprotect madvise posix_madvise \ mmap mremap memcpy mallopt sbrk _sbrk __sbrk brk _brk __brk \ flockfile fstat strlcpy strlcat setsid posix2time time2posix \ setlocale nl_langinfo poll mlockall ppoll vsyslog]) +## We have a special check for inet_pton as AC_CHECK_FUCNS does not work +## on windows 32-bit as there a macro is used to rename the symbol... +AC_MSG_CHECKING([for inet_pton]) +AC_TRY_LINK([ +#ifdef WIN32 +#include <ws2tcpip.h> +#else +#include <arpa/inet.h> +#endif +],[inet_pton(2,"",(void*)0)], have_inet_pton=yes, have_inet_pton=no) + +if test $have_inet_pton = yes; then + AC_DEFINE(HAVE_INET_PTON,[1], + [Define to 1 if you have the `inet_pton' function.]) + AC_MSG_RESULT(yes) +else + AC_MSG_RESULT(no) +fi + AC_MSG_CHECKING([for isfinite]) AC_TRY_LINK([#include <math.h>], - [isfinite(0);], have_isfinite=yes, have_isfinite=no), + [isfinite(0);], have_isfinite=yes, have_isfinite=no) if test $have_isfinite = yes; then AC_DEFINE(HAVE_ISFINITE,[1], @@ -2847,9 +2866,11 @@ AC_CHECK_PROG(M4, m4, m4) dnl Test if JIT can be enabled +JIT_ARCH= if test ${enable_jit} != no; then case "$ARCH" in amd64) + JIT_ARCH=x86 ;; *) if test ${enable_jit} = yes; then @@ -2990,6 +3011,7 @@ else fi AC_SUBST(JIT_ENABLED) +AC_SUBST(JIT_ARCH) AC_SUBST(PRIMARY_FLAVOR) AC_SUBST(FLAVORS) diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index d8bb731d64..015ecec6ba 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -971,7 +971,11 @@ DiB == gen_digest(ChA, ICA)? <section> <marker id="dflags"/> <title>Distribution Flags</title> - <p>The following capability flags are defined:</p> + <p>Early in the distribution handshake the two participating nodes + exchange capability flags. This is done in order to determine how the + communication between the two nodes should be performed. The intersection + of the capabilities presented by the two nodes defines the capabilities + that will be used. The following capability flags are defined:</p> <taglist> <tag><c>-define(DFLAG_PUBLISHED,16#1).</c></tag> <item> @@ -1093,6 +1097,14 @@ DiB == gen_digest(ChA, ICA)? <p>The node supports the new connection setup handshake (version 6) introduced in OTP 23.</p> </item> + <tag><marker id="DFLAG_UNLINK_ID"/><c>-define(DFLAG_UNLINK_ID, 16#2000000).</c></tag> + <item> + <p>Use the <seeguide marker="#new_link_protocol">new link protocol</seeguide>.</p> + <note><p>This flag will become mandatory in OTP 26.</p></note> + <p>Unless both nodes have set the <c>DFLAG_UNLINK_ID</c> flag, the + <seeguide marker="#old_link_protocol">old link protocol</seeguide> + will be used as a fallback.</p> + </item> <tag><marker id="DFLAG_SPAWN"/><c>-define(DFLAG_SPAWN, (1 bsl 32)).</c></tag> <item> <p>Set if the <seeguide marker="#SPAWN_REQUEST"><c>SPAWN_REQUEST</c></seeguide>, @@ -1110,7 +1122,7 @@ DiB == gen_digest(ChA, ICA)? <tag><marker id="DFLAG_V4_NC"/><c>-define(DFLAG_V4_NC, (1 bsl 34)).</c></tag> <item> <p>The node accepts a larger amount of data in pids, ports and - references (node container types). In the pid case full 32-bit + references (node container types version 4). In the pid case full 32-bit <c>ID</c> and <c>Serial</c> fields in <seeguide marker="erl_ext_dist#NEW_PID_EXT"><c>NEW_PID_EXT</c></seeguide>, in the port case a 64-bit integer in @@ -1118,6 +1130,7 @@ DiB == gen_digest(ChA, ICA)? and in the reference case up to 5 32-bit ID words are now accepted in <seeguide marker="erl_ext_dist#NEWER_REFERENCE_EXT"><c>NEWER_REFERENCE_EXT</c></seeguide>. Introduced in OTP 24.</p> + <note><p>This flag will become mandatory in OTP 26.</p></note> </item> <tag><marker id="DFLAG_ALIAS"/><c>-define(DFLAG_ALIAS, (1 bsl 35)).</c></tag> <item> @@ -1238,9 +1251,12 @@ DiB == gen_digest(ChA, ICA)? which distributed operation it encodes:</p> <taglist> - <tag><c>LINK</c></tag> + <tag><marker id="LINK"/><c>LINK</c></tag> <item> <p><c>{1, FromPid, ToPid}</c></p> + <p>This signal is sent by <c>FromPid</c> in order + to create a link between <c>FromPid</c> and + <c>ToPid</c>.</p> </item> <tag><c>SEND</c></tag> <item> @@ -1253,9 +1269,18 @@ DiB == gen_digest(ChA, ICA)? <p><c>{3, FromPid, ToPid, Reason}</c></p> <p>This signal is sent when a link has been broken</p> </item> - <tag><c>UNLINK</c></tag> + <tag><marker id="UNLINK"/><c>UNLINK</c> (deprecated)</tag> <item> <p><c>{4, FromPid, ToPid}</c></p> + <p>This signal is sent by <c>FromPid</c> in order to remove + a link between <c>FromPid</c> and <c>ToPid</c>, when using the + <seeguide marker="#old_link_protocol">old link + protocol</seeguide>.</p> + <warning><p>This signal has been deprecated and will not + be supported in OTP 26. For more information see the + documentation of the + <seeguide marker="#new_link_protocol">new link protocol</seeguide>. + </p></warning> </item> <tag><c>NODE_LINK</c></tag> <item> @@ -1317,9 +1342,8 @@ DiB == gen_digest(ChA, ICA)? <c>Reason</c> = exit reason for the monitored process</p> </item> </taglist> - </section> - <section> + <section> <title>New Ctrlmessages for Erlang/OTP 21</title> <taglist> <tag><c>SEND_SENDER</c></tag> @@ -1530,6 +1554,47 @@ DiB == gen_digest(ChA, ICA)? has been passed. </p> </item> + <tag><marker id="UNLINK_ID"/><c>UNLINK_ID</c></tag> + <item> + <p><c>{35, Id, FromPid, ToPid}</c></p> + <p>This signal is sent by <c>FromPid</c> in order to remove a + link between <c>FromPid</c> and <c>ToPid</c>. This unlink signal + replaces the <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide> + signal. Besides process identifiers of the sender and receiver + the <c>UNLINK_ID</c> signal also contains an integer identifier + <c>Id</c>. Valid range of <c>Id</c> is <c>[1, (1 bsl 64) - 1]</c>. + <c>Id</c> is to be passed back to the sender by the receiver in an + <seeguide marker="#UNLINK_ID_ACK"><c>UNLINK_ID_ACK</c></seeguide> + signal. <c>Id</c> must uniquely identify the <c>UNLINK_ID</c> signal + among all not yet acknowledged <c>UNLINK_ID</c> signals from + <c>FromPid</c> to <c>ToPid</c>.</p> + <p> + This signal is only passed when the + <seeguide marker="#new_link_protocol">new link protocol</seeguide> + has been negotiated using the + <seeguide marker="erl_dist_protocol#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> + <seeguide marker="erl_dist_protocol#dflags">distribution flag</seeguide>. + </p> + </item> + <tag><marker id="UNLINK_ID_ACK"/><c>UNLINK_ID_ACK</c></tag> + <item> + <p><c>{36, Id, FromPid, ToPid}</c></p> + <p>An unlink acknowledgement signal. This signal is sent as an + acknowledgement of the reception of an + <seeguide marker="#UNLINK_ID"><c>UNLINK_ID</c></seeguide> + signal. The <c>Id</c> element should be the same <c>Id</c> + as present in the <c>UNLINK_ID</c> signal. <c>FromPid</c> + identifies the sender of the <c>UNLINK_ID_ACK</c> signal and + <c>ToPid</c> identifies the sender of the <c>UNLINK_ID</c> + signal.</p> + <p> + This signal is only passed when the + <seeguide marker="#new_link_protocol">new link protocol</seeguide> + has been negotiated using the + <seeguide marker="erl_dist_protocol#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> + <seeguide marker="erl_dist_protocol#dflags">distribution flag</seeguide>. + </p> + </item> </taglist> </section> <section> @@ -1556,4 +1621,204 @@ DiB == gen_digest(ChA, ICA)? </item> </taglist> </section> + <section> + <marker id="link_protocol"/> + <title>Link Protocol</title> + + <section> + <marker id="new_link_protocol"/> + <title>New Link Protocol</title> + + <p> + The new link protocol will be used when both nodes flag that + they understand it using the + <seeguide marker="#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> + <seeguide marker="#dflags">distribution flag</seeguide>. If + one of the nodes does not understand the new link protocol, the + <seeguide marker="#old_link_protocol">old link protocol</seeguide> + will be used as a fallback. + </p> + + <p> + The new link protocol introduces two new signals, + <seeguide marker="#UNLINK_ID"><c>UNLINK_ID</c></seeguide> and + <seeguide marker="#UNLINK_ID"><c>UNLINK_ID_ACK</c></seeguide>, + which replace the old + <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide> + signal. The old <seeguide marker="#LINK"><c>LINK</c></seeguide> + signal is still sent in order to set up a link, but handled + differently upon reception. + </p> + + <p> + In order to set up a link, a <c>LINK</c> signal is sent, from + the process initiating the operation, to the process that it + wants to link to. In order to remove a link, an + <c>UNLINK_ID</c> signal is sent, from the process initiating + the operation, to the linked process. The receiver of an + <c>UNLINK_ID</c> signal responds with an <c>UNLINK_ID_ACK</c> + signal. Upon reception of an <c>UNLINK_ID</c> signal, the + corresponding <c>UNLINK_ID_ACK</c> signal <em>must</em> be + sent before any other signals are sent to the sender of the + <c>UNLINK_ID</c> signal. Together with + <seeguide marker="erts:communication#passing-of-signals">the + signal ordering guarantees</seeguide> of Erlang this makes it + possible for the sender of the <c>UNLINK_ID</c> signal to know + the order of other signals which is essential for the protocol. + The <c>UNLINK_ID_ACK</c> signal should contain the same + <c>Id</c> as the <c>Id</c> contained in the <c>UNLINK_ID</c> + signal being acknowledged. + </p> + + <p> + Processes also need to maintain process local information about + links. The state of this process local information is changed + when the signals above are sent and received. This process + local information also determines if a signal should be sent + when a process calls + <seemfa marker="erlang#link/1"><c>link/1</c></seemfa> or + <seemfa marker="erlang#unlink/1"><c>unlink/1</c></seemfa>. + A <c>LINK</c> signal is only sent if there does not currently + exist an active link between the processes according to the + process local information and an <c>UNLINK_ID</c> signal is + only sent if there currently exists an active link between the + processes according to the process local information. + </p> + + <p> + The process local information about a link contains: + </p> + <taglist> + <tag>Pid</tag> + <item> + Process identifier of the linked process. + </item> + <tag>Active Flag</tag> + <item> + If set, the link is active and the process will react on + <seeguide marker="system/reference_manual:processes#receiving_exit_signals">incoming + exit signals</seeguide> issued due to the link. If not set, + the link is inactive and incoming exit signals, issued due + to the link, will be ignored. That is, the processes are + considered as <em>not</em> linked. + </item> + <tag>Unlink Id</tag> + <item> + Identifier of an outstanding unlink operation. That is, + an unlink operation that has not yet been acknowledged. + This information is only used when the active flag is not + set. + </item> + </taglist> + + <p> + A process is only considered linked to another process + if it has process local information about the link + containing the process identifier of the other process and + with the active flag set. + </p> + + <p> + The process local information about a link is updated as + follows: + </p> + <taglist> + <tag>A <c>LINK</c> signal is sent</tag> + <item> + Link information is created if not already existing. The + active flag is set, and unlink id is cleared. That is, + if we had an outstanding unlink operation we will ignore + the result of that operation and enable the link. + </item> + <tag>A <c>LINK</c> signal is received</tag> + <item> + If no link information already exists, it is created, the + active flag is set and unlink id is cleared. If the link + information already exists, the signal is silently ignored, + regardless of whether the active flag is set or not. + That is, if we have an outstanding unlink operation we will + <em>not</em> activate the link. In this scenario, the sender + of the <c>LINK</c> signal has not yet sent an + <c>UNLINK_ID_ACK</c> signal corresponding to our + <c>UNLINK_ID</c> signal which means that it will receive + our <c>UNLINK_ID</c> signal after it sent its + <c>LINK</c> signal. This in turn means that both processes + in the end will agree that there is no link between them. + </item> + <tag>An <c>UNLINK_ID</c> signal is sent</tag> + <item> + Link information already exists and the active flag is set + (otherwise the signal would not be sent). The active flag + is unset, and the unlink id of the signal is saved in the + link information. + </item> + <tag>An <c>UNLINK_ID</c> signal is received</tag> + <item> + If the active flag is set, information about the link + is removed. If the active flag is not set (that is, we have + an outstanding unlink operation), the information about the + link is left unchanged. + </item> + <tag>An <c>UNLINK_ID_ACK</c> signal is sent</tag> + <item> + This is done when an <c>UNLINK_ID</c> signal is received and + causes no further changes of the link information. + </item> + <tag>An <c>UNLINK_ID_ACK</c> signal is received</tag> + <item> + If information about the link exists, the active flag is not + set, and the unlink id in the link information equals the + <c>Id</c> in the signal, the link information is removed; + otherwise, the signal is ignored. + </item> + </taglist> + + <p> + When a process receives an exit signal due to a link, the + process will first react to the exit signal if the link + is active and then remove the process local information about + the link. + </p> + <p> + In case the connection is lost between two nodes, exit signals + with exit reason <c>noconnection</c> are sent to all processes + with links over the connection. This will cause all process + local information about links over the connection to be + removed. + </p> + <p> + Exactly the same link protocol is also used internally on an + Erlang node. The signals however have different formats since + they do not have to be sent over the wire. + </p> + </section> + + <section> + <marker id="old_link_protocol"/> + <title>Old Link Protocol</title> + + <p> + The old link protocol utilize two signals + <seeguide marker="#LINK"><c>LINK</c></seeguide>, and + <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide>. The + <c>LINK</c> signal informs the other process that a link + should be set up, and the <c>UNLINK</c> signal informs the + other process that a link should be removed. This protocol + is however a bit too naive. If both processes operate on the + link simultaneously, the link may end up in an inconsistent + state where one process thinks it is linked while the other + thinks it is not linked. + </p> + <p> + This protocol is deprecated and support for it will be removed + in OTP 26. Until then, it will be used as fallback when + communicating with old nodes that do not understand the + <seeguide marker="#new_link_protocol">new link + protocol</seeguide>. + </p> + </section> + + </section> + </section> + </chapter> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index e4c45a5260..384b0f21e0 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -3079,39 +3079,85 @@ is_process_alive(P2Pid), <func> <name name="link" arity="1" since=""/> - <fsummary>Create a link to another process (or port).</fsummary> + <fsummary>Create a link to another process or port.</fsummary> <desc> - <p>Creates a link between the calling process and another process (or - port) <c><anno>PidOrPort</anno></c>. If the link already exists or a - process attempts to create a link to itself, nothing is done. Returns - <c>true</c> if the link is set up.</p> - - <p>If <c><anno>PidOrPort</anno></c> does not exist and checking it is - cheap, a <c>noproc</c> error is raised. Currently, checking is cheap - if the <c><anno>PidOrPort</anno></c> is local and the caller does not - trap exits (see <seemfa marker="#process_flag/2"><c>process_flag/2 - </c></seemfa>).</p> - - <p>Apart from any exit signals from the linked process itself, two - special exit signals may be sent to the calling process:</p> - - <list type="bulleted"> + <p> + Sets up and activates a link between the calling process and + another process or a port identified by + <c><anno>PidOrPort</anno></c>. We will from here on call the + identified process or port linkee. If the linkee is a port, it + must reside on the same node as the caller. + </p> - <item><p><c>noproc</c> is sent immediately if - <c><anno>PidOrPort</anno></c> does not exist at the time of linking - (if the caller is trapping exits or <c><anno>PidOrPort</anno></c> is - remote).</p></item> + <p> + If one of the participants of a link terminates, it will + <seeguide marker="system/reference_manual:processes#sending_exit_signals">send + an exit signal</seeguide> to the other participant. The exit + signal will contain the + <seeguide marker="system/reference_manual:processes#link_exit_signal_reason">exit + reason</seeguide> of the terminated participant. Other cases when + exit signals are triggered due to a link are when no linkee exist + (<c>noproc</c> exit reason) and when the connection between linked + processes on different nodes is lost or cannot be established + (<c>noconnection</c> exit reason). + </p> - <item><p><c>noconnection</c> if <c><anno>PidOrPort</anno></c> is - remote and a connection between the nodes could not be established - or was severed.</p></item> + <p> + An existing link can be removed by calling + <seemfa marker="#unlink/1"><c>unlink/1</c></seemfa>. + For more information on links and exit signals due to links, see + the <i>Processes</i> chapter in the <i>Erlang Reference Manual</i>: + </p> + <list> + <item> + <seeguide marker="system/reference_manual:processes#links">Links</seeguide> + </item> + <item> + <seeguide marker="system/reference_manual:processes#sending_exit_signals">Sending + Exit Signals</seeguide> + </item> + <item> + <seeguide marker="system/reference_manual:processes#receiving_exit_signals">Receiving + Exit Signals</seeguide> + </item> + </list> - </list> + <p> + For historical reasons, <c>link/1</c> has a strange + semi-synchronous behavior when it is "cheap" to check if the + linkee exists or not, and the caller does not + <seeerl marker="erlang#process_flag_trap_exit">trap exits</seeerl>. + If the above is true and the linkee does not exist, <c>link/1</c> + will raise a <c>noproc</c> error <em>exception</em>. The expected + behavior would instead have been that <c>link/1</c> returned + <c>true</c>, and the caller later was sent an exit signal + with <c>noproc</c> exit reason, but this is unfortunately not the + case. The <c>noproc</c> + <seeguide marker="system/reference_manual:errors#exceptions"> + exception</seeguide> is not to be confused with an + <seeguide marker="system/reference_manual:processes#sending_exit_signals">exit + signal</seeguide> with exit reason <c>noproc</c>. Currently it is "cheap" + to check if the linkee exists when it is supposed to reside on + the same node as the calling process. + </p> - <p>See <seeguide marker="system/reference_manual:processes#links">Processes - ➜ Links</seeguide> in the Erlang Reference Manual for more details.</p> + <p> + The link setup and activation is performed asynchronously. If the + link already exists, or if the caller attempts to create a link + to itself, nothing is done. A detailed description of the + <seeguide marker="erts:erl_dist_protocol#link_protocol">link + protocol</seeguide> can be found in the <i>Distribution Protocol</i> + chapter of the <i>ERTS User's Guide</i>. + </p> + <p>Failure:</p> + <list> + <item><c>badarg</c> if <c><anno>PidOrPort</anno></c> does not identify + a process or a node local port.</item> + <item><c>noproc</c> linkee does not exist and it is "cheap" to check + if it exists as described above.</item> + </list> </desc> </func> @@ -5518,7 +5564,8 @@ RealSystem = system + MissedSystem</code> </func> <func> - <name name="process_flag" arity="2" clause_i="1" since=""/> + <name name="process_flag" arity="2" clause_i="1" + anchor="process_flag_trap_exit" since=""/> <fsummary>Set process flag trap_exit for the calling process.</fsummary> <desc> <p>When <c>trap_exit</c> is set to <c>true</c>, exit signals @@ -6563,7 +6610,7 @@ true</pre> send operator</seeguide>: <c><anno>Dest</anno> ! <anno>Msg</anno></c>.</p> <p><c><anno>Dest</anno></c> can be a remote or local process identifier, - a (local) port, a locally registered name, or a tuple + an alias, a (local) port, a locally registered name, or a tuple <c>{<anno>RegName</anno>, <anno>Node</anno>}</c> for a registered name at another node.</p> <p>The function fails with a <c>badarg</c> run-time error if @@ -11561,13 +11608,6 @@ improper_end</pre> <p>When <c>PidPort</c> gets unlinked from a process <c>Pid2</c>.</p> </item> <tag> - <marker id="trace_3_trace_messages_exit"></marker> - <c>{trace, Pid, exit, Reason}</c> - </tag> - <item> - <p>When <c>Pid</c> exits with reason <c>Reason</c>.</p> - </item> - <tag> <marker id="trace_3_trace_messages_open"></marker> <c>{trace, Port, open, Pid, Driver}</c> </tag> @@ -12545,27 +12585,50 @@ improper_end</pre> <name name="unlink" arity="1" since=""/> <fsummary>Remove a link to another process or port.</fsummary> <desc> - <p>Removes the link, if there is one, between the calling - process and the process or port referred to by - <c><anno>Id</anno></c>.</p> - <p>Returns <c>true</c> and does not fail, even if there is no - link to <c><anno>Id</anno></c>, or if <c><anno>Id</anno></c> - does not exist.</p> - <p>Once <c>unlink(<anno>Id</anno>)</c> has returned, - it is guaranteed that - the link between the caller and the entity referred to by - <c><anno>Id</anno></c> has no effect on the caller - in the future (unless - the link is setup again). If the caller is trapping exits, an - <c>{'EXIT', <anno>Id</anno>, _}</c> message from the link - can have been placed in the caller's message queue before - the call.</p> - <p>Notice that the <c>{'EXIT', <anno>Id</anno>, _}</c> - message can be the - result of the link, but can also be the result of <c>Id</c> - calling <c>exit/2</c>. Therefore, it <em>can</em> be - appropriate to clean up the message queue when trapping exits - after the call to <c>unlink(<anno>Id</anno>)</c>, as follows:</p> + <p> + Removes a link between the calling process and another process + or a port identified by <c><anno>Id</anno></c>. We will from + here on call the identified process or port unlinkee. + </p> + + <p> + A link can be set up using the + <seemfa marker="#link/1"><c>link/1</c></seemfa> BIF. For more + information on links and exit signals due to links, see the + <i>Processes</i> chapter in the <i>Erlang Reference Manual</i>: + </p> + <list> + <item> + <seeguide marker="system/reference_manual:processes#links">Links</seeguide> + </item> + <item> + <seeguide marker="system/reference_manual:processes#sending_exit_signals">Sending + Exit Signals</seeguide> + </item> + <item> + <seeguide marker="system/reference_manual:processes#receiving_exit_signals">Receiving + Exit Signals</seeguide> + </item> + </list> + + <p> + Once <c>unlink(<anno>Id</anno>)</c> has returned, it is + guaranteed that the link between the caller and the unlinkee + has no effect on the caller in the future (unless the link is + setup again). Note that if the caller is + <seeerl marker="erts:erlang#process_flag_trap_exit">trapping + exits</seeerl>, an <c>{'EXIT', <anno>Id</anno>, ExitReason}</c> + message due to the link may have been placed in the message + queue of the caller before the <c>unlink(<anno>Id</anno>)</c> + call completed. Also note that the <c>{'EXIT', <anno>Id</anno>, + ExitReason}</c> message may be the result of the link, but + may also be the result of the unlikee sending the caller an + exit signal by calling the + <seemfa marker="#exit/2"><c>exit/2</c></seemfa> BIF. + Therefore, it may or may not be appropriate to clean up + the message queue after a call to <c>unlink(<anno>Id</anno>)</c> + as follows, when trapping exits: + </p> <code type="none"> unlink(Id), receive @@ -12574,16 +12637,19 @@ receive after 0 -> true end</code> - <note> - <p>Before Erlang/OTP R11B (ERTS 5.5) <c>unlink/1</c> - behaved completely asynchronously, that is, the link was active - until the "unlink signal" reached the linked entity. This - had an undesirable effect, as you could never know when - you were guaranteed <em>not</em> to be effected by the link.</p> - <p>The current behavior can be viewed as two combined operations: - asynchronously send an "unlink signal" to the linked entity - and ignore any future results of the link.</p> - </note> + + <p> + The link removal is performed asynchronously. If such a link + does not exist, nothing is done. A detailed description of the + <seeguide marker="erts:erl_dist_protocol#link_protocol">link + protocol</seeguide> can be found in the <i>Distribution Protocol</i> + chapter of the <i>ERTS User's Guide</i>. + </p> + + <p> + Failure: <c>badarg</c> if <c>Id</c> does not identify a process + or a node local port. + </p> </desc> </func> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 04edba9bef..de11e50ec2 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,159 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 11.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix memory leak of about 6 words when + <c>erlang:process_flag/3</c> is called with a pid of an + already dead process. Bug exists since OTP-21.0.</p> + <p> + Own Id: OTP-17081 Aux Id: PR-2930 </p> + </item> + <item> + <p> + Fixed small memory leak in <c>erl_drv_send_term</c> and + <c>erl_drv_output_term</c> when failing due to the term + being invalid.</p> + <p> + Own Id: OTP-17089 Aux Id: PR-2934 </p> + </item> + <item> + <p> + The DTrace/SystemTap <c>process_heap_grow</c> probe is + now called with valid the heap and stack pointers for the + process in question.</p> + <p> + Own Id: OTP-17096 Aux Id: PR-2932 </p> + </item> + <item> + <p>Fixed a performance issue in memory allocation for + Linux kernels that didn't support <c>MADV_FREE</c>.</p> + <p> + Own Id: OTP-17124</p> + </item> + <item> + <p> + A <seeguide + marker="erts:erl_dist_protocol#new_link_protocol">new + link protocol</seeguide> has been introduced which + prevents links from ending up in an inconsistent state + where one participant considers itself linked while the + other doesn't. This bug has always existed in the + distributed case, but has since OTP 21 also existed in + the node local case since the distributed link protocol + then was adopted also for node local links. The bug + could, however, only trigger if both participants + operated on the link simultaneously.</p> + <p> + Own Id: OTP-17127</p> + </item> + <item> + <p> + Fix memory leak when receiving sigchld from port program + to already dead port.</p> + <p> + Own Id: OTP-17163</p> + </item> + <item> + <p> + Fix bug where complex seq_trace tokens (that is lists, + tuples, maps etc) could becomes corrupted by the GC. The + bug was introduced in OTP-21.</p> + <p> + Own Id: OTP-17209 Aux Id: PR-3039 </p> + </item> + <item> + <p> + Fixed WSLPATH environment variable addition to PATH on + windows, the last character was lost.</p> + <p> + Own Id: OTP-17229</p> + </item> + <item> + <p> + Fixed a bug in the timer implementation which could cause + timers that were set to more than 37.25 hours in the + future to be delayed. This could occur if there were + multiple timers scheduled to be triggered very close in + time, but still at different times, and the scheduler + thread handling the timers was not able to handle them + quickly enough. Delayed timers were in this case + triggered when another unrelated timer was triggered.</p> + <p> + Own Id: OTP-17253</p> + </item> + <item> + <p> + Fixed small memory leak in <c>erlang:trace/3</c> if + option <c>{tracer,_}</c> is included and the option list + is invalid or the call races with a concurrent trace or + code change operation.</p> + <p> + Own Id: OTP-17265 Aux Id: PR-4596 </p> + </item> + <item> + <p> + Fix configure check for <c>inet_pton</c> on 32-bit + windows. The failure of this check would cause epmd to be + built without ipv6 support.</p> + <p> + Own Id: OTP-17283</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Various address sanitizer support.</p> + <p> + Own Id: OTP-16959 Aux Id: PR-2965 </p> + </item> + <item> + <p>The emulator will now honor <c>cgroup2</c> CPU + quotas.</p> + <p> + Own Id: OTP-17002</p> + </item> + <item> + <p> + Improved memory barrier usage on ARMv8 hardware, and + specifically on Apple silicon.</p> + <p> + Own Id: OTP-17195 Aux Id: PR-4505, PR-4538 </p> + </item> + <item> + <p> + Improved memory barrier usage on 64-bit POWER hardware.</p> + <p> + Own Id: OTP-17200 Aux Id: PR-4510 </p> + </item> + <item> + <p> + Fix a file descriptor leak when using sendfile and the + remote side closes the connection. This bug has been + present since OTP-21.0.</p> + <p> + Own Id: OTP-17244</p> + </item> + <item> + <p> + Refinement of the documentation of the + <c>message_queue_data</c> process flag.</p> + <p> + Own Id: OTP-17252 Aux Id: PR-4568 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 11.1.8</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -1216,6 +1369,50 @@ </section> +<section><title>Erts 10.7.2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in the timer implementation which could cause + timers that were set to more than 37.25 hours in the + future to be delayed. This could occur if there were + multiple timers scheduled to be triggered very close in + time, but still at different times, and the scheduler + thread handling the timers was not able to handle them + quickly enough. Delayed timers were in this case + triggered when another unrelated timer was triggered.</p> + <p> + Own Id: OTP-17253</p> + </item> + <item> + <p> + Fix bug in call_time tracing (used by eprof) that could + cause VM crash. Bug exists since OTP-22.2 (but not in + OTP-23).</p> + <p> + Own Id: OTP-17290 Aux Id: GH-4635 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix a file descriptor leak when using sendfile and the + remote side closes the connection. This bug has been + present since OTP-21.0.</p> + <p> + Own Id: OTP-17244</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.7.2.8</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3236,6 +3433,42 @@ </section> +<section><title>Erts 10.3.5.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in the timer implementation which could cause + timers that were set to more than 37.25 hours in the + future to be delayed. This could occur if there were + multiple timers scheduled to be triggered very close in + time, but still at different times, and the scheduler + thread handling the timers was not able to handle them + quickly enough. Delayed timers were in this case + triggered when another unrelated timer was triggered.</p> + <p> + Own Id: OTP-17253</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix a file descriptor leak when using sendfile and the + remote side closes the connection. This bug has been + present since OTP-21.0.</p> + <p> + Own Id: OTP-17244</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.3.5.16</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml index 7b471bb95c..5d3fb9d8eb 100644 --- a/erts/doc/src/zlib.xml +++ b/erts/doc/src/zlib.xml @@ -59,7 +59,7 @@ list_to_binary([Compressed|Last])</pre> <p>In all functions errors, <c>{'EXIT',{Reason,Backtrace}}</c>, can be thrown, where <c>Reason</c> describes the error.</p> - <p>Typical <c>Reasons</c>s:</p> + <p>Typical <c>Reason</c>s:</p> <taglist> <tag><c>badarg</c></tag> @@ -326,9 +326,10 @@ list_to_binary([B1,B2])</pre> <p><c><anno>Level</anno></c> decides the compression level to be used:</p> <list type="bulleted"> - <item>0 (<c>none</c>), gives no compression</item> - <item>1 (<c>best_speed</c>) gives best speed</item> - <item>9 (<c>best_compression</c>) gives best compression</item> + <item><c>default</c> gives default compromise between speed and compression</item> + <item><c>none</c> (0) gives no compression</item> + <item><c>best_speed</c> (1) gives best speed</item> + <item><c>best_compression</c> (9) gives best compression</item> </list> </desc> </func> @@ -343,9 +344,10 @@ list_to_binary([B1,B2])</pre> <item> <p>Compression level to use:</p> <list type="bulleted"> - <item>0 (<c>none</c>), gives no compression</item> - <item>1 (<c>best_speed</c>) gives best speed</item> - <item>9 (<c>best_compression</c>) gives best compression</item> + <item><c>default</c> gives default compromise between speed and compression</item> + <item><c>none</c> (0) gives no compression</item> + <item><c>best_speed</c> (1) gives best speed</item> + <item><c>best_compression</c> (9) gives best compression</item> </list> </item> <tag><c><anno>Method</anno></c></tag> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 5320de8bef..c3449c5f73 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -25,6 +25,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk ENABLE_ALLOC_TYPE_VARS = @ENABLE_ALLOC_TYPE_VARS@ JIT_ENABLED=@JIT_ENABLED@ +JIT_ARCH=@JIT_ARCH@ DTRACE_ENABLED=@DTRACE_ENABLED@ DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@ USE_VM_PROBES=@USE_VM_PROBES@ @@ -57,9 +58,9 @@ endif ifeq ($(FLAVOR),jit) OMIT_OMIT_FP=no OPCODE_TABLES += \ - beam/jit/ops.tab \ - beam/jit/predicates.tab \ - beam/jit/generators.tab + beam/jit/$(JIT_ARCH)/ops.tab \ + beam/jit/$(JIT_ARCH)/predicates.tab \ + beam/jit/$(JIT_ARCH)/generators.tab else OMIT_OMIT_FP=no OPCODE_TABLES += \ @@ -414,8 +415,8 @@ CREATE_DIRS += $(OBJDIR) \ $(ZLIB_OBJDIR) ifeq ($(FLAVOR),jit) -CREATE_DIRS+=$(OBJDIR)/asmjit/ $(OBJDIR)/asmjit/core $(OBJDIR)/asmjit/x86 -CREATE_DIRS+=$(TTF_DIR)/asmjit $(TTF_DIR)/asmjit/core $(TTF_DIR)/asmjit/x86 +CREATE_DIRS+=$(OBJDIR)/asmjit/ $(OBJDIR)/asmjit/core $(OBJDIR)/asmjit/$(JIT_ARCH) +CREATE_DIRS+=$(TTF_DIR)/asmjit $(TTF_DIR)/asmjit/core $(TTF_DIR)/asmjit/$(JIT_ARCH) endif BINDIR = $(ERL_TOP)/bin/$(TARGET) @@ -690,8 +691,8 @@ $(UTILS_YCF_OUTPUT): beam/utils.c $(ERTS_LIB) GENERATE += $(DB_INSERT_LIST_OUTPUT) $(MAPS_YCF_OUTPUT) $(UTILS_YCF_OUTPUT) ## ASMJIT source files and headers -ASMJIT_CPP = $(wildcard asmjit/core/*.cpp) $(wildcard asmjit/x86/*.cpp) -ASMJIT_H = $(wildcard asmjit/*.h) $(wildcard asmjit/core/*.h) $(wildcard asmjit/x86/*.h) +ASMJIT_CPP = $(wildcard asmjit/core/*.cpp) $(wildcard asmjit/$(JIT_ARCH)/*.cpp) +ASMJIT_H = $(wildcard asmjit/*.h) $(wildcard asmjit/core/*.h) $(wildcard asmjit/$(JIT_ARCH)/*.h) ASMJIT_TTF_H = $(foreach HEADER,$(ASMJIT_H),$(TTF_DIR)/$(HEADER)) ifeq ($(FLAVOR),jit) @@ -776,6 +777,10 @@ COMMON_INCLUDES += -Ipcre COMMON_INCLUDES += -I../include -I../include/$(TARGET) COMMON_INCLUDES += -I../include/internal -I../include/internal/$(TARGET) +ifeq ($(FLAVOR),jit) +COMMON_INCLUDES += -Ibeam/jit -Ibeam/jit/$(JIT_ARCH) +endif + INCLUDES = -I$(TTF_DIR) $(COMMON_INCLUDES) ifeq ($(TARGET),win32) @@ -901,7 +906,7 @@ ASMJIT_FLAGS=-DASMJIT_EMBED=1 -DASMJIT_NO_BUILDER=1 -DASMJIT_NO_DEPRECATED=1 -DA ASMJIT_PCH_OBJ=$(TTF_DIR)/asmjit/asmjit.hpp.gch ASMJIT_PCH_SRC=$(TTF_DIR)/asmjit/asmjit.hpp -$(OBJDIR)/%.o: beam/jit/%.cpp beam/jit/beam_asm.hpp $(ASMJIT_PCH_OBJ) +$(OBJDIR)/%.o: beam/jit/$(JIT_ARCH)/%.cpp beam/jit/$(JIT_ARCH)/beam_asm.hpp $(ASMJIT_PCH_OBJ) $(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES) \ $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS)) \ -include $(ASMJIT_PCH_SRC) -c $< -o $@ @@ -987,7 +992,11 @@ COMMON_OBJS = \ ASMJIT_OBJS = $(patsubst %.cpp,$(OBJDIR)/%.o,$(ASMJIT_CPP)) JIT_OBJS = \ - $(OBJDIR)/asm_load.o \ + $(OBJDIR)/asm_load.o \ + $(OBJDIR)/beam_jit_common.o + +ifeq ($(JIT_ARCH), x86) +JIT_OBJS += \ $(OBJDIR)/beam_asm.o \ $(OBJDIR)/beam_asm_global.o \ $(OBJDIR)/beam_asm_module.o \ @@ -1002,9 +1011,10 @@ JIT_OBJS = \ $(OBJDIR)/instr_map.o \ $(OBJDIR)/instr_msg.o \ $(OBJDIR)/instr_select.o \ - $(OBJDIR)/instr_trace.o \ - $(ASMJIT_OBJS) + $(OBJDIR)/instr_trace.o +endif +JIT_OBJS += $(ASMJIT_OBJS) EMU_OBJS = \ $(OBJDIR)/emu_load.o @@ -1262,7 +1272,7 @@ BEAM_SRC=$(wildcard beam/*.c) ifeq ($(FLAVOR),emu) BEAM_SRC+=$(wildcard beam/emu/*.c) else -BEAM_CPP_SRC=$(wildcard beam/*.cpp) $(wildcard beam/jit/*.cpp) +BEAM_CPP_SRC=$(wildcard beam/*.cpp) $(wildcard beam/jit/*.cpp) $(wildcard beam/jit/$(JIT_ARCH)/*.cpp) endif DRV_COMMON_SRC=$(wildcard drivers/common/*.c) DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c) @@ -1315,7 +1325,7 @@ DEP_INCLUDES=$(INCLUDES) \ -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \ -Inifs/common -Inifs/$(ERLANG_OSTYPE) DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(DEP_INCLUDES) -DEP_CXXFLAGS=-MM $(MG_FLAG) $(CXXFLAGS) $(DEP_INCLUDES) +DEP_CXXFLAGS=-MM $(MG_FLAG) $(CXXFLAGS) $(DEP_INCLUDES) $(ASMJIT_FLAGS) SYS_SRC=$(ALL_SYS_SRC) endif @@ -1378,9 +1388,12 @@ compdb: cat $(TTF_DIR)/*.json > $(ERL_TOP)/compile_commands.json format-check: - clang-format --Werror --dry-run -i beam/jit/*.cpp beam/jit/*.hpp beam/jit/*.c beam/jit/*.h + clang-format --Werror --dry-run -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \ + beam/jit/*/*.cpp beam/jit/*/*.hpp + format: - clang-format -i beam/jit/*.cpp beam/jit/*.hpp beam/jit/*.c beam/jit/*.h + clang-format -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \ + beam/jit/*/*.cpp beam/jit/*/*.hpp ifneq ($(ERTS_SKIP_DEPEND),true) ifneq ($(MAKECMDGOALS),clean) diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index eab9d51b5e..7530bfa318 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -115,12 +115,27 @@ static int try_seize_cwp(Process* c_p, void (*aux_func)(void *), void *aux_arg); +#if defined(DEBUG) || defined(ADDRESS_SANITIZER) +# define CWP_DBG_FORCE_TRAP +#endif + /* * Calller _must_ yield if we return 0 */ int erts_try_seize_code_write_permission(Process* c_p) { ASSERT(c_p != NULL); + +#ifdef CWP_DBG_FORCE_TRAP + if (!(c_p->flags & F_DBG_FORCED_TRAP)) { + c_p->flags |= F_DBG_FORCED_TRAP; + return 0; + } else { + /* back from forced trap */ + c_p->flags &= ~F_DBG_FORCED_TRAP; + } +#endif + return try_seize_cwp(c_p, NULL, NULL); } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 671b56f6c8..6bb9a46fc3 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -822,16 +822,16 @@ DbTable* db_get_table_aux(Process *p, * can handle a failed TRAP return correctly. */ if (what != DB_READ_TBL_STRUCT && tb->common.dbg_force_trap) { - if (!(p->flags & F_ETS_FORCED_TRAP)) { + if (!(p->flags & F_DBG_FORCED_TRAP)) { db_unlock(tb, kind); tb = NULL; *freason_p = TRAP; p->fvalue = EXI_TYPE; - p->flags |= F_ETS_FORCED_TRAP; + p->flags |= F_DBG_FORCED_TRAP; return tb; } else { /* back from forced trap */ - p->flags &= ~F_ETS_FORCED_TRAP; + p->flags &= ~F_DBG_FORCED_TRAP; } } #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 7cdb532f75..28b0e80e02 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -3477,19 +3477,21 @@ reached_max_heap_size(Process *p, Uint total_heap_size, Eterm *o_hp, *hp, args = NIL; /* Build the format message */ - erts_dsprintf(dsbufp, " Process: ~p "); + erts_dsprintf(dsbufp, " Process: ~p "); if (alive) erts_dsprintf(dsbufp, "on node ~p"); - erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); - erts_dsprintf(dsbufp, " Max Heap Size: ~p~n"); - erts_dsprintf(dsbufp, " Total Heap Size: ~p~n"); - erts_dsprintf(dsbufp, " Kill: ~p~n"); - erts_dsprintf(dsbufp, " Error Logger: ~p~n"); - erts_dsprintf(dsbufp, " GC Info: ~p~n"); + erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); + erts_dsprintf(dsbufp, " Max Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Total Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Kill: ~p~n"); + erts_dsprintf(dsbufp, " Error Logger: ~p~n"); + erts_dsprintf(dsbufp, " Message Queue Len: ~p~n"); + erts_dsprintf(dsbufp, " GC Info: ~p~n"); /* Build the args in reverse order */ - o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm)); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 8 : 7) * sizeof(Eterm)); args = CONS(hp, msg, args); hp += 2; + args = CONS(hp, make_small((p)->sig_inq.len), args); hp += 2; args = CONS(hp, am_true, args); hp += 2; args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2; args = CONS(hp, make_small(total_heap_size), args); hp += 2; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index c61094c95b..6ea1759cf8 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -205,7 +205,7 @@ int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ int erts_no_line_info = 0; /* -L: Don't load line information */ #ifdef BEAMASM -int erts_asm_dump = 0; /* -asmdump: Dump assembly code */ +int erts_jit_asm_dump = 0; /* -JDdump: Dump assembly code */ #endif /* @@ -1629,6 +1629,20 @@ erl_start(int argc, char **argv) switch (sub_param[0]) { + case 'D': + sub_param++; + if (has_prefix("dump", sub_param)) { + arg = get_arg(sub_param+4, argv[i + 1], &i); + if (sys_strcmp(arg, "true") == 0) { + erts_jit_asm_dump = 1; + } else if (sys_strcmp(arg, "false") == 0) { + erts_jit_asm_dump = 0; + } else { + erts_fprintf(stderr, "bad +JDdump flag %s\n", arg); + erts_usage(); + } + } + break; case 'P': sub_param++; @@ -1655,7 +1669,7 @@ erl_start(int argc, char **argv) } break; default: - erts_fprintf(stderr, "invalid JIT option %s\n", arg); + erts_fprintf(stderr, "invalid JIT option %s\n", argv[i]); erts_usage(); break; } @@ -2116,12 +2130,6 @@ erl_start(int argc, char **argv) break; case 'a': -#ifdef BEAMASM - if (strcmp(argv[i]+2, "smdump") == 0) { - erts_asm_dump = 1; - break; - } -#endif /* suggested stack size (Kilo Words) for threads in thread pool */ arg = get_arg(argv[i]+2, argv[i+1], &i); erts_async_thread_suggested_stack_size = atoi(arg); diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 5d19ee8ead..8b8b3fdbe3 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -6258,6 +6258,22 @@ handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing, next_sig = tracing->messages.next; sig = *next_sig; + if (!sig) { + ASSERT(!*next_nm_sig); + return 1; /* end... */ + } + + if (ERTS_SIG_IS_RECV_MARKER(sig) && ((ErtsRecvMarker *) sig)->in_msgq) { + /* + * Skip already handled receive marker that just entered + * the message queue... + */ + next_sig = &sig->next; + sig = *next_sig; + ASSERT(!ERTS_SIG_IS_RECV_MARKER(sig) + || !((ErtsRecvMarker *) sig)->in_msgq); + } + /* * Receive tracing active. Handle all messages * until next non-message signal... diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 1ba83f8aee..dfd3549391 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -661,7 +661,7 @@ typedef struct ErtsSchedulerRegisters_ { ErtsCodePtr start_time_i; UWord start_time; -#if !defined(NATIVE_ERLANG_STACK) && defined(HARD_DEBUG) +#if !defined(NATIVE_ERLANG_STACK) && defined(JIT_HARD_DEBUG) /* Holds the initial thread stack pointer. Used to ensure that everything * that is pushed to the stack is also popped. */ UWord *initial_sp; @@ -1541,7 +1541,7 @@ extern int erts_system_profile_ts_type; #define F_DIRTY_MINOR_GC (1 << 20) /* Dirty minor GC scheduled */ #define F_HIBERNATED (1 << 21) /* Hibernated */ #define F_TRAP_EXIT (1 << 22) /* Trapping exit */ -#define F_ETS_FORCED_TRAP (1 << 23) /* DEBUG: Last ETS call was a forced trap */ +#define F_DBG_FORCED_TRAP (1 << 23) /* DEBUG: Last BIF call was a forced trap */ /* Signal queue flags */ #define FS_OFF_HEAP_MSGQ (1 << 0) /* Off heap msg queue */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2051377398..956d1dc468 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1263,7 +1263,7 @@ extern int erts_initialized; extern int erts_compat_rel; #ifdef BEAMASM -extern int erts_asm_dump; +extern int erts_jit_asm_dump; #endif void erl_start(int, char**); diff --git a/erts/emulator/beam/jit/beam_jit_common.c b/erts/emulator/beam/jit/beam_jit_common.c new file mode 100644 index 0000000000..9b56fb1ac1 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.c @@ -0,0 +1,756 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "bif.h" +#include "code_ix.h" +#include "erl_proc_sig_queue.h" +#include "erl_binary.h" +#include "erl_bits.h" +#include "erl_map.h" +#include "beam_common.h" +#ifdef USE_VM_PROBES +# include "dtrace-wrapper.h" +#endif + +#include "beam_jit_common.h" + +#if defined(DEBUG) && defined(JIT_HARD_DEBUG) +void beam_jit_validate_term(Eterm term) { + if (is_boxed(term)) { + Eterm header = *boxed_val(term); + + if (header_is_bin_matchstate(header)) { + return; + } + } + + size_object_x(term, NULL); +} +#endif + +Eterm beam_jit_call_bif(Process *c_p, + Eterm *reg, + ErtsCodePtr I, + ErtsBifFunc vbf, + Uint arity) { + ErlHeapFragment *live_hf_end; + Eterm result; + + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + { + live_hf_end = c_p->mbuf; + + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + result = vbf(c_p, reg, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_CHK_MBUF_SZ(c_p); + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_HOLE_CHECK(c_p); + } + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + + if (ERTS_IS_GC_DESIRED(c_p)) { + result = erts_gc_after_bif_call_lhf(c_p, + live_hf_end, + result, + reg, + arity); + } + + return result; +} + +Eterm beam_jit_call_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg, + BeamJitNifF *fp, + struct erl_module_nif *NifMod) { + Eterm nif_bif_result; + Eterm bif_nif_arity; + ErlHeapFragment *live_hf_end; + const ErtsCodeMFA *codemfa; + + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + bif_nif_arity = codemfa->arity; + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + + { + struct enif_environment_t env; + ASSERT(c_p->scheduler_data); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + erts_pre_nif(&env, c_p, NifMod, NULL); + + ASSERT((c_p->scheduler_data)->current_nif == NULL); + (c_p->scheduler_data)->current_nif = &env; + + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; + + ASSERT((c_p->scheduler_data)->current_nif == &env); + (c_p->scheduler_data)->current_nif = NULL; + + erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ASSERT(!env.exiting); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + ERTS_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, + live_hf_end, + nif_bif_result, + reg, + bif_nif_arity); + } + + return nif_bif_result; +} + +enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg) { + if (erts_try_seize_code_write_permission(c_p)) { + Eterm result; + + PROCESS_MAIN_CHK_LOCKS((c_p)); + ERTS_UNREQ_PROC_MAIN_LOCK((c_p)); + result = erts_load_nif(c_p, I, reg[0], reg[1]); + erts_release_code_write_permission(); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + + if (ERTS_LIKELY(is_value(result))) { + reg[0] = result; + return RET_NIF_success; + } else { + c_p->freason = BADARG; + return RET_NIF_error; + } + } else { + /* Yield and try again. */ + c_p->current = NULL; + c_p->arity = 2; + return RET_NIF_yield; + } +} + +Uint beam_jit_get_map_elements(Eterm map, + Eterm *reg, + Eterm *E, + Uint n, + Eterm *fs) { + Uint sz; + + /* This instruction assumes Arg1 is a map, i.e. that it follows a test + * is_map if needed. */ + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; + + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); + + if (sz == 0) { + return 0; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while (sz) { + if (EQ(fs[0], *ks)) { + PUT_TERM_REG(*vs, fs[1]); + + n--; + fs += 3; + + /* no more values to fetch, we are done */ + if (n == 0) { + return 1; + } + } + + ks++, sz--, vs++; + } + return 0; + } else { + ASSERT(is_hashmap(map)); + + while (n--) { + const Eterm *v; + Uint32 hx; + + hx = fs[2]; + ASSERT(hx == hashmap_make_hash(fs[0])); + + if ((v = erts_hashmap_get(hx, fs[0], map)) == NULL) { + return 0; + } + + PUT_TERM_REG(*v, fs[1]); + fs += 3; + } + + return 1; + } +} + +static void test_bin_vheap(Process *c_p, + Eterm *reg, + Uint VNh, + Uint Nh, + Uint Live) { + int need = Nh; + + if (c_p->stop - c_p->htop < (need + S_RESERVED) || + MSO(c_p).overhead + VNh >= BIN_VHEAP_SZ(c_p)) { + c_p->fcalls -= + erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); + } +} + +static void gc_test(Process *c_p, Eterm *reg, Uint Ns, Uint Nh, Uint Live) { + int need = Nh + Ns; + + if (ERTS_UNLIKELY(c_p->stop - c_p->htop < (need + S_RESERVED))) { + c_p->fcalls -= + erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); + } +} + +void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size) { + if (((is_small(size) && signed_val(size) >= 0) || + (is_big(size) && !big_sign(size)))) { + /* If the argument is a positive integer, we must've had a system_limit + * error. */ + c_p->freason = SYSTEM_LIMIT; + } else { + c_p->freason = BADARG; + } +} + +/* Set the exception code for bs_add argument errors after the fact, which is + * much easier and more compact than discriminating within module code. */ +void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B) { + if (((is_small(A) && signed_val(A) >= 0) || (is_big(A) && !big_sign(A))) && + ((is_small(B) && signed_val(B) >= 0) || (is_big(B) && !big_sign(B)))) { + /* If all arguments are positive integers, we must've had a system_limit + * error. */ + c_p->freason = SYSTEM_LIMIT; + } else { + c_p->freason = BADARG; + } +} + +static Eterm i_bs_start_match2_gc_test_preserve(Process *c_p, + Eterm *reg, + Uint need, + Uint live, + Eterm preserve) { + Uint words_left = (Uint)(STACK_TOP(c_p) - HEAP_TOP(c_p)); + + if (ERTS_UNLIKELY(words_left < need + S_RESERVED)) { + reg[live] = preserve; + PROCESS_MAIN_CHK_LOCKS(c_p); + c_p->fcalls -= erts_garbage_collect_nobump(c_p, + need, + reg, + live + 1, + c_p->fcalls); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + preserve = reg[live]; + } + + return preserve; +} + +Eterm beam_jit_bs_start_match2(Eterm context, + Uint live, + Uint slots, + Process *c_p, + Eterm *reg) { + Eterm header; + if (!is_boxed(context)) { + return THE_NON_VALUE; + } + header = *boxed_val(context); + + slots++; + + if (header_is_bin_matchstate(header)) { + ErlBinMatchState *ms = (ErlBinMatchState *)boxed_val(context); + Uint actual_slots = HEADER_NUM_SLOTS(header); + + /* We're not compatible with contexts created by bs_start_match3. */ + ASSERT(actual_slots >= 1); + + ms->save_offset[0] = ms->mb.offset; + if (ERTS_UNLIKELY(actual_slots < slots)) { + ErlBinMatchState *expanded; + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + context = i_bs_start_match2_gc_test_preserve(c_p, + reg, + wordsneeded, + live, + context); + ms = (ErlBinMatchState *)boxed_val(context); + expanded = (ErlBinMatchState *)HEAP_TOP(c_p); + *expanded = *ms; + *HEAP_TOP(c_p) = HEADER_BIN_MATCHSTATE(slots); + HEAP_TOP(c_p) += wordsneeded; + context = make_matchstate(expanded); + } + return context; + } else if (is_binary_header(header)) { + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + context = i_bs_start_match2_gc_test_preserve(c_p, + reg, + wordsneeded, + live, + context); + return erts_bs_start_match_2(c_p, context, slots); + } else { + return THE_NON_VALUE; + } +} + +Eterm beam_jit_bs_init(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Eterm BsOp1, + Eterm BsOp2, + unsigned Live) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin *hb; + Uint bin_need; + + bin_need = heap_bin_size(BsOp1); + erts_bin_offset = 0; + erts_writable_bin = 0; + gc_test(c_p, reg, 0, bin_need + BsOp2 + ERL_SUB_BIN_SIZE, Live); + hb = (ErlHeapBin *)c_p->htop; + c_p->htop += bin_need; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; + erts_current_bin = (byte *)hb->data; + return make_binary(hb); + } else { + Binary *bptr; + ProcBin *pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + test_bin_vheap(c_p, + reg, + BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, + Live); + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(BsOp1); + erts_current_bin = (byte *)bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *)c_p->htop; + c_p->htop += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = BsOp1; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header *)pb; + pb->val = bptr; + pb->bytes = (byte *)bptr->orig_bytes; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); + + return make_binary(pb); + } +} + +Eterm beam_jit_bs_init_bits(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Uint num_bits, + Uint alloc, + unsigned Live) { + Eterm new_binary; + Uint num_bytes = ((Uint64)num_bits + (Uint64)7) >> 3; + + if (num_bits & 7) { + alloc += ERL_SUB_BIN_SIZE; + } + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + alloc += heap_bin_size(num_bytes); + } else { + alloc += PROC_BIN_SIZE; + } + gc_test(c_p, reg, 0, alloc, Live); + + /* num_bits = Number of bits to build + * num_bytes = Number of bytes to allocate in the binary + * alloc = Total number of words to allocate on heap + * Operands: NotUsed NotUsed Dst + */ + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin *hb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + hb = (ErlHeapBin *)c_p->htop; + c_p->htop += heap_bin_size(num_bytes); + hb->thing_word = header_heap_bin(num_bytes); + hb->size = num_bytes; + erts_current_bin = (byte *)hb->data; + new_binary = make_binary(hb); + + do_bits_sub_bin: + if (num_bits & 7) { + ErlSubBin *sb; + + sb = (ErlSubBin *)c_p->htop; + c_p->htop += ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = num_bytes - 1; + sb->bitsize = num_bits & 7; + sb->offs = 0; + sb->bitoffs = 0; + sb->is_writable = 0; + sb->orig = new_binary; + new_binary = make_binary(sb); + } + /* HEAP_SPACE_VERIFIED(0); */ + return new_binary; + } else { + Binary *bptr; + ProcBin *pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(num_bytes); + erts_current_bin = (byte *)bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *)c_p->htop; + c_p->htop += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = num_bytes; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header *)pb; + pb->val = bptr; + pb->bytes = (byte *)bptr->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); + new_binary = make_binary(pb); + goto do_bits_sub_bin; + } +} + +Eterm beam_jit_bs_get_integer(Process *c_p, + Eterm *reg, + Eterm context, + Uint flags, + Uint size, + Uint Live) { + ErlBinMatchBuffer *mb; + + if (size >= SMALL_BITS) { + Uint wordsneeded; + + /* Check bits size before potential gc. + * We do not want a gc and then realize we don't need + * the allocated space (i.e. if the op fails). + * + * Remember to re-acquire the matchbuffer after gc. + */ + mb = ms_matchbuffer(context); + if (mb->size - mb->offset < size) { + return THE_NON_VALUE; + } + + wordsneeded = 1 + WSIZE(NBYTES((Uint)size)); + reg[Live] = context; + gc_test(c_p, reg, 0, wordsneeded, Live + 1); + context = reg[Live]; + } + + mb = ms_matchbuffer(context); + return erts_bs_get_integer_2(c_p, size, flags, mb); +} + +void beam_jit_bs_context_to_binary(Eterm context) { + if (is_boxed(context) && header_is_bin_matchstate(*boxed_val(context))) { + Uint orig, size, offs, hole_size; + ErlBinMatchBuffer *mb; + ErlBinMatchState *ms; + ErlSubBin *sb; + ms = (ErlBinMatchState *)boxed_val(context); + mb = &ms->mb; + offs = ms->save_offset[0]; + size = mb->size - offs; + orig = mb->orig; + sb = (ErlSubBin *)boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); + hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = BYTE_OFFSET(size); + sb->bitsize = BIT_OFFSET(size); + sb->offs = BYTE_OFFSET(offs); + sb->bitoffs = BIT_OFFSET(offs); + sb->is_writable = 0; + sb->orig = orig; + if (hole_size) { + sb[1].thing_word = make_pos_bignum_header(hole_size - 1); + } + } +} + +ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp) { + if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + + /* TODO: Add DTrace probe for this bad message situation? */ + erts_msgq_unlink_msg(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + + return NULL; + } + + return msgp; +} + +/* Remove a (matched) message from the message queue. */ +Sint beam_jit_remove_message(Process *c_p, + Sint FCALLS, + Eterm *HTOP, + Eterm *E, + Uint32 active_code_ix) { + ErtsMessage *msgp; + + ERTS_CHK_MBUF_SZ(c_p); + + if (active_code_ix == ERTS_SAVE_CALLS_CODE_IX) { + save_calls(c_p, &exp_receive); + } + + msgp = erts_msgq_peek_msg(c_p); + + if (ERL_MESSAGE_TOKEN(msgp) == NIL) { +#ifdef USE_VM_PROBES + if (DT_UTAG(c_p) != NIL) { + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { + SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; + } else { + DT_UTAG(c_p) = NIL; + SEQ_TRACE_TOKEN(c_p) = NIL; + } + } else { +#endif + SEQ_TRACE_TOKEN(c_p) = NIL; +#ifdef USE_VM_PROBES + } + DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; +#endif + } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { + Eterm msg; + SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); +#ifdef USE_VM_PROBES + if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { + if (DT_UTAG(c_p) == NIL) { + DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); + } + DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; + } else { +#endif + ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); + ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); + ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); + ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)) || + is_atom(SEQ_TRACE_TOKEN_SENDER(c_p))); + c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + if (c_p->seq_trace_clock < + unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { + c_p->seq_trace_clock = + unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + } + msg = ERL_MESSAGE_TERM(msgp); + seq_trace_output(SEQ_TRACE_TOKEN(c_p), + msg, + SEQ_TRACE_RECEIVE, + c_p->common.id, + c_p); +#ifdef USE_VM_PROBES + } +#endif + } +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_receive)) { + Eterm token2 = NIL; + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Sint len = erts_proc_sig_privqs_len(c_p); + + dtrace_proc_str(c_p, receiver_name); + token2 = SEQ_TRACE_TOKEN(c_p); + if (have_seqtrace(token2)) { + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); + } + DTRACE6(message_receive, + receiver_name, + size_object(ERL_MESSAGE_TERM(msgp)), + len, /* This is NOT message queue len, but its something... */ + tok_label, + tok_lastcnt, + tok_serial); + } +#endif + erts_msgq_unlink_msg(c_p, msgp); + erts_msgq_set_save_first(c_p); + CANCEL_TIMER(c_p); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } + + ERTS_CHK_MBUF_SZ(c_p); + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + return FCALLS; +} + +void beam_jit_take_receive_lock(Process *c_p) { + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); +} + +void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp) { + c_p->arity = 0; + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) { + erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->current = NULL; + c_p->i = cp; +} + +void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp) { + beam_jit_take_receive_lock(c_p); + beam_jit_wait_locked(c_p, cp); +} + +enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p, + Eterm timeout_value, + ErtsCodePtr next) { + /* + * If we have already set the timer, we must NOT set it again. Therefore, + * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. + */ + if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { + if (timeout_value == make_small(0)) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + return RET_next; + } else if (timeout_value == am_infinity) { + c_p->flags |= F_TIMO; + } else { + int tres = erts_set_proc_timer_term(c_p, timeout_value); + if (tres == 0) { + /* + * The timer routiner will set c_p->i to the value in + * c_p->def_arg_reg[0]. Note that it is safe to use this + * location because there are no living x registers in + * a receive statement. + */ + c_p->def_arg_reg[0] = (Eterm)next; + } else { /* Wrong time */ + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->freason = EXC_TIMEOUT_VALUE; + return RET_badarg; + } + } + } + return RET_wait; +} + +void beam_jit_timeout(Process *c_p) { + if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { + trace_receive(c_p, am_clock_service, am_timeout, NULL); + } + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + save_calls(c_p, &exp_timeout); + } + c_p->flags &= ~F_TIMO; + erts_msgq_set_save_first(c_p); +} + +void beam_jit_timeout_locked(Process *c_p) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + beam_jit_timeout(c_p); +} diff --git a/erts/emulator/beam/jit/beam_jit_common.h b/erts/emulator/beam/jit/beam_jit_common.h new file mode 100644 index 0000000000..b2da85d0b8 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.h @@ -0,0 +1,101 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" + +#if defined(DEBUG) && defined(JIT_HARD_DEBUG) +void beam_jit_validate_term(Eterm term); +#endif + +typedef Eterm BeamJitNifF(struct enif_environment_t *, int argc, Eterm argv[]); +enum beam_jit_nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield }; + +Eterm beam_jit_call_bif(Process *c_p, + Eterm *reg, + ErtsCodePtr I, + ErtsBifFunc vbf, + Uint arity); +Eterm beam_jit_call_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg, + BeamJitNifF *fp, + struct erl_module_nif *NifMod); +enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p, + ErtsCodePtr I, + Eterm *reg); + +Uint beam_jit_get_map_elements(Eterm map, + Eterm *reg, + Eterm *E, + Uint n, + Eterm *fs); + +void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size); +void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B); +Eterm beam_jit_bs_start_match2(Eterm context, + Uint live, + Uint slots, + Process *c_p, + Eterm *reg); +Eterm beam_jit_bs_init(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Eterm BsOp1, + Eterm BsOp2, + unsigned Live); +Eterm beam_jit_bs_init_bits(Process *c_p, + Eterm *reg, + ERL_BITS_DECLARE_STATEP, + Uint num_bits, + Uint alloc, + unsigned Live); +Eterm beam_jit_bs_get_integer(Process *c_p, + Eterm *reg, + Eterm context, + Uint flags, + Uint size, + Uint Live); + +ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp); +Sint beam_jit_remove_message(Process *c_p, + Sint FCALLS, + Eterm *HTOP, + Eterm *E, + Uint32 active_code_ix); +void beam_jit_bs_context_to_binary(Eterm context); + +void beam_jit_take_receive_lock(Process *c_p); +void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp); +void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp); + +enum beam_jit_tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 }; + +enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p, + Eterm timeout_value, + ErtsCodePtr next); + +void beam_jit_timeout(Process *c_p); +void beam_jit_timeout_locked(Process *c_p); diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp new file mode 100644 index 0000000000..f8f894c676 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_common.hpp @@ -0,0 +1,137 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include <string> +#include <vector> +#include <unordered_map> +#include <map> + +extern "C" +{ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" + +#include "beam_jit_common.h" +} + +class ArgVal { + BeamOpArg gen_op; + +public: + enum TYPE { + u = TAG_u, + i = TAG_i, + x = TAG_x, + y = TAG_y, + f = TAG_f, + q = TAG_q, + e = TAG_r, + l = TAG_l /* float register */ + }; + + ArgVal(const BeamOpArg &arg) { + gen_op = arg; + } + + ArgVal(enum TYPE t, BeamInstr val) { + gen_op.type = t; + gen_op.val = val; + } + + ArgVal(unsigned t, BeamInstr val) { +#ifdef DEBUG + switch (t) { + case TAG_u: + break; + case TAG_i: + break; + case TAG_x: + break; + case TAG_y: + break; + case TAG_f: + break; + case TAG_q: + break; + case TAG_r: + break; + case TAG_l: + break; + default: + ASSERT(0); + } +#endif + + gen_op.type = t; + gen_op.val = val; + } + + constexpr enum TYPE getType() const { + return (enum TYPE)gen_op.type; + } + + constexpr uint64_t getValue() const { + return gen_op.val; + } + + constexpr bool isMem() const { + return gen_op.type == x || gen_op.type == y; + } + + constexpr bool isLiteral() const { + return gen_op.type == q; + } + + constexpr bool isImmed() const { + return gen_op.type == i; + } + + template<typename T> + ArgVal operator+(T val) const { + return ArgVal(gen_op.type, val + gen_op.val); + } + + template<typename T> + ArgVal operator*(T val) const { + return ArgVal(gen_op.type, val * gen_op.val); + } + + enum Relation { none, consecutive, reverse_consecutive }; + + static Relation register_relation(const ArgVal &arg1, const ArgVal &arg2) { + TYPE type = arg1.getType(); + bool same_reg_types = + type == arg2.getType() && (type == TYPE::x || type == TYPE::y); + if (!same_reg_types) { + return none; + } else if (arg1.getValue() + 1 == arg2.getValue()) { + return consecutive; + } else if (arg1.getValue() == arg2.getValue() + 1) { + return reverse_consecutive; + } else { + return none; + } + }; +}; diff --git a/erts/emulator/beam/jit/beam_asm.cpp b/erts/emulator/beam/jit/x86/beam_asm.cpp index c4a7ff0c61..4368cc9479 100644 --- a/erts/emulator/beam/jit/beam_asm.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm.cpp @@ -355,12 +355,9 @@ void BeamGlobalAssembler::emit_process_main() { x86::qword_ptr(x86::rsp, offsetof(ErtsSchedulerRegisters, x_reg_array.d))); - load_erl_bits_state(ARG1); - runtime_call<1>(erts_bits_init_state); - #if defined(DEBUG) && defined(NATIVE_ERLANG_STACK) /* Save stack bounds so they can be tested without clobbering anything. */ - runtime_call<0>(erts_get_stacklimit); + a.call(erts_get_stacklimit); a.mov(getSchedulerRegRef( offsetof(ErtsSchedulerRegisters, runtime_stack_end)), @@ -371,7 +368,7 @@ void BeamGlobalAssembler::emit_process_main() { #elif !defined(NATIVE_ERLANG_STACK) /* Save the initial SP of the thread so that we can verify that it * doesn't grow. */ -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG a.mov(getInitialSPRef(), x86::rsp); # endif @@ -386,6 +383,9 @@ void BeamGlobalAssembler::emit_process_main() { a.and_(x86::rsp, imm(-16)); #endif + load_erl_bits_state(ARG1); + runtime_call<1>(erts_bits_init_state); + a.mov(start_time_i, imm(0)); a.mov(start_time, imm(0)); diff --git a/erts/emulator/beam/jit/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp index f97aaee840..9e1254fec1 100644 --- a/erts/emulator/beam/jit/beam_asm.hpp +++ b/erts/emulator/beam/jit/x86/beam_asm.hpp @@ -41,105 +41,7 @@ extern "C" #include "beam_asm.h" } -class ArgVal { - BeamOpArg gen_op; - -public: - enum TYPE { - u = TAG_u, - i = TAG_i, - x = TAG_x, - y = TAG_y, - f = TAG_f, - q = TAG_q, - e = TAG_r, - l = TAG_l /* float register */ - }; - - ArgVal(const BeamOpArg &arg) { - gen_op = arg; - } - - ArgVal(enum TYPE t, BeamInstr val) { - gen_op.type = t; - gen_op.val = val; - } - - ArgVal(unsigned t, BeamInstr val) { -#ifdef DEBUG - switch (t) { - case TAG_u: - break; - case TAG_i: - break; - case TAG_x: - break; - case TAG_y: - break; - case TAG_f: - break; - case TAG_q: - break; - case TAG_r: - break; - case TAG_l: - break; - default: - ASSERT(0); - } -#endif - - gen_op.type = t; - gen_op.val = val; - } - - constexpr enum TYPE getType() const { - return (enum TYPE)gen_op.type; - } - - constexpr uint64_t getValue() const { - return gen_op.val; - } - - constexpr bool isMem() const { - return gen_op.type == x || gen_op.type == y; - } - - constexpr bool isLiteral() const { - return gen_op.type == q; - } - - constexpr bool isImmed() const { - return gen_op.type == i; - } - - template<typename T> - ArgVal operator+(T val) const { - return ArgVal(gen_op.type, val + gen_op.val); - } - - template<typename T> - ArgVal operator*(T val) const { - return ArgVal(gen_op.type, val * gen_op.val); - } - - enum Relation { none, consecutive, reverse_consecutive }; - - static Relation register_relation(const ArgVal &arg1, const ArgVal &arg2) { - TYPE type = arg1.getType(); - bool same_reg_types = - type == arg2.getType() && (type == TYPE::x || type == TYPE::y); - if (!same_reg_types) { - return none; - } else if (arg1.getValue() + 1 == arg2.getValue()) { - return consecutive; - } else if (arg1.getValue() == arg2.getValue() + 1) { - return reverse_consecutive; - } else { - return none; - } - }; -}; +#include "beam_jit_common.hpp" using namespace asmjit; @@ -285,7 +187,7 @@ public: } BeamAssembler(const std::string &log) : BeamAssembler() { - if (erts_asm_dump) { + if (erts_jit_asm_dump) { setLogger(log + ".asm"); } } @@ -366,9 +268,9 @@ protected: } #if !defined(NATIVE_ERLANG_STACK) -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG constexpr x86::Mem getInitialSPRef() const { - int base = offsetof(ErtsSchedulerRegisters, aux_regs.d.initial_sp); + int base = offsetof(ErtsSchedulerRegisters, initial_sp); return getSchedulerRegRef(base); } @@ -446,21 +348,27 @@ protected: a.lea(reg, getSchedulerRegRef(offset)); } + /* Ensure that the Erlang stack is used and the redzone is unused. + * We combine those test to minimize the number of instructions. + */ void emit_assert_redzone_unused() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG const int REDZONE_BYTES = S_REDZONE * sizeof(Eterm); - Label next = a.newLabel(); + Label ok = a.newLabel(), crash = a.newLabel(); /* We modify the stack pointer to avoid spilling into a register, * TMP_MEM, or using the stack. */ a.sub(E, imm(REDZONE_BYTES)); a.cmp(HTOP, E); - a.add(E, imm(REDZONE_BYTES)); + a.short_().ja(crash); + a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, hend))); + a.short_().jle(ok); - a.jbe(next); + a.bind(crash); a.ud2(); - a.bind(next); + a.bind(ok); + a.add(E, imm(REDZONE_BYTES)); #endif } @@ -472,7 +380,6 @@ protected: #ifdef NATIVE_ERLANG_STACK /* We use the Erlang stack as the native stack. We can use a * native `call` instruction. */ - emit_assert_erlang_stack(); emit_assert_redzone_unused(); aligned_call(Target); #else @@ -498,10 +405,9 @@ protected: */ template<typename Any> void fragment_call(Any Target) { - emit_assert_erlang_stack(); emit_assert_redzone_unused(); -#if defined(HARD_DEBUG) && !defined(NATIVE_ERLANG_STACK) +#if defined(JIT_HARD_DEBUG) && !defined(NATIVE_ERLANG_STACK) /* Verify that the stack has not grown. */ Label next = a.newLabel(); a.cmp(x86::rsp, getInitialSPRef()); @@ -515,7 +421,7 @@ protected: /* * Calls the given function pointer. In a debug build with - * HARD_DEBUG defined, it will be enforced that the redzone is + * JIT_HARD_DEBUG defined, it will be enforced that the redzone is * unused. * * The return will NOT be aligned, and thus will not form a valid @@ -525,7 +431,6 @@ protected: * switch. */ void safe_fragment_call(void (*Target)()) { - emit_assert_erlang_stack(); emit_assert_redzone_unused(); a.call(imm(Target)); } @@ -574,7 +479,7 @@ protected: a.embed(nops[nop_count - 1], nop_count); } -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG /* TODO: When frame pointers are in place, assert (at runtime) that the * destination has a `push rbp; mov rbp, rsp` sequence. */ #endif @@ -697,13 +602,9 @@ protected: } void emit_assert_runtime_stack() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG Label crash = a.newLabel(), next = a.newLabel(); - /* Are we 16-byte aligned? */ - a.test(E, (16 - 1)); - a.jne(crash); - # ifdef NATIVE_ERLANG_STACK /* Ensure that we are using the runtime stack. */ int end_offs, start_offs; @@ -712,32 +613,37 @@ protected: start_offs = offsetof(ErtsSchedulerRegisters, runtime_stack_start); a.cmp(E, getSchedulerRegRef(end_offs)); - a.short_().jl(crash); + a.short_().jbe(crash); a.cmp(E, getSchedulerRegRef(start_offs)); - a.short_().jle(next); - + a.short_().ja(crash); # endif + + /* Are we 16-byte aligned? */ + a.test(x86::rsp, (16 - 1)); + a.short_().je(next); + a.bind(crash); a.ud2(); + a.bind(next); #endif } void emit_assert_erlang_stack() { -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG Label crash = a.newLabel(), next = a.newLabel(); /* Are we term-aligned? */ a.test(E, imm(sizeof(Eterm) - 1)); - a.jne(crash); + a.short_().jne(crash); a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, heap))); - a.jl(crash); + a.short_().jl(crash); a.cmp(E, x86::qword_ptr(c_p, offsetof(Process, hend))); - a.jle(next); + a.short_().jle(next); a.bind(crash); - a.hlt(); + a.ud2(); a.bind(next); #endif } diff --git a/erts/emulator/beam/jit/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp index 1c6479186a..1c6479186a 100644 --- a/erts/emulator/beam/jit/beam_asm_global.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp diff --git a/erts/emulator/beam/jit/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index 0da710d878..0da710d878 100644 --- a/erts/emulator/beam/jit/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp diff --git a/erts/emulator/beam/jit/beam_asm_perf.cpp b/erts/emulator/beam/jit/x86/beam_asm_perf.cpp index 79c0977339..d0beec7eb3 100644 --- a/erts/emulator/beam/jit/beam_asm_perf.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_perf.cpp @@ -141,6 +141,7 @@ public: for (BeamAssembler::AsmRange &r : ranges) { size_t nameLen = r.name.size(); ptrdiff_t codeSize = (char *)r.stop - (char *)r.start; + ASSERT(codeSize > 0); record.header.total_size = sizeof(record) + nameLen + 1 + codeSize; record.vma = (Uint64)r.start; record.code_addr = (Uint64)r.start; @@ -184,6 +185,7 @@ public: ptrdiff_t size = stop - start; fprintf(file, "%p %tx $%s\n", start, size, r.name.c_str()); } + fflush(file); } }; diff --git a/erts/emulator/beam/jit/generators.tab b/erts/emulator/beam/jit/x86/generators.tab index ab755fde4c..ab755fde4c 100644 --- a/erts/emulator/beam/jit/generators.tab +++ b/erts/emulator/beam/jit/x86/generators.tab diff --git a/erts/emulator/beam/jit/instr_arith.cpp b/erts/emulator/beam/jit/x86/instr_arith.cpp index 5615230551..5615230551 100644 --- a/erts/emulator/beam/jit/instr_arith.cpp +++ b/erts/emulator/beam/jit/x86/instr_arith.cpp diff --git a/erts/emulator/beam/jit/instr_bif.cpp b/erts/emulator/beam/jit/x86/instr_bif.cpp index 456b837099..7d130595ba 100644 --- a/erts/emulator/beam/jit/instr_bif.cpp +++ b/erts/emulator/beam/jit/x86/instr_bif.cpp @@ -439,6 +439,32 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { runtime_call(RET, 3); #endif +#ifdef ERTS_MSACC_EXTENDED_STATES + { + Label skip_msacc = a.newLabel(); + + a.cmp(erts_msacc_cache, imm(0)); + a.short_().je(skip_msacc); + + /* update cache if it was changed in the bif */ + a.mov(TMP_MEM1q, RET); + a.lea(ARG1, erts_msacc_cache); + runtime_call<1>(erts_msacc_update_cache); + a.mov(RET, TMP_MEM1q); + + /* set state to emulator if msacc has been enabled */ + a.cmp(erts_msacc_cache, imm(0)); + a.short_().je(skip_msacc); + a.mov(ARG1, erts_msacc_cache); + a.mov(ARG2, imm(ERTS_MSACC_STATE_EMULATOR)); + a.mov(ARG3, imm(1)); + runtime_call<3>(erts_msacc_set_state_m__); + a.mov(RET, TMP_MEM1q); + + a.bind(skip_msacc); + } +#endif + /* ERTS_IS_GC_DESIRED_INTERNAL */ { a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, stop))); @@ -474,7 +500,8 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { a.bind(check_bif_return); emit_test_the_non_value(ARG3); - a.short_().je(trap); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.je(trap); a.mov(HTOP, ARG5); #ifdef NATIVE_ERLANG_STACK @@ -639,99 +666,6 @@ void BeamModuleAssembler::emit_send() { fragment_call(ga->get_call_light_bif_shared()); } -static Eterm call_bif(Process *c_p, - Eterm *reg, - ErtsCodePtr I, - ErtsBifFunc vbf, - Uint arity) { - ErlHeapFragment *live_hf_end; - Eterm result; - - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - { - live_hf_end = c_p->mbuf; - - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = vbf(c_p, reg, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_HOLE_CHECK(c_p); - } - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - - if (ERTS_IS_GC_DESIRED(c_p)) { - result = erts_gc_after_bif_call_lhf(c_p, - live_hf_end, - result, - reg, - arity); - } - - return result; -} - -typedef Eterm NifF(struct enif_environment_t *, int argc, Eterm argv[]); - -static Eterm call_nif(Process *c_p, - ErtsCodePtr I, - Eterm *reg, - NifF *fp, - struct erl_module_nif *NifMod) { - Eterm nif_bif_result; - Eterm bif_nif_arity; - ErlHeapFragment *live_hf_end; - const ErtsCodeMFA *codemfa; - - codemfa = erts_code_to_codemfa(I); - - c_p->current = codemfa; /* current and vbf set to please handle_error */ - - bif_nif_arity = codemfa->arity; - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - - { - struct enif_environment_t env; - ASSERT(c_p->scheduler_data); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, NifMod, NULL); - - ASSERT((c_p->scheduler_data)->current_nif == NULL); - (c_p->scheduler_data)->current_nif = &env; - - nif_bif_result = (*fp)(&env, bif_nif_arity, reg); - if (env.exception_thrown) - nif_bif_result = THE_NON_VALUE; - - ASSERT((c_p->scheduler_data)->current_nif == &env); - (c_p->scheduler_data)->current_nif = NULL; - - erts_post_nif(&env); - ERTS_CHK_MBUF_SZ(c_p); - - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ASSERT(!env.exiting); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } - ERTS_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - - if (ERTS_IS_GC_DESIRED(c_p)) { - nif_bif_result = erts_gc_after_bif_call_lhf(c_p, - live_hf_end, - nif_bif_result, - reg, - bif_nif_arity); - } - - return nif_bif_result; -} - void BeamGlobalAssembler::emit_bif_nif_epilogue(void) { Label check_trap = a.newLabel(), trap = a.newLabel(), error = a.newLabel(); @@ -862,7 +796,7 @@ void BeamGlobalAssembler::emit_call_bif_shared(void) { a.mov(ARG1, c_p); load_x_reg_array(ARG2); /* ARG3 (I), ARG4 (func), and ARG5 (arity) have already been provided. */ - runtime_call<5>(call_bif); + runtime_call<5>(beam_jit_call_bif); #ifdef ERTS_MSACC_EXTENDED_STATES a.mov(TMP_MEM1q, RET); @@ -986,7 +920,7 @@ void BeamGlobalAssembler::emit_call_nif_shared(void) { a.mov(ARG4, x86::qword_ptr(ARG2, 8 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); a.mov(ARG5, x86::qword_ptr(ARG2, 16 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); a.mov(ARG6, x86::qword_ptr(ARG2, 24 + BEAM_ASM_FUNC_PROLOGUE_SIZE)); - runtime_call<5>(call_nif); + runtime_call<5>(beam_jit_call_nif); emit_bif_nif_epilogue(); } @@ -1042,33 +976,6 @@ void BeamModuleAssembler::emit_call_nif(const ArgVal &Func, } } -enum nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield }; - -static enum nif_load_ret load_nif(Process *c_p, ErtsCodePtr I, Eterm *reg) { - if (erts_try_seize_code_write_permission(c_p)) { - Eterm result; - - PROCESS_MAIN_CHK_LOCKS((c_p)); - ERTS_UNREQ_PROC_MAIN_LOCK((c_p)); - result = erts_load_nif(c_p, I, reg[0], reg[1]); - erts_release_code_write_permission(); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - - if (ERTS_LIKELY(is_value(result))) { - reg[0] = result; - return RET_NIF_success; - } else { - c_p->freason = BADARG; - return RET_NIF_error; - } - } else { - /* Yield and try again. */ - c_p->current = NULL; - c_p->arity = 2; - return RET_NIF_yield; - } -} - /* ARG2 = entry address. */ void BeamGlobalAssembler::emit_i_load_nif_shared() { static ErtsCodeMFA bif_mfa = {am_erlang, am_load_nif, 2}; @@ -1082,7 +989,7 @@ void BeamGlobalAssembler::emit_i_load_nif_shared() { a.mov(ARG1, c_p); /* ARG2 has already been set by caller */ load_x_reg_array(ARG3); - runtime_call<3>(load_nif); + runtime_call<3>(beam_jit_load_nif); emit_leave_runtime<Update::eStack | Update::eHeap>(); @@ -1142,7 +1049,7 @@ void BeamModuleAssembler::emit_i_load_nif() { a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(currLabel)); load_x_reg_array(ARG3); - runtime_call<3>(load_nif); + runtime_call<3>(beam_jit_load_nif); emit_leave_runtime<Update::eStack | Update::eHeap>(); diff --git a/erts/emulator/beam/jit/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index 939b6c6b9f..7df0b84cae 100644 --- a/erts/emulator/beam/jit/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -95,82 +95,6 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgVal &Size, } } -void TEST_BIN_VHEAP(Process *c_p, Eterm *reg, Uint VNh, Uint Nh, Uint Live) { - int need = Nh; - - if (c_p->stop - c_p->htop < (need + S_RESERVED) || - MSO(c_p).overhead + VNh >= BIN_VHEAP_SZ(c_p)) { - c_p->fcalls -= - erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); - } -} - -void GC_TEST(Process *c_p, Eterm *reg, Uint Ns, Uint Nh, Uint Live) { - int need = Nh + Ns; - - if (ERTS_UNLIKELY(c_p->stop - c_p->htop < (need + S_RESERVED))) { - c_p->fcalls -= - erts_garbage_collect_nobump(c_p, need, reg, Live, c_p->fcalls); - } -} - -Eterm i_bs_init(Process *c_p, - Eterm *reg, - ERL_BITS_DECLARE_STATEP, - Eterm BsOp1, - Eterm BsOp2, - unsigned Live) { - if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin *hb; - Uint bin_need; - - bin_need = heap_bin_size(BsOp1); - erts_bin_offset = 0; - erts_writable_bin = 0; - GC_TEST(c_p, reg, 0, bin_need + BsOp2 + ERL_SUB_BIN_SIZE, Live); - hb = (ErlHeapBin *)c_p->htop; - c_p->htop += bin_need; - hb->thing_word = header_heap_bin(BsOp1); - hb->size = BsOp1; - erts_current_bin = (byte *)hb->data; - return make_binary(hb); - } else { - Binary *bptr; - ProcBin *pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - TEST_BIN_VHEAP(c_p, - reg, - BsOp1 / sizeof(Eterm), - BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, - Live); - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(BsOp1); - erts_current_bin = (byte *)bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *)c_p->htop; - c_p->htop += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = BsOp1; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header *)pb; - pb->val = bptr; - pb->bytes = (byte *)bptr->orig_bytes; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); - - return make_binary(pb); - } -} - void BeamModuleAssembler::emit_i_bs_init_heap(const ArgVal &Size, const ArgVal &Heap, const ArgVal &Live, @@ -185,29 +109,18 @@ void BeamModuleAssembler::emit_i_bs_init_heap(const ArgVal &Size, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init); + runtime_call<6>(beam_jit_bs_init); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); mov_arg(Dst, RET); } -static void bs_field_size_argument_error(Process *c_p, Eterm size) { - if (((is_small(size) && signed_val(size) >= 0) || - (is_big(size) && !big_sign(size)))) { - /* If the argument is a positive integer, we must've had a system_limit - * error. */ - c_p->freason = SYSTEM_LIMIT; - } else { - c_p->freason = BADARG; - } -} - /* Set the error reason when a size check has failed. */ void BeamGlobalAssembler::emit_bs_size_check_shared() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<2>(bs_field_size_argument_error); + runtime_call<2>(beam_jit_bs_field_size_argument_error); emit_leave_runtime(); a.ret(); } @@ -236,7 +149,7 @@ void BeamModuleAssembler::emit_i_bs_init_fail_heap(const ArgVal &Size, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init); + runtime_call<6>(beam_jit_bs_init); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -276,90 +189,6 @@ void BeamModuleAssembler::emit_i_bs_init_fail(const ArgVal &Size, emit_i_bs_init_fail_heap(Size, Heap, Fail, Live, Dst); } -static Eterm i_bs_init_bits(Process *c_p, - Eterm *reg, - ERL_BITS_DECLARE_STATEP, - Uint num_bits, - Uint alloc, - unsigned Live) { - Eterm new_binary; - Uint num_bytes = ((Uint64)num_bits + (Uint64)7) >> 3; - - if (num_bits & 7) { - alloc += ERL_SUB_BIN_SIZE; - } - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - alloc += heap_bin_size(num_bytes); - } else { - alloc += PROC_BIN_SIZE; - } - GC_TEST(c_p, reg, 0, alloc, Live); - - /* num_bits = Number of bits to build - * num_bytes = Number of bytes to allocate in the binary - * alloc = Total number of words to allocate on heap - * Operands: NotUsed NotUsed Dst - */ - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin *hb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - hb = (ErlHeapBin *)c_p->htop; - c_p->htop += heap_bin_size(num_bytes); - hb->thing_word = header_heap_bin(num_bytes); - hb->size = num_bytes; - erts_current_bin = (byte *)hb->data; - new_binary = make_binary(hb); - - do_bits_sub_bin: - if (num_bits & 7) { - ErlSubBin *sb; - - sb = (ErlSubBin *)c_p->htop; - c_p->htop += ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = num_bytes - 1; - sb->bitsize = num_bits & 7; - sb->offs = 0; - sb->bitoffs = 0; - sb->is_writable = 0; - sb->orig = new_binary; - new_binary = make_binary(sb); - } - /* HEAP_SPACE_VERIFIED(0); */ - return new_binary; - } else { - Binary *bptr; - ProcBin *pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(num_bytes); - erts_current_bin = (byte *)bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *)c_p->htop; - c_p->htop += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = num_bytes; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header *)pb; - pb->val = bptr; - pb->bytes = (byte *)bptr->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); - new_binary = make_binary(pb); - goto do_bits_sub_bin; - } -} - void BeamModuleAssembler::emit_i_bs_init_bits(const ArgVal &NumBits, const ArgVal &Live, const ArgVal &Dst) { @@ -381,7 +210,7 @@ void BeamModuleAssembler::emit_i_bs_init_bits_heap(const ArgVal &NumBits, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init_bits); + runtime_call<6>(beam_jit_bs_init_bits); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -422,7 +251,7 @@ void BeamModuleAssembler::emit_i_bs_init_bits_fail_heap(const ArgVal &NumBits, a.mov(ARG1, c_p); load_x_reg_array(ARG2); load_erl_bits_state(ARG3); - runtime_call<6>(i_bs_init_bits); + runtime_call<6>(beam_jit_bs_init_bits); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -728,7 +557,11 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src, a.and_(RETb, imm(_HEADER_SUBTAG_MASK)); a.cmp(RETb, imm(BIN_MATCHSTATE_SUBTAG)); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif if (Fail.getValue() != 0) { comment("is_binary_header"); @@ -818,38 +651,6 @@ void BeamModuleAssembler::emit_i_bs_get_position(const ArgVal &Ctx, mov_arg(Dst, ARG1); } -static Eterm i_bs_get_integer(Process *c_p, - Eterm *reg, - Eterm context, - Uint flags, - Uint size, - Uint Live) { - ErlBinMatchBuffer *mb; - - if (size >= SMALL_BITS) { - Uint wordsneeded; - - /* Check bits size before potential gc. - * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails). - * - * Remember to re-acquire the matchbuffer after gc. - */ - mb = ms_matchbuffer(context); - if (mb->size - mb->offset < size) { - return THE_NON_VALUE; - } - - wordsneeded = 1 + WSIZE(NBYTES((Uint)size)); - reg[Live] = context; - GC_TEST(c_p, reg, 0, wordsneeded, Live + 1); - context = reg[Live]; - } - - mb = ms_matchbuffer(context); - return erts_bs_get_integer_2(c_p, size, flags, mb); -} - /* ARG3 = flags | (size << 3), * ARG4 = tagged match context */ void BeamGlobalAssembler::emit_bs_fixed_integer_shared() { @@ -890,7 +691,11 @@ x86::Mem BeamModuleAssembler::emit_bs_get_integer_prologue(Label next, /* The above call can't fail since we work on small numbers and * bounds-tested above. */ +#ifdef JIT_HARD_DEBUG + a.jmp(next); +#else a.short_().jmp(next); +#endif a.bind(aligned); { @@ -1108,7 +913,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer(const ArgVal &Ctx, a.mov(ARG1, c_p); load_x_reg_array(ARG2); - runtime_call<6>(i_bs_get_integer); + runtime_call<6>(beam_jit_bs_get_integer); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -1569,24 +1374,11 @@ void BeamModuleAssembler::emit_bs_test_unit(const ArgVal &Fail, a.jnz(labels[Fail.getValue()]); } -/* Set the exception code for bs_add argument errors after the fact, which is - * much easier and more compact than discriminating within module code. */ -static void bs_add_argument_error(Process *c_p, Eterm A, Eterm B) { - if (((is_small(A) && signed_val(A) >= 0) || (is_big(A) && !big_sign(A))) && - ((is_small(B) && signed_val(B) >= 0) || (is_big(B) && !big_sign(B)))) { - /* If all arguments are positive integers, we must've had a system_limit - * error. */ - c_p->freason = SYSTEM_LIMIT; - } else { - c_p->freason = BADARG; - } -} - /* Set the error reason when bs_add has failed. */ void BeamGlobalAssembler::emit_bs_add_shared() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<3>(bs_add_argument_error); + runtime_call<3>(beam_jit_bs_add_argument_error); emit_leave_runtime(); a.ret(); } @@ -1762,79 +1554,6 @@ void BeamModuleAssembler::emit_bs_init_writable() { emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); } -static Eterm i_bs_start_match2_gc_test_preserve(Process *c_p, - Eterm *reg, - Uint need, - Uint live, - Eterm preserve) { - Uint words_left = (Uint)(STACK_TOP(c_p) - HEAP_TOP(c_p)); - - if (ERTS_UNLIKELY(words_left < need + S_RESERVED)) { - reg[live] = preserve; - PROCESS_MAIN_CHK_LOCKS(c_p); - c_p->fcalls -= erts_garbage_collect_nobump(c_p, - need, - reg, - live + 1, - c_p->fcalls); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - preserve = reg[live]; - } - - return preserve; -} - -static Eterm i_bs_start_match2(Eterm context, - Uint live, - Uint slots, - Process *c_p, - Eterm *reg) { - Eterm header; - if (!is_boxed(context)) { - return THE_NON_VALUE; - } - header = *boxed_val(context); - - slots++; - - if (header_is_bin_matchstate(header)) { - ErlBinMatchState *ms = (ErlBinMatchState *)boxed_val(context); - Uint actual_slots = HEADER_NUM_SLOTS(header); - - /* We're not compatible with contexts created by bs_start_match3. */ - ASSERT(actual_slots >= 1); - - ms->save_offset[0] = ms->mb.offset; - if (ERTS_UNLIKELY(actual_slots < slots)) { - ErlBinMatchState *expanded; - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - context = i_bs_start_match2_gc_test_preserve(c_p, - reg, - wordsneeded, - live, - context); - ms = (ErlBinMatchState *)boxed_val(context); - expanded = (ErlBinMatchState *)HEAP_TOP(c_p); - *expanded = *ms; - *HEAP_TOP(c_p) = HEADER_BIN_MATCHSTATE(slots); - HEAP_TOP(c_p) += wordsneeded; - context = make_matchstate(expanded); - } - return context; - } else if (is_binary_header(header)) { - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - context = i_bs_start_match2_gc_test_preserve(c_p, - reg, - wordsneeded, - live, - context); - return erts_bs_start_match_2(c_p, context, slots); - } else { - return THE_NON_VALUE; - } -} - /* Old compatibility instructions for <= OTP-21. Kept in order to be able to * load old code. While technically we could remove these in OTP-24, we've * decided to keep them until at least OTP-25 to make things easier for @@ -1852,7 +1571,7 @@ void BeamModuleAssembler::emit_i_bs_start_match2(const ArgVal &Src, a.mov(ARG4, c_p); load_x_reg_array(ARG5); - runtime_call<5>(i_bs_start_match2); + runtime_call<5>(beam_jit_bs_start_match2); emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); @@ -1883,41 +1602,12 @@ void BeamModuleAssembler::emit_i_bs_restore2(const ArgVal &Ctx, a.mov(emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.offset)), ARG2); } -static void bs_context_to_binary(Eterm context) { - if (is_boxed(context) && header_is_bin_matchstate(*boxed_val(context))) { - Uint orig, size, offs, hole_size; - ErlBinMatchBuffer *mb; - ErlBinMatchState *ms; - ErlSubBin *sb; - ms = (ErlBinMatchState *)boxed_val(context); - mb = &ms->mb; - offs = ms->save_offset[0]; - size = mb->size - offs; - orig = mb->orig; - sb = (ErlSubBin *)boxed_val(context); - /* Since we're going to overwrite the match state with the result, an - * ErlBinMatchState must be at least as large as an ErlSubBin. */ - ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); - hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(size); - sb->bitsize = BIT_OFFSET(size); - sb->offs = BYTE_OFFSET(offs); - sb->bitoffs = BIT_OFFSET(offs); - sb->is_writable = 0; - sb->orig = orig; - if (hole_size) { - sb[1].thing_word = make_pos_bignum_header(hole_size - 1); - } - } -} - void BeamModuleAssembler::emit_bs_context_to_binary(const ArgVal &Src) { mov_arg(ARG1, Src); emit_enter_runtime(); - runtime_call<1>(bs_context_to_binary); + runtime_call<1>(beam_jit_bs_context_to_binary); emit_leave_runtime(); } diff --git a/erts/emulator/beam/jit/instr_call.cpp b/erts/emulator/beam/jit/x86/instr_call.cpp index 51b09edaf9..b79ef5567c 100644 --- a/erts/emulator/beam/jit/instr_call.cpp +++ b/erts/emulator/beam/jit/x86/instr_call.cpp @@ -41,7 +41,7 @@ void BeamGlobalAssembler::emit_dispatch_return() { void BeamModuleAssembler::emit_return() { Label dispatch_return = a.newLabel(); -#ifdef HARD_DEBUG +#ifdef JIT_HARD_DEBUG /* Validate return address and {x,0} */ emit_validate(ArgVal(ArgVal::u, 1)); #endif @@ -236,6 +236,8 @@ x86::Gp BeamModuleAssembler::emit_apply_fun() { a.lea(ARG5, TMP_MEM1q); runtime_call<5>(apply_fun); + emit_leave_runtime<Update::eStack | Update::eHeap>(); + a.mov(ARG2, RET); a.test(RET, RET); @@ -244,8 +246,6 @@ x86::Gp BeamModuleAssembler::emit_apply_fun() { * `emit_setup_export_call` for details. */ a.mov(RET, TMP_MEM1q); - emit_leave_runtime<Update::eStack | Update::eHeap>(); - a.short_().jne(dispatch); emit_handle_error(); a.bind(dispatch); @@ -284,6 +284,8 @@ x86::Gp BeamModuleAssembler::emit_call_fun(const ArgVal &Fun) { a.lea(ARG5, TMP_MEM1q); runtime_call<5>(call_fun); + emit_leave_runtime<Update::eStack | Update::eHeap>(); + a.mov(ARG2, RET); a.test(RET, RET); @@ -292,8 +294,6 @@ x86::Gp BeamModuleAssembler::emit_call_fun(const ArgVal &Fun) { * `emit_setup_export_call` for details. */ a.mov(RET, TMP_MEM1q); - emit_leave_runtime<Update::eStack | Update::eHeap>(); - a.short_().jne(dispatch); emit_handle_error(); a.bind(dispatch); diff --git a/erts/emulator/beam/jit/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp index 4c278d7a5a..4ca7a10629 100644 --- a/erts/emulator/beam/jit/instr_common.cpp +++ b/erts/emulator/beam/jit/x86/instr_common.cpp @@ -133,20 +133,6 @@ void BeamModuleAssembler::emit_gc_test(const ArgVal &Ns, a.bind(after_gc_check); } -#if defined(DEBUG) && defined(HARD_DEBUG) -static void validate_term(Eterm term) { - if (is_boxed(term)) { - Eterm header = *boxed_val(term); - - if (header_is_bin_matchstate(header)) { - return; - } - } - - size_object_x(term, nullptr); -} -#endif - void BeamModuleAssembler::emit_validate(const ArgVal &arity) { #ifdef DEBUG Label next = a.newLabel(), crash = a.newLabel(); @@ -169,16 +155,17 @@ void BeamModuleAssembler::emit_validate(const ArgVal &arity) { a.hlt(); a.bind(next); -# ifdef HARD_DEBUG +# ifdef JIT_HARD_DEBUG emit_enter_runtime(); for (unsigned i = 0; i < arity.getValue(); i++) { a.mov(ARG1, getXRef(i)); - runtime_call<1>(validate_term); + runtime_call<1>(beam_jit_validate_term); } emit_leave_runtime(); # endif + #endif } @@ -1220,7 +1207,11 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, mov_arg(ARG1, X); a.cmp(ARG1, ARG2); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif /* Fancy way of checking if both are immediates. */ a.mov(RETd, ARG1d); @@ -1279,7 +1270,11 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, a.and_(RETd, ARG2d); a.and_(RETb, imm(_TAG_PRIMARY_MASK)); a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); +#ifdef JIT_HARD_DEBUG + a.je(next); +#else a.short_().je(next); +#endif emit_enter_runtime(); @@ -1437,7 +1432,9 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { a.sub(ARG3d, imm(_TAG_HEADER_FLOAT)); a.sub(ARG5d, imm(_TAG_HEADER_FLOAT)); a.or_(ARG3d, ARG5d); - a.short_().jne(generic_compare); + + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(generic_compare); boxed_ptr = emit_ptr_val(ARG1, ARG1); a.movsd(x86::xmm0, emit_boxed_val(boxed_ptr, sizeof(Eterm))); @@ -1660,7 +1657,8 @@ void BeamGlobalAssembler::emit_catch_end_shared() { a.bind(not_throw); { a.cmp(getXRef(1), imm(am_error)); - a.short_().jne(not_error); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(not_error); /* This is an error, attach a stacktrace to the reason. */ emit_enter_runtime<Update::eStack | Update::eHeap>(); diff --git a/erts/emulator/beam/jit/instr_float.cpp b/erts/emulator/beam/jit/x86/instr_float.cpp index 43acf485ff..43acf485ff 100644 --- a/erts/emulator/beam/jit/instr_float.cpp +++ b/erts/emulator/beam/jit/x86/instr_float.cpp diff --git a/erts/emulator/beam/jit/instr_guard_bifs.cpp b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp index 1084a1c0e9..1084a1c0e9 100644 --- a/erts/emulator/beam/jit/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp diff --git a/erts/emulator/beam/jit/instr_map.cpp b/erts/emulator/beam/jit/x86/instr_map.cpp index c3444c9755..a1fb7394d9 100644 --- a/erts/emulator/beam/jit/instr_map.cpp +++ b/erts/emulator/beam/jit/x86/instr_map.cpp @@ -134,71 +134,6 @@ void BeamModuleAssembler::emit_i_get_map_element(const ArgVal &Fail, } } -static Uint i_get_map_elements(Eterm map, - Eterm *reg, - Eterm *E, - Uint n, - Eterm *fs) { - Uint sz; - - /* This instruction assumes Arg1 is a map, i.e. that it follows a test - * is_map if needed. */ - - if (is_flatmap(map)) { - flatmap_t *mp; - Eterm *ks; - Eterm *vs; - - mp = (flatmap_t *)flatmap_val(map); - sz = flatmap_get_size(mp); - - if (sz == 0) { - return 0; - } - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - - while (sz) { - if (EQ(fs[0], *ks)) { - PUT_TERM_REG(*vs, fs[1]); - - n--; - fs += 3; - - /* no more values to fetch, we are done */ - if (n == 0) { - return 1; - } - } - - ks++, sz--, vs++; - } - return 0; - } else { - ASSERT(is_hashmap(map)); - - while (n--) { - const Eterm *v; - Uint32 hx; - - hx = fs[2]; - ASSERT(hx == hashmap_make_hash(fs[0])); - - if ((v = erts_hashmap_get(hx, fs[0], map)) == NULL) { - return 0; - } - - PUT_TERM_REG(*v, fs[1]); - fs += 3; - } - - return 1; - } -} - -#undef PUT_TERM_REG - void BeamModuleAssembler::emit_i_get_map_elements( const ArgVal &Fail, const ArgVal &Src, @@ -216,7 +151,7 @@ void BeamModuleAssembler::emit_i_get_map_elements( mov_imm(ARG4, args.size() / 3); a.lea(ARG5, x86::qword_ptr(data)); load_x_reg_array(ARG2); - runtime_call<5>(i_get_map_elements); + runtime_call<5>(beam_jit_get_map_elements); emit_leave_runtime(); diff --git a/erts/emulator/beam/jit/instr_msg.cpp b/erts/emulator/beam/jit/x86/instr_msg.cpp index a929fec93f..e9110e174b 100644 --- a/erts/emulator/beam/jit/instr_msg.cpp +++ b/erts/emulator/beam/jit/x86/instr_msg.cpp @@ -30,24 +30,6 @@ extern "C" #endif } -static ErtsMessage *decode_dist(Process *c_p, ErtsMessage *msgp) { - if (!erts_proc_sig_decode_dist(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - - /* TODO: Add DTrace probe for this bad message situation? */ - erts_msgq_unlink_msg(c_p, msgp); - msgp->next = NULL; - erts_cleanup_messages(msgp); - - return nullptr; - } - - return msgp; -} - #ifdef ERTS_SUPPORT_OLD_RECV_MARK_INSTRS static void recv_mark(Process *p) { @@ -219,7 +201,8 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { /* Need to spill message_ptr to ARG1 as check_is_distributed uses it */ a.mov(ARG1, message_ptr); a.test(ARG1, ARG1); - a.short_().jne(check_is_distributed); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(check_is_distributed); /* Did we receive a signal or run out of reds? */ a.cmp(get_out, imm(0)); @@ -252,7 +235,8 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { { a.cmp(x86::qword_ptr(ARG1, offsetof(ErtsSignal, common.tag)), imm(THE_NON_VALUE)); - a.short_().jne(done); + /* NOTE: Short won't reach if JIT_HARD_DEBUG is defined. */ + a.jne(done); a.sub(FCALLS, imm(10)); @@ -260,7 +244,7 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { a.mov(ARG2, ARG1); a.mov(ARG1, c_p); - runtime_call<2>(decode_dist); + runtime_call<2>(beam_jit_decode_dist); emit_leave_runtime(); @@ -291,119 +275,6 @@ void BeamModuleAssembler::emit_i_loop_rec(const ArgVal &Wait) { fragment_call(ga->get_i_loop_rec_shared()); } -/* Remove a (matched) message from the message queue. */ -static Sint remove_message(Process *c_p, - Sint FCALLS, - Eterm *HTOP, - Eterm *E, - Uint32 active_code_ix) { - ErtsMessage *msgp; - - ERTS_CHK_MBUF_SZ(c_p); - - if (active_code_ix == ERTS_SAVE_CALLS_CODE_IX) { - save_calls(c_p, &exp_receive); - } - - msgp = erts_msgq_peek_msg(c_p); - - if (ERL_MESSAGE_TOKEN(msgp) == NIL) { -#ifdef USE_VM_PROBES - if (DT_UTAG(c_p) != NIL) { - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { - SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; - } else { - DT_UTAG(c_p) = NIL; - SEQ_TRACE_TOKEN(c_p) = NIL; - } - } else { -#endif - SEQ_TRACE_TOKEN(c_p) = NIL; -#ifdef USE_VM_PROBES - } - DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; -#endif - } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { - Eterm msg; - SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); -#ifdef USE_VM_PROBES - if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { - if (DT_UTAG(c_p) == NIL) { - DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); - } - DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; - } else { -#endif - ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)) || - is_atom(SEQ_TRACE_TOKEN_SENDER(c_p))); - c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - if (c_p->seq_trace_clock < - unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { - c_p->seq_trace_clock = - unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - } - msg = ERL_MESSAGE_TERM(msgp); - seq_trace_output(SEQ_TRACE_TOKEN(c_p), - msg, - SEQ_TRACE_RECEIVE, - c_p->common.id, - c_p); -#ifdef USE_VM_PROBES - } -#endif - } -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_receive)) { - Eterm token2 = NIL; - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - Sint len = erts_proc_sig_privqs_len(c_p); - - dtrace_proc_str(c_p, receiver_name); - token2 = SEQ_TRACE_TOKEN(c_p); - if (have_seqtrace(token2)) { - tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); - } - DTRACE6(message_receive, - receiver_name, - size_object(ERL_MESSAGE_TERM(msgp)), - len, /* This is NOT message queue len, but its something... */ - tok_label, - tok_lastcnt, - tok_serial); - } -#endif - erts_msgq_unlink_msg(c_p, msgp); - erts_msgq_set_save_first(c_p); - CANCEL_TIMER(c_p); - - erts_save_message_in_proc(c_p, msgp); - c_p->flags &= ~F_DELAY_GC; - - if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { - /* - * We want to GC soon but we leave a few - * reductions giving the message some time - * to turn into garbage. - */ - ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); - } - - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - return FCALLS; -} - void BeamModuleAssembler::emit_remove_message() { /* HTOP and E are passed explicitly and only read from, so we don't need to * swap them out. */ @@ -415,21 +286,17 @@ void BeamModuleAssembler::emit_remove_message() { a.mov(ARG1, c_p); a.mov(ARG2, FCALLS); a.mov(ARG5, active_code_ix); - runtime_call<5>(remove_message); + runtime_call<5>(beam_jit_remove_message); a.mov(FCALLS, RET); emit_leave_runtime(); } -static void save_message(Process *c_p) { - erts_msgq_set_save_next(c_p); -} - void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(save_message); + runtime_call<1>(erts_msgq_set_save_next); emit_leave_runtime(); @@ -437,32 +304,12 @@ void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) { a.jmp(labels[Dest.getValue()]); } -static void take_receive_lock(Process *c_p) { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); -} - -static void wait_locked(Process *c_p, ErtsCodePtr cp) { - c_p->arity = 0; - if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) { - erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->current = NULL; - c_p->i = cp; -} - -static void wait_unlocked(Process *c_p, ErtsCodePtr cp) { - take_receive_lock(c_p); - wait_locked(c_p, cp); -} - void BeamModuleAssembler::emit_wait_unlocked(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); - runtime_call<2>(wait_unlocked); + runtime_call<2>(beam_jit_wait_unlocked); emit_leave_runtime(); @@ -474,54 +321,19 @@ void BeamModuleAssembler::emit_wait_locked(const ArgVal &Dest) { a.mov(ARG1, c_p); a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); - runtime_call<2>(wait_locked); + runtime_call<2>(beam_jit_wait_locked); emit_leave_runtime(); abs_jmp(ga->get_do_schedule()); } -enum tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 }; - -static enum tmo_ret wait_timeout(Process *c_p, - Eterm timeout_value, - ErtsCodePtr next) { - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - if (timeout_value == make_small(0)) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - return RET_next; - } else if (timeout_value == am_infinity) { - c_p->flags |= F_TIMO; - } else { - int tres = erts_set_proc_timer_term(c_p, timeout_value); - if (tres == 0) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - */ - c_p->def_arg_reg[0] = (Eterm)next; - } else { /* Wrong time */ - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->freason = EXC_TIMEOUT_VALUE; - return RET_badarg; - } - } - } - return RET_wait; -} - void BeamModuleAssembler::emit_wait_timeout_unlocked(const ArgVal &Src, const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(take_receive_lock); + runtime_call<1>(beam_jit_take_receive_lock); emit_leave_runtime(); @@ -538,14 +350,18 @@ void BeamModuleAssembler::emit_wait_timeout_locked(const ArgVal &Src, a.mov(ARG1, c_p); a.lea(ARG3, x86::qword_ptr(next)); - runtime_call<3>(wait_timeout); + runtime_call<3>(beam_jit_wait_timeout); emit_leave_runtime(); ERTS_CT_ASSERT(RET_next < RET_wait && RET_wait < RET_badarg); a.cmp(RET, RET_wait); a.short_().je(wait); +#ifdef JIT_HARD_DEBUG + a.jl(next); +#else a.short_().jl(next); +#endif emit_handle_error(currLabel, (ErtsCodeMFA *)nullptr); @@ -556,27 +372,11 @@ void BeamModuleAssembler::emit_wait_timeout_locked(const ArgVal &Src, a.bind(next); } -static void timeout(Process *c_p) { - if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_clock_service, am_timeout, NULL); - } - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_timeout); - } - c_p->flags &= ~F_TIMO; - erts_msgq_set_save_first(c_p); -} - -static void timeout_locked(Process *c_p) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - timeout(c_p); -} - void BeamModuleAssembler::emit_timeout_locked() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(timeout_locked); + runtime_call<1>(beam_jit_timeout_locked); emit_leave_runtime(); } @@ -585,7 +385,7 @@ void BeamModuleAssembler::emit_timeout() { emit_enter_runtime(); a.mov(ARG1, c_p); - runtime_call<1>(timeout); + runtime_call<1>(beam_jit_timeout); emit_leave_runtime(); } diff --git a/erts/emulator/beam/jit/instr_select.cpp b/erts/emulator/beam/jit/x86/instr_select.cpp index 8aaa80267f..8aaa80267f 100644 --- a/erts/emulator/beam/jit/instr_select.cpp +++ b/erts/emulator/beam/jit/x86/instr_select.cpp diff --git a/erts/emulator/beam/jit/instr_trace.cpp b/erts/emulator/beam/jit/x86/instr_trace.cpp index a1b87624c7..a1b87624c7 100644 --- a/erts/emulator/beam/jit/instr_trace.cpp +++ b/erts/emulator/beam/jit/x86/instr_trace.cpp diff --git a/erts/emulator/beam/jit/ops.tab b/erts/emulator/beam/jit/x86/ops.tab index e33cc110a5..e33cc110a5 100644 --- a/erts/emulator/beam/jit/ops.tab +++ b/erts/emulator/beam/jit/x86/ops.tab diff --git a/erts/emulator/beam/jit/predicates.tab b/erts/emulator/beam/jit/x86/predicates.tab index a4e00586f6..a4e00586f6 100644 --- a/erts/emulator/beam/jit/predicates.tab +++ b/erts/emulator/beam/jit/x86/predicates.tab diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index afe02f8f13..7df04420da 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2061,7 +2061,7 @@ trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p) * One IMPORTANT property must hold (for hamt). * EVERY BIT of the term that is significant for equality (see EQ) * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a - * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]). + * chance of hashing different when salted. * * This is why we cannot use cached hash values for atoms for example. * @@ -2073,14 +2073,19 @@ do { /* Lightweight mixing of constant (type info) */ \ hash = (hash << 17) ^ (hash >> (32-17)); \ } while (0) +/* + * Start with salt, 32-bit prime number, to avoid getting same hash as phash2 + * which can cause bad hashing in distributed ETS tables for example. + */ +#define INTERNAL_HASH_SALT 3432918353U + Uint32 make_internal_hash(Eterm term, Uint32 salt) { - Uint32 hash; + Uint32 hash = salt ^ INTERNAL_HASH_SALT; /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { - hash = salt; #if ERTS_SIZEOF_ETERM == 8 UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); #elif ERTS_SIZEOF_ETERM == 4 @@ -2094,7 +2099,6 @@ make_internal_hash(Eterm term, Uint32 salt) Eterm tmp; DECLARE_ESTACK(s); - hash = salt; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: diff --git a/erts/emulator/internal_doc/NewLinking.tla b/erts/emulator/internal_doc/NewLinking.tla new file mode 100644 index 0000000000..832bbaf037 --- /dev/null +++ b/erts/emulator/internal_doc/NewLinking.tla @@ -0,0 +1,194 @@ +\* +\* %CopyrightBegin% +\* +\* Copyright Ericsson AB 2021. All Rights Reserved. +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* +\* %CopyrightEnd% +\* + +(* + --- Model of the new link protocol introduced in Erlang/OTP 23.3 --- + + The protocol is documented in the ERTS User's Guide -> + Distribution Protocol -> Protocol Between Connected Nodes -> + Link Protocol -> New Link Protocol + + This model only models a link between two processes. This since a link + between one pair of processes is completely independent of links between + other pairs of processes. This model also assumes that the connection + between the processes does not fail. In the real world a connection can + of course fail. This is however taken care of by clearing the link + information on both ends when a connection fails, and tracking which + instantiation of a connection between the nodes signals arrive on and + ignoring signals from old instantiations of connections. That is, + connection loss is trivially taken care of since we just start over + again from scratch if the connection is lost. + + The documentation of the protocol talks about "process local information + about links". This information is stored in the process state record below. + The 'other' field contains the identifier of the other process. The 'type' + field acts as "active flag". The link is active when 'type' equals + "linked" and not active when 'type' equals "unlinked". If the + 'wait_unlink_ack' field contains a value larger than or equal to zero + it is the "unlink id" of an unlink request we have issued and are waiting + for to get acknowledged. If the 'wait_unlink_ack' field contains -1 we + are not waiting for an acknowledgment. When 'type' equals "unlinked" and + 'wait_unlink_ack' equals -1, we would in the documented protocol have + removed the "process local information about the link". In this model we, + however, keep the state, but in this state instead of removing it. + Messages are tagged with a message number in order to model the signal + order of Erlang. The message number of the unlink signal is also used as + "unlink id". + + The model has been checked with the following parameters: + + Declared constants: + PROCS <- {"a", "b"} + + Temporal formula: + FairSpec + + Deadlock: + disabled + + Invariants: + TypeOK + ValidState + + State Constraint: + /\ \E proc \in PROCS : procState[proc].next_send =< 15 + /\ Cardinality(msgs) =< 10 + + That is, we have checked all states where processes send up to 15 signals + with at most 10 outstanding signals. + + Deadlock checking has been disabled since we intentionally stop when + we have no outstanding signals (in 'Next') in order to avoid checking + signal sequences equivalent to sequences we already have checked. + +*) +------------------------------ MODULE NewLinking ------------------------------ +EXTENDS Integers, TLC, FiniteSets +CONSTANTS PROCS \* PROCS should be a set of exactly two process names + +VARIABLES + procState, \* Set of process states; procState[proc] is state of proc + msgs \* Set of messages sent + +vars == <<procState, msgs>> + +\* Set of possible process states... +procStateRec == + [self : PROCS, + other: PROCS, + type : {"linked", "unlinked"}, + wait_unlink_ack : Nat \cup {-1}, + next_send : Nat, + next_recv : Nat] + +\* Set of possible messages... +Messages == + [type : {"link", "unlink", "unlink_ack"}, from : PROCS, to : PROCS, msg_no : Nat, ack : Nat \cup {-1}] + +TypeOK == + /\ procState \in [PROCS -> procStateRec] + /\ msgs \subseteq Messages + +Init == + /\ msgs = {} + /\ procState = [p \in PROCS |-> [self |-> p, + other |-> CHOOSE p2 \in PROCS : p2 /= p, + type |-> "unlinked", + next_send |-> 0, + next_recv |-> 0, + wait_unlink_ack |-> -1]] + +MkMsg(self, mtype, accnr) == + [type |-> mtype, + from |-> procState[self].self, + to |-> procState[self].other, + msg_no |-> procState[self].next_send, + ack |-> accnr] + +Link(self) == + /\ procState[self].type = "unlinked" + /\ msgs' = msgs \cup {MkMsg(self, "link", -1)} + /\ procState' = [procState EXCEPT ![self].type = "linked", + ![self].next_send = @ + 1, + ![self].wait_unlink_ack = -1] + +Unlink(self) == + /\ procState[self].type = "linked" + /\ msgs' = msgs \cup {MkMsg(self, "unlink", -1)} + /\ procState' = [procState EXCEPT ![self].type = "unlinked", + ![self].next_send = @ + 1, + ![self].wait_unlink_ack = procState[self].next_send] + +RecvLink(self, msg) == + LET type == IF procState[self].wait_unlink_ack /= -1 + THEN "unlinked" + ELSE "linked" + IN /\ msgs' = msgs \ {msg} + /\ procState' = [procState EXCEPT ![self].type = type, + ![self].next_recv = @ + 1] + +RecvUnlink(self, msg) == + /\ msgs' = (msgs \ {msg}) \cup {MkMsg(self, + "unlink_ack", + procState[self].next_recv)} + /\ procState' = [procState EXCEPT ![self].type = "unlinked", + ![self].next_recv = @ + 1, + ![self].next_send = @ + 1] + +RecvUnlinkAck(self, msg) == + LET wack == IF procState[self].wait_unlink_ack = msg.ack + THEN -1 + ELSE procState[self].wait_unlink_ack + IN /\ msgs' = msgs \ {msg} + /\ procState' = [procState EXCEPT ![self].next_recv = @ + 1, + ![self].wait_unlink_ack = wack] + +Recv(self) == + /\ \E m \in msgs : /\ m.to = self + /\ m.msg_no = procState[self].next_recv + /\ LET msg == CHOOSE m \in msgs : /\ m.to = self + /\ m.msg_no = procState[self].next_recv + IN CASE msg.type = "link" -> RecvLink(self, msg) + [] msg.type = "unlink" -> RecvUnlink(self, msg) + [] msg.type = "unlink_ack" -> RecvUnlinkAck(self, msg) + +(* + If we have no outstanding messages; both processes should + have the same view about whether they are linked or not... +*) +ValidState == + IF msgs /= {} + THEN TRUE + ELSE \A p \in PROCS : \A p2 \in PROCS : procState[p].type = procState[p2].type + +Next == + /\ (msgs /= {} \/ \A p \in PROCS : procState[p].next_send = 0) + /\ \E p \in PROCS : \/ Recv(p) + \/ Link(p) + \/ Unlink(p) + +Spec == Init /\ [][Next]_vars + +FairSpec == Spec /\ WF_vars(Next) + +============================================================================= +\* Modification History +\* Last modified Mon Jan 25 11:26:06 CET 2021 by rickard.green +\* Created Wed Jan 20 13:11:46 CET 2021 by rickard.green diff --git a/erts/emulator/nifs/common/prim_socket_nif.c b/erts/emulator/nifs/common/prim_socket_nif.c index 5b81c703a8..5dabd15965 100644 --- a/erts/emulator/nifs/common/prim_socket_nif.c +++ b/erts/emulator/nifs/common/prim_socket_nif.c @@ -650,8 +650,9 @@ static const struct msg_flag { #define ESOCK_CNT_INC( __E__, __D__, SF, ACNT, CNT, INC) \ do { \ - if (cnt_inc((CNT), (INC)) && (__D__)->iow) \ - esock_send_wrap_msg((__E__), (__D__), (SF), (ACNT)); \ + if (cnt_inc((CNT), (INC))) { \ + esock_send_wrap_msg((__E__), (__D__), (SF), (ACNT)); \ + } \ } while (0) @@ -799,41 +800,57 @@ typedef struct { } ESockRequestQueue; /*** The point of this is primarily testing ***/ - +/* +#if defined(ESOCK_COUNTER_SIZE) +#undef ESOCK_COUNTER_SIZE // #define ESOCK_COUNTER_SIZE 16 // #define ESOCK_COUNTER_SIZE 24 // #define ESOCK_COUNTER_SIZE 32 // #define ESOCK_COUNTER_SIZE 48 +// #define ESOCK_COUNTER_SIZE 64 + +#endif +*/ #if ESOCK_COUNTER_SIZE == 16 -typedef Uint16 ESockCounter; -#define ESOCK_COUNTER_MAX 0xFFFF -#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKUI((ENV), (CNT))) +typedef Uint16 ESockCounter; +#define ESOCK_COUNTER_MAX ((ESockCounter) 0xFFFF) +#define MKCNT(ENV, CNT) MKUI((ENV), (CNT)) +#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKCNT((CNT))) +#define ESOCK_COUNTER_FORMAT_STR "%u" #elif ESOCK_COUNTER_SIZE == 24 -typedef Uint32 ESockCounter; -#define ESOCK_COUNTER_MAX 0xFFFFFF -#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKUI((ENV), (CNT))) +typedef Uint32 ESockCounter; +#define ESOCK_COUNTER_MAX ((ESockCounter) 0xFFFFFF) +#define MKCNT(ENV, CNT) MKUI((ENV), (CNT)) +#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKCNT((ENV), (CNT))) +#define ESOCK_COUNTER_FORMAT_STR "%lu" #elif ESOCK_COUNTER_SIZE == 32 typedef Uint32 ESockCounter; -#define ESOCK_COUNTER_MAX 0xFFFFFFFF -#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKUI((ENV), (CNT))) +#define ESOCK_COUNTER_MAX (~((ESockCounter) 0)) +#define MKCNT(ENV, CNT) MKUI((ENV), (CNT)) +#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKCNT((ENV), (CNT))) +#define ESOCK_COUNTER_FORMAT_STR "%lu" #elif ESOCK_COUNTER_SIZE == 48 -typedef Uint64 ESockCounter; -#define ESOCK_COUNTER_MAX 0xFFFFFFFFFFFF -#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKUI64((ENV), (CNT))) +typedef Uint64 ESockCounter; +#define ESOCK_COUNTER_MAX ((ESockCounter) 0xFFFFFFFFFFFF) +#define MKCNT(ENV, CNT) MKUI64((ENV), (CNT)) +#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKCNT((ENV), (CNT))) +#define ESOCK_COUNTER_FORMAT_STR "%llu" #elif ESOCK_COUNTER_SIZE == 64 -typedef Uint64 ESockCounter; -#define ESOCK_COUNTER_MAX 0xFFFFFFFFFFFFFFFF -#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKUI64((ENV), (CNT))) +typedef Uint64 ESockCounter; +#define ESOCK_COUNTER_MAX (~((ESockCounter) 0)) +#define MKCNT(ENV, CNT) MKUI64((ENV), (CNT)) +#define MKCT(ENV, TAG, CNT) MKT2((ENV), (TAG), MKCNT((ENV), (CNT))) +#define ESOCK_COUNTER_FORMAT_STR "%llu" #else @@ -841,6 +858,8 @@ typedef Uint64 ESockCounter; #endif +// static const ESockCounter esock_counter_max = ESOCK_COUNTER_MAX; + typedef struct { /* * +++ This is a way to, possibly, detect memory overrides "and stuff" +++ @@ -1109,6 +1128,8 @@ static ERL_NIF_TERM esock_socket_info_counters(ErlNifEnv* env, ESockDescriptor* descP); static ERL_NIF_TERM esock_socket_info_ctype(ErlNifEnv* env, ESockDescriptor* descP); +static ERL_NIF_TERM esock_socket_info_state(ErlNifEnv* env, + unsigned int state); #define ESOCK_SOCKET_INFO_REQ_FUNCS \ ESOCK_SOCKET_INFO_REQ_FUNC_DEF(readers); \ ESOCK_SOCKET_INFO_REQ_FUNC_DEF(writers); \ @@ -3545,6 +3566,7 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') /* *** Local atoms *** */ #define LOCAL_ATOMS \ + LOCAL_ATOM_DECL(accepting); \ LOCAL_ATOM_DECL(acc_success); \ LOCAL_ATOM_DECL(acc_fails); \ LOCAL_ATOM_DECL(acc_tries); \ @@ -3559,11 +3581,14 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(assoc_id); \ LOCAL_ATOM_DECL(authentication); \ LOCAL_ATOM_DECL(boolean); \ + LOCAL_ATOM_DECL(bound); \ LOCAL_ATOM_DECL(bufsz); \ LOCAL_ATOM_DECL(close); \ LOCAL_ATOM_DECL(closed); \ LOCAL_ATOM_DECL(closing); \ LOCAL_ATOM_DECL(code); \ + LOCAL_ATOM_DECL(connected); \ + LOCAL_ATOM_DECL(connecting); \ LOCAL_ATOM_DECL(cookie_life); \ LOCAL_ATOM_DECL(counter_wrap); \ LOCAL_ATOM_DECL(counters); \ @@ -3575,6 +3600,7 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(dest_unreach); \ LOCAL_ATOM_DECL(do); \ LOCAL_ATOM_DECL(dont); \ + LOCAL_ATOM_DECL(dtor); \ LOCAL_ATOM_DECL(dup); \ LOCAL_ATOM_DECL(exclude); \ LOCAL_ATOM_DECL(false); \ @@ -3590,6 +3616,7 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(integer); \ LOCAL_ATOM_DECL(iov_max); \ LOCAL_ATOM_DECL(iow); \ + LOCAL_ATOM_DECL(listening); \ LOCAL_ATOM_DECL(local_rwnd); \ LOCAL_ATOM_DECL(max); \ LOCAL_ATOM_DECL(max_attempts); \ @@ -3632,6 +3659,7 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(origin); \ LOCAL_ATOM_DECL(otp); \ LOCAL_ATOM_DECL(otp_socket_option);\ + LOCAL_ATOM_DECL(owner); \ LOCAL_ATOM_DECL(partial_delivery); \ LOCAL_ATOM_DECL(peer_error); \ LOCAL_ATOM_DECL(peer_rwnd); \ @@ -3652,7 +3680,9 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(registry); \ LOCAL_ATOM_DECL(reject_route); \ LOCAL_ATOM_DECL(remote); \ + LOCAL_ATOM_DECL(rstates); \ LOCAL_ATOM_DECL(select); \ + LOCAL_ATOM_DECL(selected); \ LOCAL_ATOM_DECL(sender_dry); \ LOCAL_ATOM_DECL(send_failure); \ LOCAL_ATOM_DECL(shutdown); \ @@ -3679,6 +3709,7 @@ ERL_NIF_TERM esock_atom_socket_tag; // This has a "special" name ('$socket') LOCAL_ATOM_DECL(write_pkg_max); \ LOCAL_ATOM_DECL(write_tries); \ LOCAL_ATOM_DECL(write_waits); \ + LOCAL_ATOM_DECL(wstates); \ LOCAL_ATOM_DECL(zero); \ LOCAL_ATOM_DECL(zerocopy) @@ -3915,7 +3946,9 @@ ERL_NIF_TERM esock_global_info(ErlNifEnv* env) * domain: The domain of the socket * type: The type of the socket * protocol: The protocol of the socket - * ctrl: Controlling process of the socket) + * owner: Controlling process (owner) of the socket) + * rstates: Socket read state(s) + * wstates: Socket write state(s) * counters: A list of each socket counter and there current values * readers: The number of current and waiting readers * writers: The number of current and waiting writers @@ -3930,6 +3963,8 @@ ERL_NIF_TERM esock_socket_info(ErlNifEnv* env, ERL_NIF_TERM type = esock_socket_info_type(env, descP); ERL_NIF_TERM protocol = MKI(env, descP->protocol); ERL_NIF_TERM ctrlPid = MKPID(env, &descP->ctrlPid); + ERL_NIF_TERM rstates = esock_socket_info_state(env, descP->readState); + ERL_NIF_TERM wstates = esock_socket_info_state(env, descP->writeState); ERL_NIF_TERM ctype = esock_socket_info_ctype(env, descP); ERL_NIF_TERM counters = esock_socket_info_counters(env, descP); ERL_NIF_TERM readers = esock_socket_info_readers(env, descP); @@ -3941,7 +3976,9 @@ ERL_NIF_TERM esock_socket_info(ErlNifEnv* env, = {esock_atom_domain, esock_atom_type, esock_atom_protocol, - esock_atom_ctrl, + atom_owner, + atom_rstates, + atom_wstates, atom_ctype, atom_counters, atom_num_readers, @@ -3952,6 +3989,8 @@ ERL_NIF_TERM esock_socket_info(ErlNifEnv* env, type, protocol, ctrlPid, + rstates, + wstates, ctype, counters, readers, @@ -4053,6 +4092,100 @@ ERL_NIF_TERM esock_socket_info_ctype(ErlNifEnv* env, /* + * Encode the socket "state" + * That is "show" how this socket was created: + * + * normal | fromfd | {fromfd, integer()} + */ +#ifndef __WIN32__ +static +ERL_NIF_TERM esock_socket_info_state(ErlNifEnv* env, + unsigned int state) +{ + SocketTArray estate = TARRAY_CREATE(10); + ERL_NIF_TERM estateList; + + + if ((state & ESOCK_STATE_BOUND) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> bound" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_bound); + } + + if ((state & ESOCK_STATE_LISTENING) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> listening" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_listening); + } + + if ((state & ESOCK_STATE_ACCEPTING) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> accepting" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_accepting); + } + + if ((state & ESOCK_STATE_CONNECTING) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> connecting" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_connecting); + } + + if ((state & ESOCK_STATE_CONNECTED) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> connected" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_connected); + } + + if ((state & ESOCK_STATE_SELECTED) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> selected" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_selected); + } + + if ((state & ESOCK_STATE_CLOSING) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> closing" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_closing); + } + + if ((state & ESOCK_STATE_CLOSED) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> closed" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_closed); + } + + if ((state & ESOCK_STATE_DTOR) != 0) { + /* + SSDBG( descP, ("SOCKET", "esock_socket_info_state {%d} -> dror" + "\r\n", descP->sock) ); + */ + TARRAY_ADD(estate, atom_dtor); + } + + TARRAY_TOLIST(estate, env, &estateList); + + return estateList; +} +#endif // #ifndef __WIN32__ + + +/* * Collect all counters for a socket. */ #ifndef __WIN32__ @@ -4077,24 +4210,24 @@ ERL_NIF_TERM esock_socket_info_counters(ErlNifEnv* env, atom_acc_tries, atom_acc_waits}; unsigned int numKeys = NUM(keys); - ERL_NIF_TERM vals[] = {MKUI(env, descP->readByteCnt), - MKUI(env, descP->readFails), - MKUI(env, descP->readPkgCnt), - MKUI(env, descP->readPkgMax), - MKUI(env, descP->readTries), - MKUI(env, descP->readWaits), - MKUI(env, descP->writeByteCnt), - MKUI(env, descP->writeFails), - MKUI(env, descP->writePkgCnt), - MKUI(env, descP->writePkgMax), - MKUI(env, descP->writeTries), - MKUI(env, descP->writeWaits), - MKUI(env, descP->accSuccess), - MKUI(env, descP->accFails), - MKUI(env, descP->accTries), - MKUI(env, descP->accWaits)}; + ERL_NIF_TERM vals[] = {MKCNT(env, descP->readByteCnt), + MKCNT(env, descP->readFails), + MKCNT(env, descP->readPkgCnt), + MKCNT(env, descP->readPkgMax), + MKCNT(env, descP->readTries), + MKCNT(env, descP->readWaits), + MKCNT(env, descP->writeByteCnt), + MKCNT(env, descP->writeFails), + MKCNT(env, descP->writePkgCnt), + MKCNT(env, descP->writePkgMax), + MKCNT(env, descP->writeTries), + MKCNT(env, descP->writeWaits), + MKCNT(env, descP->accSuccess), + MKCNT(env, descP->accFails), + MKCNT(env, descP->accTries), + MKCNT(env, descP->accWaits)}; unsigned int numVals = NUM(vals); - ERL_NIF_TERM info; + ERL_NIF_TERM cnts; SSDBG( descP, ("SOCKET", "esock_socket_info_counters -> " "\r\n numKeys: %d" @@ -4102,13 +4235,13 @@ ERL_NIF_TERM esock_socket_info_counters(ErlNifEnv* env, "\r\n", numKeys, numVals) ); ESOCK_ASSERT( numKeys == numVals ); - ESOCK_ASSERT( MKMA(env, keys, vals, numKeys, &info) ); + ESOCK_ASSERT( MKMA(env, keys, vals, numKeys, &cnts) ); SSDBG( descP, ("SOCKET", "esock_socket_info_counters -> done with" - "\r\n info: %T" - "\r\n", info) ); + "\r\n cnts: %T" + "\r\n", cnts) ); - return info; + return cnts; } #endif // #ifndef __WIN32__ @@ -4918,18 +5051,20 @@ static BOOLEAN_T esock_open2_get_domain(ErlNifEnv* env, ERL_NIF_TERM eopts, int* domain) { - int edomain; - + ERL_NIF_TERM edomain; + SGDBG( ("SOCKET", "esock_open2_get_domain -> entry with" - "\r\n eopts: %T" - "\r\n", eopts) ); + "\r\n eopts: %T" + "\r\n", eopts) ); - if (esock_extract_int_from_map(env, eopts, - esock_atom_domain, &edomain)) { - return esock_decode_domain(env, edomain, domain); - } else { - return FALSE; - } + if (!GET_MAP_VAL(env, eopts, + esock_atom_domain, &edomain)) + return FALSE; + + if (! esock_decode_domain(env, edomain, domain)) + return FALSE; + + return TRUE; } #endif // #ifndef __WIN32__ @@ -16121,11 +16256,11 @@ BOOLEAN_T cnt_inc(ESockCounter* cnt, ESockCounter inc) ESockCounter current = *cnt; if ((max - inc) >= current) { - *cnt += inc; - wrap = FALSE; + *cnt += inc; + wrap = FALSE; } else { - *cnt = inc - (max - current) - 1; - wrap = TRUE; + *cnt = inc - (max - current) - 1; + wrap = TRUE; } return (wrap); @@ -16967,7 +17102,7 @@ ErlNifFunc esock_funcs[] = #ifndef __WIN32__ static char* extract_debug_filename(ErlNifEnv* env, - ERL_NIF_TERM map) + ERL_NIF_TERM map) { /* See the functions above */ ERL_NIF_TERM val; diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 7ddfed7d25..be98b244e5 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -677,9 +677,6 @@ void erts_replace_intr(void) { void init_break_handler(void) { sys_signal(SIGINT, request_break); -#ifndef ETHR_UNUSABLE_SIGUSRX - sys_signal(SIGUSR1, generic_signal_handler); -#endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */ sys_signal(SIGQUIT, generic_signal_handler); } @@ -694,6 +691,9 @@ void erts_sys_unix_later_init(void) { sys_signal(SIGTERM, generic_signal_handler); +#ifndef ETHR_UNUSABLE_SIGUSRX + sys_signal(SIGUSR1, generic_signal_handler); +#endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */ /* Ignore SIGCHLD to ensure orphaned processes don't turn into zombies on * death when we're pid 1. */ diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index fd47688628..6a6e0930bd 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1357,9 +1357,9 @@ id(I) -> I. %% Get code path, including the path for the erts application. get_code_path() -> - case code:lib_dir(erts) of - {error,bad_name} -> - Erts = filename:join([code:root_dir(),"erts","preloaded","ebin"]), + Erts = filename:join([code:root_dir(),"erts","preloaded","ebin"]), + case filelib:is_dir(Erts) of + true-> [Erts|code:get_path()]; _ -> code:get_path() diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index 457c84cf11..7ace75cab4 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -2792,6 +2792,13 @@ t_hashmap_balance(_Config) -> ok. hashmap_balance(KeyFun) -> + %% For uniformly distributed hash values, the average number of nodes N + %% in a hashmap varies between 0.3*K and 0.4*K where K is number of keys. + %% The standard deviation of N is about sqrt(K)/3. + + %% For simplicity we use the higher expected average 0.4*K + %% and verfies that no map is too many standard deviation above it. + F = fun(I, {M0,Max0}) -> Key = KeyFun(I), M1 = M0#{Key => Key}, @@ -2803,9 +2810,9 @@ hashmap_balance(KeyFun) -> SD_diff = abs(Nodes - Avg) / StdDev, %%io:format("~p keys: ~p nodes avg=~p SD_diff=~p\n", %% [maps:size(M1), Nodes, Avg, SD_diff]), - {MaxDiff0, _} = Max0, + {MaxDiff0, _, Cnt} = Max0, case {Nodes > Avg, SD_diff > MaxDiff0} of - {true, true} -> {SD_diff, M1}; + {true, true} -> {SD_diff, M1, Cnt+1}; _ -> Max0 end; @@ -2814,16 +2821,23 @@ hashmap_balance(KeyFun) -> {M1, Max1} end, - {_,{MaxDiff,MaxMap}} = lists:foldl(F, - {#{}, {0, 0}}, - lists:seq(1,10000)), - io:format("Max std dev diff ~p for map of size ~p (nodes=~p, flatsize=~p)\n", - [MaxDiff, maps:size(MaxMap), hashmap_nodes(MaxMap), erts_debug:flat_size(MaxMap)]), + {_,{MaxDiff,MaxMap, FatCnt}} = lists:foldl(F, + {#{}, {0, undefined, 0}}, + lists:seq(1,10000)), + case MaxMap of + undefined -> + io:format("Wow, no map with more than \"average\" number of nodes\n"); + _ -> + io:format("Found ~p maps with more than \"average\" number of nodes\n", + [FatCnt]), + io:format("Max std dev diff ~p for map of size ~p (nodes=~p, flatsize=~p)\n", + [MaxDiff, maps:size(MaxMap), hashmap_nodes(MaxMap), + erts_debug:flat_size(MaxMap)]) + end, true = (MaxDiff < 6), % The probability of this line failing is about 0.000000001 % for a uniform hash. I've set the probability this "high" for now % to detect flaws in our make_internal_hash. - % Hard limit is 15 (see hashmap_over_estimated_heap_size). ok. hashmap_nodes(M) -> diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl index 7bd8985dc7..5242d96571 100644 --- a/erts/emulator/test/os_signal_SUITE.erl +++ b/erts/emulator/test/os_signal_SUITE.erl @@ -52,14 +52,14 @@ suite() -> all() -> case os:type() of {win32, _} -> []; - _ -> [set_unset, - t_sighup, + _ -> [t_sighup, t_sigusr1, t_sigusr2, t_sigterm, t_sigalrm, t_sigchld, - t_sigchld_fork] + t_sigchld_fork, + set_unset] end. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> @@ -135,7 +135,7 @@ t_sighup(_Config) -> t_sigusr1(_Config) -> Pid1 = setup_service(), OsPid = os:getpid(), - os:set_signal(sigusr1, handle), + % shouldn't need to explicitly set_signal to handle USR1, it's the default ok = kill("USR1", OsPid), ok = kill("USR1", OsPid), ok = kill("USR1", OsPid), @@ -157,7 +157,7 @@ t_sigusr1(_Config) -> Msgs2 = fetch_msgs(Pid2), io:format("Msgs2: ~p~n", [Msgs2]), [] = Msgs2, - %% reset to ignore (it's the default) + %% reset to handle (default) so as not to break other tests os:set_signal(sigusr1, handle), ok. diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl index 15dafaf2e7..e9ace3cd99 100644 --- a/erts/emulator/test/persistent_term_SUITE.erl +++ b/erts/emulator/test/persistent_term_SUITE.erl @@ -34,6 +34,10 @@ %% -export([test_init_restart_cmd/1]). +%% Test writing helper +-export([find_colliding_keys/0]). + + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,10}}]. @@ -579,30 +583,28 @@ collisions_delete([], _) -> ok. colliding_keys() -> - %% Collisions found by Jesper L. Andersen for breaking maps. - L = [[764492191,2361333849], - [49527266765044,90940896816021,20062927283041,267080852079651], - [249858369443708,206247021789428,20287304470696,25847120931175], - [10645228898670,224705626119556,267405565521452,258214397180678], - [264783762221048,166955943492306,98802957003141,102012488332476], - [69425677456944,177142907243411,137138950917722,228865047699598], - [116031213307147,29203342183358,37406949328742,255198080174323], - [200358182338308,235207156008390,120922906095920,116215987197289], - [58728890318426,68877471005069,176496507286088,221041411345780], - [91094120814795,50665258299931,256093108116737,19777509566621], - [74646746200247,98350487270564,154448261001199,39881047281135], - [23408943649483,164410325820923,248161749770122,274558342231648], - [169531547115055,213630535746863,235098262267796,200508473898303], - [235098564415817,85039146398174,51721575960328,173069189684390], - [176136386396069,155368359051606,147817099696487,265419485459634], - [137542881551462,40028925519736,70525669519846,63445773516557], - [173854695142814,114282444507812,149945832627054,99605565798831], - [177686773562184,127158716984798,132495543008547], - [227073396444896,139667311071766,158915951283562], - [26212438434289,94902985796531,198145776057315], - [266279278943923,58550737262493,74297973216378], - [32373606512065,131854353044428,184642643042326], - [34335377662439,85341895822066,273492717750246]], + %% Collisions found by find_colliding_keys() below + L = [[77674392,148027], + [103370644,950908], + [106444046,870178], + [22217246,735880], + [18088843,694607], + [63426007,612179], + [117354942,906431], + [121434305,94282311,816072], + [118441466,93873772,783366], + [124338174,1414801,123089], + [20240282,17113486,923647], + [126495528,61463488,164994], + [125341723,5729072,445539], + [127450932,80442669,348245], + [123354692,85724182,14241288,180793], + [99159367,65959274,61680971,289939], + [107637580,104512101,62639807,181644], + [139547511,51654420,2062545,151944], + [88078274,73031465,53388204,428872], + [141314238,75761379,55699508,861797], + [88045216,59272943,21030492,180903]], %% Verify that the keys still collide (this will fail if the %% internal hash function has been changed). @@ -626,6 +628,58 @@ verify_colliding_keys([]) -> internal_hash(Term) -> erts_debug:get_internal_state({internal_hash,Term}). +%% Use this function to (re)generate the list in colliding_keys/0 +find_colliding_keys() -> + MaxCollSz = 4, + OfEachSz = 7, + erts_debug:set_internal_state(available_internal_state, true), + MaxInserts = 1 bsl 20, + T = ets:new(x, [set, private]), + ok = fck_loop_1(T, 1, MaxInserts, MaxCollSz, OfEachSz), + fck_collect(T, MaxCollSz, OfEachSz, []). + +fck_collect(_T, 1, _OfEachSz, Acc) -> + Acc; +fck_collect(T, CollSz, OfEachSz, Acc) -> + {List, _} = ets:select(T, + [{{'$1','$2'}, [{'==',{length,'$2'},CollSz}], ['$2']}], + OfEachSz), + fck_collect(T, CollSz-1, OfEachSz, List ++ Acc). + + +fck_loop_1(T, Key, 0, MaxCollSz, MaxSzLeft) -> + fck_loop_2(T, Key, MaxCollSz, MaxSzLeft); +fck_loop_1(T, Key, Inserts, MaxCollSz, MaxSzLeft) -> + Hash = internal_hash(Key), + case ets:insert_new(T, {Hash, [Key]}) of + true -> + fck_loop_1(T, Key+1, Inserts-1, MaxCollSz, MaxSzLeft); + false -> + [{Hash, KeyList}] = ets:lookup(T, Hash), + true = ets:insert(T, {Hash, [Key | KeyList]}), + fck_loop_1(T, Key+1, Inserts, MaxCollSz, MaxSzLeft) + end. + +fck_loop_2(_T, _Key, _MaxCollSz, 0) -> + ok; +fck_loop_2(T, Key, MaxCollSz, MaxSzLeft0) -> + Hash = internal_hash(Key), + case ets:lookup(T, Hash) of + [] -> + fck_loop_2(T, Key+1, MaxCollSz, MaxSzLeft0); + [{Hash, KeyList}] -> + true = ets:insert(T, {Hash, [Key | KeyList]}), + MaxSzLeft1 = case length(KeyList)+1 of + MaxCollSz -> + MaxSzLeft0 - 1; + _ -> + MaxSzLeft0 + end, + fck_loop_2(T, Key+1, MaxCollSz, MaxSzLeft1) + end. + + + %% Test that all persistent terms are erased by init:restart/0. init_restart(_Config) -> diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl index 206d2c1bfc..986d6e335e 100644 --- a/erts/emulator/test/sensitive_SUITE.erl +++ b/erts/emulator/test/sensitive_SUITE.erl @@ -290,7 +290,15 @@ running_trace(Config) when is_list(Config) -> wait_trace(Self), {messages,Messages} = process_info(Tracer, messages), [{trace,Self,out,{sensitive_SUITE,running_trace,1}}, - {trace,Self,in,{sensitive_SUITE,running_trace,1}}] = Messages, + {trace,Self,in,{sensitive_SUITE,running_trace,1}} | Extra] = Messages, + case erlang:system_info(emu_type) of + ET when ET =:= debug; ET =:= asan -> + %% F_DBG_FORCED_TRAP in erts_try_size_code_write_permission + [{trace,Self,out,{erts_internal,trace,3}}, + {trace,Self,in,{erts_internal,trace,3}}] = Extra; + _ -> + [] = Extra + end, unlink(Tracer), exit(Tracer, kill), ok. diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl index 52cf029915..678134bf1c 100644 --- a/erts/emulator/test/trace_call_time_SUITE.erl +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -65,6 +65,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). -export([basic/1, on_and_off/1, info/1, + apply_bif_bug/1, abb_worker/1, disable_ongoing/1, pause_and_restart/1, scheduling/1, called_function/1, combo/1, bif/1, nif/1]). @@ -88,6 +89,7 @@ suite() -> all() -> [basic, on_and_off, info, pause_and_restart, scheduling, disable_ongoing, + apply_bif_bug, combo, bif, nif, called_function, dead_tracer, return_stop, catch_crash]. @@ -824,3 +826,26 @@ loop() -> Pid ! {self(), answer, erlang:apply(M, F, A)}, loop() end. + +%% OTP-17290, GH-4635 +apply_bif_bug(_Config) -> + Pid = spawn(?MODULE, abb_worker, [self()]), + erlang:trace(Pid, true, [call]), + erlang:trace_pattern({?MODULE,abb_foo,'_'}, true, [call_time]), + erlang:trace_pattern({erlang,display,1}, true, [call_time]), + Pid ! {call, erlang, display, ["Hej"]}, + receive + done -> ok + end, + erlang:trace_pattern({'_','_','_'}, false, [call_time]). + +abb_worker(Papa) -> + receive + {call, M, F, Args} -> + abb_foo(M, F, Args), + Papa ! done + end. + + +abb_foo(M,F,Args) -> + apply(M,F,Args). diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index f617736e43..adc20f5ac6 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -34,6 +34,8 @@ # You have to start beam in gdb using "run". # -rgdb Run the debug compiled emulator in gdb. # You have to start beam in gdb using "run". +# -lldb Run the debug compiled emulator in lldb. +# You have to start beam in lldb using "run". # -dump Dump the bt of all threads in a core. # -break F Run the debug compiled emulator in emacs and gdb and set break. # The session is started, i.e. "run" is already don for you. @@ -170,6 +172,10 @@ while [ $# -gt 0 ]; do shift GDB=gdb ;; + "-lldb") + shift + GDB=lldb + ;; "-break") shift GDB=gdb @@ -421,6 +427,20 @@ elif [ "x$GDB" = "xgdb" ]; then echo "source $ROOTDIR/erts/etc/unix/etp-commands" > $cmdfile # Fire up gdb in emacs... exec gdb $GDBBP -x $cmdfile $gdbcmd +elif [ "x$GDB" = "xlldb" ]; then + case "x$core" in + x) + beam_args=`$EXEC -emu_args_exit ${1+"$@"}` + lldbcmd="-- $beam_args" + ;; + *) + lldbcmd="--core ${core}" + ;; + esac + cmdfile="/tmp/.cerllldb.$$" + echo "env TERM=dumb" > $cmdfile + echo "command script import $ROOTDIR/erts/etc/unix/etp.py" >> $cmdfile + exec lldb -s $cmdfile $EMU_NAME $lldbcmd elif [ "x$GDB" = "xegdb" ]; then if [ "x$EMACS" = "x" ]; then EMACS=emacs diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 1648daefa6..ba35ee7091 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3024,51 +3024,57 @@ define etp-aux-work-flags # Args: int # if ($arg0 & 0x1) - printf " delayed-dealloc" + printf " delayed-aw-wakeup" end if ($arg0 & 0x2) - printf " delayed-dealloc-thr-prgr" + printf " delayed-dealloc" end if ($arg0 & 0x4) - printf " fix-alloc-dealloc" + printf " delayed-dealloc-thr-prgr" end if ($arg0 & 0x8) - printf " fix-alloc-lower-lim" + printf " fix-alloc-dealloc" end if ($arg0 & 0x10) - printf " later-op" + printf " fix-alloc-lower-lim" end if ($arg0 & 0x20) - printf " canceled-timers" + printf " later-op" end if ($arg0 & 0x40) - printf " canceled-timers-thr-prgr" + printf " canceled-timers" end if ($arg0 & 0x80) - printf " async-ready" + printf " canceled-timers-thr-prgr" end if ($arg0 & 0x100) - printf " async-ready-clean" + printf " async-ready" end if ($arg0 & 0x200) - printf " misc-thr-prgr" + printf " async-ready-clean" end if ($arg0 & 0x400) - printf " misc" + printf " misc-thr-prgr" end if ($arg0 & 0x800) - printf " set-tmo" + printf " misc" end if ($arg0 & 0x1000) - printf " mseg-cache-check" + printf " set-tmo" end if ($arg0 & 0x2000) + printf " mseg-cache-check" + end + if ($arg0 & 0x4000) printf " yield" end - if ($arg0 & 0x1000) + if ($arg0 & 0x8000) printf " reap-ports" end - if ($arg0 & ~0x7ff) + if ($arg0 & 0x10000) + printf " debug-wait-completed" + end + if ($arg0 & ~0x1ffff) printf " GARBAGE" end printf "\n" diff --git a/erts/etc/unix/etp.py b/erts/etc/unix/etp.py new file mode 100644 index 0000000000..ca8a15d69d --- /dev/null +++ b/erts/etc/unix/etp.py @@ -0,0 +1,652 @@ +# coding=utf-8 +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2013-2022. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +# This script was orinally written by Anthony Ramine (aka nox) in 2013 +# A lot of things have changed since then, but the same base remains. +# + +import re +import lldb +import shlex + +unquoted_atom_re = re.compile(u'^[a-zß-öø-ÿ][a-zA-Zß-öø-ÿ0-9@]*$') + +def __lldb_init_module(debugger, internal_dict): + debugger.HandleCommand('type format add -f hex Eterm') + debugger.HandleCommand('type format add -f hex BeamInstr') + debugger.HandleCommand('type summary add -F etp.eterm_summary Eterm') + debugger.HandleCommand('command script add -f etp.processes_cmd etp-processes') + debugger.HandleCommand('command script add -f etp.process_info_cmd etp-process-info') + debugger.HandleCommand('command script add -f etp.stacktrace_cmd etp-stacktrace') + debugger.HandleCommand('command script add -f etp.stackdump_cmd etp-stackdump') + debugger.HandleCommand('command script add -f etp.eterm_cmd etp') + +#################################### +## Print all processes in the system +#################################### +def processes_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = erts_proc(target) + proc_r_o = proc.GetChildMemberWithName('r').GetChildMemberWithName('o') + proc_max_ix = proc_r_o.GetChildMemberWithName('max') + proc_tab = proc_r_o.GetChildMemberWithName('tab').Cast(ProcessPtrPtr(target)) + proc_cnt = proc.GetChildMemberWithName('vola').GetChildMemberWithName('tile').GetChildMemberWithName('count').GetChildMemberWithName('counter').unsigned + invalid_proc = global_var('erts_invalid_process', target).address_of + for proc_ix in range(0, proc_max_ix.unsigned): + proc = offset(proc_ix, proc_tab).deref + if proc.unsigned != 0 and proc.unsigned != invalid_proc.unsigned: + print('---') + print(' Pix: %d' % proc_ix) + process_info(proc) + proc_cnt -= 1 + if proc_cnt == 0: + break + +############################################ +## Print process-info about a single process +############################################ +def process_info_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + process_info(proc) + +def process_info(proc): + print(' Pid: %s' % eterm(proc.GetChildMemberWithName('common').GetChildMemberWithName('id'))) + print(' State: %s' % process_state(proc)) + print(' Flags: %s' % process_flags(proc)) + current = proc.GetChildMemberWithName('current') + if current.unsigned != 0 and proc.GetChildMemberWithName('state').GetChildMemberWithName('counter').unsigned & 0x800 == 0: + print(' Current function: %s' % mfa(current)) + else: + print(' Current function: %s' % 'unknown') + i = proc.GetChildMemberWithName('i') + if i.unsigned != 0: + print(' I: %s' % eterm(i)) + else: + print(' I: %s' % 'unknown') + print(' Pointer: %#x' % proc.unsigned) + +def process_state(proc): + state = proc.GetChildMemberWithName('state').unsigned + res = '' + if state & 0x80000000: + res += "GARBAGE<0x80000000> | " + if state & 0x40000000: + res += "dirty-running-sys | " + if state & 0x20000000: + res += "dirty-running | " + if state & 0x10000000: + res += "dirty-active-sys | " + if state & 0x8000000: + res += "dirty-io-proc | " + if state & 0x4000000: + res += "dirty-cpu-proc | " + if state & 0x2000000: + res += "sig-q | " + if state & 0x1000000: + res += "off-heap-msgq | " + if state & 0x800000: + res += "delayed-sys | " + if state & 0x400000: + res += "proxy | " + proxy_process = True + else: + proxy_process = False + if state & 0x200000: + res += "running-sys | " + if state & 0x100000: + res += "active-sys | " + if state & 0x80000: + res += "sig-in-q | " + if state & 0x40000: + res += "sys-tasks | " + if state & 0x20000: + res += "garbage-collecting | " + if state & 0x10000: + res += "suspended | " + if state & 0x8000: + res += "running | " + if state & 0x4000: + res += "in-run-queue | " + if state & 0x2000: + res += "active | " + if state & 0x1000: + res += "unused | " + if state & 0x800: + res += "exiting | " + if state & 0x400: + res += "free | " + if state & 0x200: + res += "in-prq-low | " + if state & 0x100: + res += "in-prq-normal | " + if state & 0x80: + res += "in-prq-high | " + if state & 0x40: + res += "in-prq-max | " + if state & 0x30 == 0x0: + res += "prq-prio-max | " + elif state & 0x30 == 0x10: + res += "prq-prio-high | " + elif state & 0x30 == 0x20: + res += "prq-prio-normal | " + else: + res += "prq-prio-low | " + if state & 0xc == 0x0: + res += "usr-prio-max | " + elif state & 0xc == 0x4: + res += "usr-prio-high | " + elif state & 0xc == 0x8: + res += "usr-prio-normal | " + else: + res += "usr-prio-low | " + if state & 0x3 == 0x0: + res += "act-prio-max" + elif state & 0x3 == 0x1: + res += "act-prio-high" + elif state & 0x3 == 0x2: + res += "act-prio-normal" + else: + res += "act-prio-low" + return res + +def process_flags(proc): + flags = proc.GetChildMemberWithName('flags').unsigned + res = '' + if flags & ~((1 << 24)-1): + res += "GARBAGE<%#x> " % (flags & ~((1 << 24)-1)) + if flags & (1 << 22): + res += "trap-exit " + if flags & (1 << 21): + res += "hibernated " + if flags & (1 << 20): + res += "dirty-minor-gc " + if flags & (1 << 19): + res += "dirty-major-gc " + if flags & (1 << 18): + res += "dirty-gc-hibernate " + if flags & (1 << 17): + res += "dirty-cla " + if flags & (1 << 16): + res += "delayed-del-proc " + if flags & (1 << 15): + res += "have-blocked-nmsb " + if flags & (1 << 14): + res += "shdlr-onln-wait-q " + if flags & (1 << 13): + res += "delay-gc " + if flags & (1 << 12): + res += "abandoned-heap-use " + if flags & (1 << 11): + res += "disable-gc " + if flags & (1 << 10): + res += "force-gc " + if flags & (1 << 9): + res += "ets-super-user " + if flags & (1 << 8): + res += "have-blocked-msb " + if flags & (1 << 7): + res += "using-ddll " + if flags & (1 << 6): + res += "distribution " + if flags & (1 << 5): + res += "using-db " + if flags & (1 << 4): + res += "need-fullsweep " + if flags & (1 << 3): + res += "heap-grow " + if flags & (1 << 2): + res += "timo " + if flags & (1 << 1): + res += "inslpqueue " + if flags & (1 << 0): + res += "hibernate-sched " + return res + +############################################ +## Print the stacktrace of a single process +############################################ +def stacktrace_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + stackdump(proc, False) + +############################################ +## Print the stackdump of a single process +############################################ +def stackdump_cmd(debugger, command, result, internal_dict): + target = debugger.GetSelectedTarget() + proc = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(command).Cast(ProcessPtr(target)) + stackdump(proc, True) + +def stackdump(proc, dump): + stop = proc.GetChildMemberWithName('stop') + send = proc.GetChildMemberWithName('hend') + cnt = 0 + if proc.GetChildMemberWithName('state').GetChildMemberWithName('counter').unsigned & 0x8000: + print('%%%%%% WARNING: The process is currently running, so c_p->stop will not be correct') + print(F'%% Stacktrace ({send.unsigned - stop.unsigned})'); + i = proc.GetChildMemberWithName('i') + if i.unsigned != 0: + print(F'I: {eterm(i)}') + while stop.unsigned < send.unsigned: + if stop.deref.unsigned & 0x3 == 0x0 or dump: + print(F'{cnt}: {eterm(stop.deref)}') + cnt += 1 + stop = offset(1, stop) + +############################################ +## Print an eterm +############################################ +def eterm_cmd(debugger, command, result, internal_dict): + args = shlex.split(command) + target = debugger.GetSelectedTarget() + term = target.process.selected_thread.GetSelectedFrame().EvaluateExpression(args[0]).Cast(EtermPtr(target)) + if len(args) >= 2: + print(eterm(term, int(args[1]))) + else: + print(eterm(term)) + +############################################ +## Print the summary of an Eterm +############################################ +def eterm_summary(valobj, internal_dict): + if valobj.TypeIsPointerType(): + return '' + return F'{valobj.unsigned:#x} {eterm(valobj, 5)}' + +def eterm(valobj, depth = float('inf')): + val = valobj.unsigned + tag = val & 0x3 + if tag == 0x1: + return cons(valobj, depth) + elif tag == 0x2: + return boxed(valobj, depth) + elif tag == 0x3: + return imm(valobj) + elif val == 0x0: + return '<the non-value>' + elif val == 0x4: + return '<the non-value debug>' + else: + return cp(valobj) + +def cons(valobj, depth = float('inf')): + items = [] + cdr = valobj + improper = False + truncated = False + depth *= 20 + + while True: + ptr = cdr.CreateValueFromData( + "unconsed", + lldb.SBData.CreateDataFromInt(cdr.unsigned - 1), + EtermPtr(cdr.target)) + items.append((ptr.deref, depth // 20)); # Append depth, car + if ptr.deref.unsigned & 0xF == 0xF: + depth -= 1 + else: + depth -= 20 + cdr = offset(1,ptr).deref + if is_nil(cdr): + break + if cdr.unsigned & 0x1 == 0: + improper = True + break + if depth <= 1: + truncated = True + break + + if improper: + return '#ImproperList' + + ## Try to print as ascii first + chars = '' + isprintable = True + for car, car_depth in items: + if car.unsigned & 0xF == 0xF: + if car.unsigned >> 4 == 10: + chars += '\\n' + elif car.unsigned >> 4 == 9: + chars += '\\t' + else: + chars += f'{car.unsigned >> 4:c}' + else: + isprintable = False + break + isprintable = isprintable and chars.isprintable() + if isprintable: + if not truncated: + return F'"{chars}"' + else: + return F'"{chars}..."' + + ## If not printable, we print the objects + objs = [] + chars = '[' + for car, car_depth in items: + objs.append(eterm(car, car_depth)) + if not truncated: + return '[' + ','.join(objs) + ']' + else: + return '[' + ','.join(objs) + '|...]' + +def boxed(valobj, depth = float('inf')): + ptr = valobj.CreateValueFromData( + "unboxed", + lldb.SBData.CreateDataFromInt(valobj.unsigned - 2), + EtermPtr(valobj.target)) + boxed_hdr = ptr.deref.unsigned + if boxed_hdr & 0x3f == 0x00: + arity = (boxed_hdr >> 6) + terms = [] + for x in range(1, arity+1): + if depth <= 1: + terms.append('...') + break + depth -= 1 + terms.append(eterm(offset(x, ptr).deref, depth)) + res = ','.join(terms) + return F"{{{res}}}" + if boxed_hdr & 0x3c == 0x3c: + if boxed_hdr & 0xc0 == 0x0: + return "flat_map" + else: + return "hash_map" + boxed_type = (boxed_hdr >> 2) & 0xF + if boxed_type == 0xC: + return '#ExternalPid' + if boxed_type == 0xD: + return '#ExternalPort' + if boxed_type == 0x2 or boxed_type == 0x3: + return '#Bignum' + if boxed_type == 0x6: + return '#Float' + if boxed_type == 0x4: + return '#Ref' + if boxed_type == 0xE: + return '#ExternalRef' + if boxed_type == 0x5: + return '#Fun' + if boxed_type == 0x8: + return '#RefcBin' + if boxed_type == 0x9: + return '#HeapBin' + if boxed_type == 0xA: + return '#SubBin' + return F'#Boxed<{valobj.unsigned}>' + +def imm(valobj): + val = valobj.unsigned + if (val & 0x3) != 3: + return '#NotImmediate<%#x>' % val + tag = val & 0xF + if tag == 0x3: + return pid(valobj) + elif tag == 0x7: + return port(valobj) + elif tag == 0xF: + return str(val >> 4) + elif tag == 0xB: + # Immediate2 + tag2 = val & 0x3F + if tag2 == 0x0B: + return atom(valobj) + elif tag2 == 0x1B: + return F'#Catch<{val>>6:#x}>' + elif is_nil(valobj): + return '[]' + return '#UnknownImmediate<%#x>' % val + +# Continuation pointers + +def cp(valobj): + mfaptr = erts_lookup_function_info(valobj) + if mfaptr == None: + return '#Cp<%#x>' % valobj.unsigned + else: + return '#Cp<%s>' % mfa(mfaptr) + +# Pids and ports + +def pid(valobj): + val = valobj.unsigned + if (val & 0xF) == 0x3: + target = valobj.target + if etp_arch_bits(target) == 64: + if etp_big_endian(target): + data = (val >> 35) & 0x0FFFFFFF + else: + data = (val >> 4) & 0x0FFFFFFF + else: + data = pixdata2data(valobj) + return '<0.%u.%u>' % (data & 0x7FFF, (data >> 15) & 0x1FFF) + else: + return '#NotPid<%#x>' % val + +def port(valobj): + val = valobj.unsigned + if (val & 0xF) == 0x7: + target = valobj.target + if etp_arch_bits(target) == 64 and not etp_halfword(target): + if etp_big_endian(target): + data = (val >> 36) & 0x0FFFFFFF + else: + data = (val >> 4) & 0x0FFFFFFF + else: + data = pixdata2data(valobj) + return '#Port<0.%u>' % data + else: + return '#NotPort<%#x>' % val + +# Strings and atoms + +def atom(valobj): + val = valobj.unsigned + if (val & 0x3F) == 0x0B: + name = atom_name(atom_tab(valobj)) + if unquoted_atom_re.match(name): + return str(name) + else: + return quoted_name(name, "'") + else: + return '#NotAtom<%#x>' % val + +def atom_name(entry): + name = entry.GetChildMemberWithName('name') + length = entry.GetChildMemberWithName('len').unsigned + data = name.GetPointeeData(0, length).uint8s + return ''.join(map(chr, data)) + +def quoted_name(name, quote): + return quote + ''.join(map(lambda c: quoted_char(c, quote), name)) + quote + +def quoted_char(c, quote): + point = ord(c) + if c == quote: + return '\\' + quote + elif point == 0x08: + return '\\b' + elif point == 0x09: + return '\\t' + elif point == 0x0A: + return '\\n' + elif point == 0x0B: + return '\\v' + elif point == 0x0C: + return '\\f' + elif point == 0x0D: + return '\\e' + elif point >= 0x20 and point <= 0x7E or point >= 0xA0: + return c + elif (point > 0xFF): + return '#NotChar<%#x>' % c + else: + return '\\%03o' % point + +# Constants + +MI_FUNCTIONS = 13 +MI_NUM_FUNCTIONS = 0 + +def is_nil(value): + ## We handle both -5 and 0x3b as NIL values so that this script + ## works with more versions + return value.signed == -5 or value.unsigned == 0x3b + +# Types + +def Atom(target): + return target.FindFirstType('Atom') + +def Char(target): + return target.FindFirstType('char') +def CharPtr(target): + return Char(target).GetPointerType() + +def Eterm(target): + return target.FindFirstType('Eterm') +def EtermPtr(target): + return Eterm(target).GetPointerType() +def EtermPtrPtr(target): + return EtermPtr(target).GetPointerType() + +def Range(target): + return target.FindFirstType('Range') + +def BeamInstr(target): + return target.FindFirstType('BeamInstr') +def BeamInstrPtr(target): + return BeamInstr(target).GetPointerType() + +def ErtsCodeInfo(target): + return target.FindFirstType('ErtsCodeInfo') +def ErtsCodeInfoPtr(target): + return ErtsCodeInfo(target).GetPointerType() + +def Process(target): + return target.FindFirstType('Process') +def ProcessPtr(target): + return Process(target).GetPointerType() +def ProcessPtrPtr(target): + return ProcessPtr(target).GetPointerType() + +# Globals + +def erts_atom_table(target): + return global_var('erts_atom_table', target) + +def erts_proc(target): + return global_var('erts_proc', target) + +def etp_arch_bits(target): + return global_var('etp_arch_bits', target).unsigned + +def etp_big_endian(target): + return global_var('etp_endianness', target).unsigned > 0 + +def etp_halfword(target): + return False + +def the_active_code_index(target): + return global_var('the_active_code_index', target) + +# Functions + +def atom_tab(valobj): + idx = valobj.unsigned + target = valobj.target + seg = erts_atom_table(target).GetChildMemberWithName('seg_table') + slot = offset(idx >> 16, seg).deref + entry = offset(idx >> 6 & 0x3FF, slot).deref + return entry.Cast(Atom(target).GetPointerType()) + +def erts_lookup_function_info(valobj): + r = find_range(valobj) + if r is None: + return None + pc = valobj.unsigned + target = valobj.target + start = r.GetChildMemberWithName('start').Cast(EtermPtr(target)) + curr = offset(MI_FUNCTIONS, start).Cast(ErtsCodeInfoPtr(target).GetPointerType()) + prev = curr + cnt = offset(MI_NUM_FUNCTIONS, start).deref.unsigned + for x in range(0, cnt): + prev = curr + curr = offset(1, curr) + if pc < curr.deref.unsigned: + return prev.deref.GetChildMemberWithName('mfa') + return None + +def find_range(valobj): + pc = valobj.unsigned + target = valobj.target + active = the_active_code_index(target).unsigned + ranges = offset(active, global_var('r', target)) + n = ranges.GetChildMemberWithName('n').unsigned + low = ranges.GetChildMemberWithName('modules') + high = offset(n, low) + range_type = Range(target) + range_pointer_type = range_type.GetPointerType() + mid = ranges.GetChildMemberWithName('mid').Cast(range_pointer_type) + while low.unsigned < high.unsigned: + if pc < mid.GetChildMemberWithName('start').unsigned: + high = mid + elif pc > mid.GetChildMemberWithName('end').GetChildMemberWithName('counter').unsigned: + low = offset(1, mid).Cast(range_pointer_type) + else: + return mid + length = (high.unsigned - low.unsigned) // range_type.size + mid = offset(length // 2, low) + return None + +def mfa(mfa): + return '%s:%s/%d' % (eterm(mfa.GetChildMemberWithName('module')), + eterm(mfa.GetChildMemberWithName('function')), + mfa.GetChildMemberWithName('arity').unsigned) + +def pixdata2data(valobj): + pixdata = valobj.unsigned + proc = erts_proc(target) + ro = proc.GetChildMemberWithName('r').GetChildMemberWithName('o') + pix_mask = ro.GetChildMemberWithName('pix_mask').unsigned + pix_cl_mask = ro.GetChildMemberWithName('pix_cl_mask').unsigned + pix_cl_shift = ro.GetChildMemberWithName('pix_cl_shift').unsigned + pix_cli_mask = ro.GetChildMemberWithName('pix_cli_mask').unsigned + pix_cli_shift = ro.GetChildMemberWithName('pix_cli_shift').unsigned + data = pixdata & ~pix_mask + data |= (pixdata >> pix_cl_shift) & pix_cl_mask + data |= (pixdata & pix_cli_mask) << pix_cli_shift + return data + +# LLDB utils + +def global_var(name, target): + return target.FindGlobalVariables(name, 1)[0] + +def offset(i, valobj): + # print("offset(" + str(i) + ", " + str(valobj.unsigned) + ")") + val = valobj.GetChildAtIndex(i, lldb.eNoDynamicValues, True) + if valobj.TypeIsPointerType(): + return val.address_of.Cast(valobj.GetType()) + else: + return val diff --git a/erts/vsn.mk b/erts/vsn.mk index cc7958578f..ef6d146974 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 11.1.8 +VSN = 11.2 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index c4f5c7bb11..1ba1765347 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,21 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.20</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Various address sanitizer support.</p> + <p> + Own Id: OTP-16959 Aux Id: PR-2965 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.19.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 661ba04fe8..90bb824873 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -11,7 +11,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -29,12 +29,12 @@ <rev></rev> <file>write_test_chapter.xml</file> </header> - + <section> <marker id="intro"></marker> <title>Support for Test Suite Authors</title> - <p>The <seeerl marker="ct"><c>ct</c></seeerl> module provides the main + <p>The <seeerl marker="ct"><c>ct</c></seeerl> module provides the main interface for writing test cases. This includes for example, the following:</p> <list type="bulleted"> @@ -45,8 +45,8 @@ </list> <p>For details about these functions, see module <seeerl marker="ct"><c>ct</c></seeerl>.</p> - - <p>The <c>Common Test</c> application also includes other modules named + + <p>The <c>Common Test</c> application also includes other modules named <c><![CDATA[ct_<component>]]></c>, which provide various support, mainly simplified use of communication protocols such as RPC, SNMP, FTP, Telnet, and others.</p> @@ -55,10 +55,10 @@ <section> <title>Test Suites</title> - + <p>A test suite is an ordinary Erlang module that contains test cases. It is recommended that the module has a name on the form - <c>*_SUITE.erl</c>. Otherwise, the directory and auto compilation + <c>*_SUITE.erl</c>. Otherwise, the directory and auto compilation function in <c>Common Test</c> cannot locate it (at least not by default). </p> @@ -66,10 +66,10 @@ in all test suite modules. </p> - <p>Each test suite module must export function + <p>Each test suite module must export function <seemfa marker="ct_suite#Module:all/0"><c>all/0</c></seemfa>, - which returns the list of all test case groups and test cases - to be executed in that module. + which returns the list of all test case groups and test cases + to be executed in that module. </p> <p>The callback functions to be implemented by the test suite are @@ -84,16 +84,16 @@ <p>Each test suite module can contain the optional configuration functions <seemfa marker="ct_suite#Module:init_per_suite/1"><c>init_per_suite/1</c></seemfa> - and <seemfa marker="ct_suite#Module:end_per_suite/1"><c>end_per_suite/1</c></seemfa>. + and <seemfa marker="ct_suite#Module:end_per_suite/1"><c>end_per_suite/1</c></seemfa>. If the init function is defined, so must the end function be. - </p> + </p> <p>If <c>init_per_suite</c> exists, it is called initially before the test cases are executed. It typically contains initializations common - for all test cases in the suite, which are only to be performed once. - <c>init_per_suite</c> is recommended for setting up and verifying state - and environment on the System Under Test (SUT) or the <c>Common Test</c> - host node, or both, so that the test cases in the suite executes correctly. + for all test cases in the suite, which are only to be performed once. + <c>init_per_suite</c> is recommended for setting up and verifying state + and environment on the System Under Test (SUT) or the <c>Common Test</c> + host node, or both, so that the test cases in the suite executes correctly. The following are examples of initial configuration operations: </p> <list type="bulleted"> @@ -103,10 +103,10 @@ </list> <p><c>end_per_suite</c> is called as the final stage of the test suite execution - (after the last test case has finished). The function is meant to be used - for cleaning up after <c>init_per_suite</c>. + (after the last test case has finished). The function is meant to be used + for cleaning up after <c>init_per_suite</c>. </p> - + <p><c>init_per_suite</c> and <c>end_per_suite</c> execute on dedicated Erlang processes, just like the test cases do. The result of these functions is however not included in the test run statistics of successful, failed, and @@ -115,16 +115,16 @@ <p>The argument to <c>init_per_suite</c> is <c>Config</c>, that is, the same key-value list of runtime configuration data that each test case takes - as input argument. <c>init_per_suite</c> can modify this parameter with + as input argument. <c>init_per_suite</c> can modify this parameter with information that the test cases need. The possibly modified <c>Config</c> list is the return value of the function. </p> <p>If <c>init_per_suite</c> fails, all test cases in the test - suite are skipped automatically (so called <em>auto skipped</em>), + suite are skipped automatically (so called <em>auto skipped</em>), including <c>end_per_suite</c>. </p> - + <p>Notice that if <c>init_per_suite</c> and <c>end_per_suite</c> do not exist in the suite, <c>Common Test</c> calls dummy functions (with the same names) instead, so that output generated by hook functions can be saved to the log @@ -139,17 +139,20 @@ <p>Each test suite module can contain the optional configuration functions <seemfa marker="ct_suite#Module:init_per_testcase/2"><c>init_per_testcase/2</c></seemfa> - and <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase/2</c></seemfa>. + and <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase/2</c></seemfa>. If the init function is defined, so must the end function be.</p> <p>If <c>init_per_testcase</c> exists, it is called before each test case in the suite. It typically contains initialization that - must be done for each test case (analog to <c>init_per_suite</c> for the + must be done for each test case (analog to <c>init_per_suite</c> for the suite).</p> <p><c>end_per_testcase/2</c> is called after each test case has finished, enabling cleanup after <c>init_per_testcase</c>.</p> + <note><p>If <c>end_per_testcase</c> crashes, however, test results are unaffected. + At the same time, this occurrence is reported in the test execution logs.</p></note> + <p>The first argument to these functions is the name of the test case. This value can be used with pattern matching in function clauses or conditional expressions to choose different initialization and cleanup @@ -159,17 +162,17 @@ <p>The second argument is the <c>Config</c> key-value list of runtime configuration data, which has the same value as the list returned by <c>init_per_suite</c>. <c>init_per_testcase/2</c> can modify this - parameter or return it "as is". The return value of <c>init_per_testcase/2</c> + parameter or return it "as is". The return value of <c>init_per_testcase/2</c> is passed as parameter <c>Config</c> to the test case itself.</p> - + <p>The return value of <c>end_per_testcase/2</c> is ignored by the - test server, with exception of the + test server, with exception of the <seeguide marker="dependencies_chapter#save_config"><c>save_config</c></seeguide> and <c>fail</c> tuple.</p> - <p><c>end_per_testcase</c> can check if the test case was successful. - (which in turn can determine how cleanup is to be performed). - This is done by reading the value tagged with <c>tc_status</c> from + <p><c>end_per_testcase</c> can check if the test case was successful. + (which in turn can determine how cleanup is to be performed). + This is done by reading the value tagged with <c>tc_status</c> from <c>Config</c>. The value is one of the following: </p> <list type="bulleted"> @@ -178,7 +181,7 @@ </item> <item> <p><c>{failed,Reason}</c></p> - <p>where <c>Reason</c> is <c>timetrap_timeout</c>, information from <c>exit/1</c>, + <p>where <c>Reason</c> is <c>timetrap_timeout</c>, information from <c>exit/1</c>, or details of a runtime error</p></item> <item> <p><c>{skipped,Reason}</c></p> @@ -186,7 +189,7 @@ </list> <p>Function <c>end_per_testcase/2</c> is even called if a - test case terminates because of a call to + test case terminates because of a call to <seemfa marker="ct#abort_current_testcase/1"><c>ct:abort_current_testcase/1</c></seemfa>, or after a timetrap time-out. However, <c>end_per_testcase</c> then executes on a different process than the test case @@ -198,7 +201,7 @@ </p> <list type="bulleted"> <item>If <c>init_per_testcase</c> crashes (called <em>auto skipped</em>).</item> - <item>If <c>init_per_testcase</c> returns a tuple <c>{skip,Reason}</c> + <item>If <c>init_per_testcase</c> returns a tuple <c>{skip,Reason}</c> (called <em>user skipped</em>).</item> </list> <p>The test case can also be marked as failed without executing it @@ -209,7 +212,7 @@ </p></note> <p>If it is determined during execution of <c>end_per_testcase</c> that - the status of a successful test case is to be changed to failed, + the status of a successful test case is to be changed to failed, <c>end_per_testcase</c> can return the tuple <c>{fail,Reason}</c> (where <c>Reason</c> describes why the test case fails).</p> @@ -221,20 +224,20 @@ <section> <marker id="test_cases"></marker> <title>Test Cases</title> - + <p>The smallest unit that the test server is concerned with is a test case. Each test case can test many things, for example, make several calls to the same interface function with different parameters. </p> - + <p>The author can choose to put many or few tests into each test case. Some things to keep in mind follows: - </p> + </p> <list type="bulleted"> <item><p>Many small test cases tend to result in extra, and possibly duplicated code, as well as slow test execution because of - large overhead for initializations and cleanups. Avoid duplicated + large overhead for initializations and cleanups. Avoid duplicated code, for example, by using common help functions. Otherwise, the resulting suite becomes difficult to read and understand, and expensive to maintain. @@ -243,52 +246,52 @@ fails. Also, large portions of test code risk being skipped when errors occur.</p> </item> - <item><p>Readability and maintainability suffer - when test cases become too large and extensive. It is not certain - that the resulting log files reflect very well the number of tests + <item><p>Readability and maintainability suffer + when test cases become too large and extensive. It is not certain + that the resulting log files reflect very well the number of tests performed. </p></item> </list> <p>The test case function takes one argument, <c>Config</c>, which contains configuration information such as <c>data_dir</c> and - <c>priv_dir</c>. (For details about these, see section + <c>priv_dir</c>. (For details about these, see section <seeguide marker="#data_priv_dir">Data and Private Directories</seeguide>. - The value of <c>Config</c> at the time of the call, is the same + The value of <c>Config</c> at the time of the call, is the same as the return value from <c>init_per_testcase</c>, mentioned earlier. </p> - <note><p>The test case function argument <c>Config</c> is not to be + <note><p>The test case function argument <c>Config</c> is not to be confused with the information that can be retrieved from the configuration files (using <seemfa marker="ct#get_config/1"><c> ct:get_config/1/2</c></seemfa>). The test case argument <c>Config</c> - is to be used for runtime configuration of the test suite and the - test cases, while configuration files are to contain data - related to the SUT. These two types of configuration data are handled + is to be used for runtime configuration of the test suite and the + test cases, while configuration files are to contain data + related to the SUT. These two types of configuration data are handled differently.</p></note> - + <p>As parameter <c>Config</c> is a list of key-value tuples, that is, a data type called a property list, it can be handled by the <seeerl marker="stdlib:proplists"><c>proplists</c></seeerl> module. - A value can, for example, be searched for and returned with function + A value can, for example, be searched for and returned with function <seemfa marker="stdlib:proplists#get_value/2"><c>proplists:get_value/2</c></seemfa>. Also, or alternatively, the general <seeerl marker="stdlib:lists"><c>lists</c></seeerl> - module contains useful functions. Normally, the only operations - performed on <c>Config</c> is insert (adding a tuple to the head of the list) - and lookup. <c>Common Test</c> provides a simple macro named <c>?config</c>, - which returns a value of an item in <c>Config</c> given the key (exactly like + module contains useful functions. Normally, the only operations + performed on <c>Config</c> is insert (adding a tuple to the head of the list) + and lookup. <c>Common Test</c> provides a simple macro named <c>?config</c>, + which returns a value of an item in <c>Config</c> given the key (exactly like <c>proplists:get_value</c>). Example: <c>PrivDir = ?config(priv_dir, Config)</c>. </p> - <p>If the test case function crashes or exits purposely, it is considered - <em>failed</em>. If it returns a value (no matter what value), it is - considered successful. An exception to this rule is the return value - <c>{skip,Reason}</c>. If this tuple is returned, the test case is considered - skipped and is logged as such.</p> + <p>If the test case function crashes or exits purposely, it is considered + <em>failed</em>. If it returns a value (no matter what value), it is + considered successful. An exception to this rule is the return value + <c>{skip,Reason}</c>. If this tuple is returned, the test case is considered + skipped and is logged as such.</p> <p>If the test case returns the tuple <c>{comment,Comment}</c>, the case - is considered successful and <c>Comment</c> is printed in the overview - log file. This is equal to calling + is considered successful and <c>Comment</c> is printed in the overview + log file. This is equal to calling <seemfa marker="ct#comment/1"><c>ct:comment(Comment)</c></seemfa>. </p> @@ -297,13 +300,13 @@ <section> <marker id="info_function"></marker> <title>Test Case Information Function</title> - + <p>For each test case function there can be an extra function with the same name but without arguments. This is the test case - information function. It is expected to return a list of tagged + information function. It is expected to return a list of tagged tuples that specifies various properties regarding the test case. </p> - + <p>The following tags have special meaning:</p> <taglist> <tag><c>timetrap</c></tag> @@ -311,9 +314,9 @@ <p> Sets the maximum time the test case is allowed to execute. If this time is exceeded, the test case fails with - reason <c>timetrap_timeout</c>. Notice that <c>init_per_testcase</c> + reason <c>timetrap_timeout</c>. Notice that <c>init_per_testcase</c> and <c>end_per_testcase</c> are included in the timetrap time. - For details, see section + For details, see section <seeguide marker="write_test_chapter#timetraps">Timetrap Time-Outs</seeguide>. </p> </item> @@ -321,7 +324,7 @@ <item> <p> Specifies any data related to the test case. This - data can be retrieved at any time using the + data can be retrieved at any time using the <seemfa marker="ct#userdata/3"><c>ct:userdata/3</c></seemfa> utility function. </p> @@ -329,7 +332,7 @@ <tag><c>silent_connections</c></tag> <item> <p> - For details, see section + For details, see section <seeguide marker="run_test_chapter#silent_connections">Silent Connections</seeguide>. </p> </item> @@ -339,25 +342,25 @@ Specifies configuration variables required by the test case. If the required configuration variables are not found in any of the test system configuration files, the test case is - skipped.</p> + skipped.</p> <p> - A required variable can also be given a default value to - be used if the variable is not found in any configuration file. To specify - a default value, add a tuple on the form - <c>{default_config,ConfigVariableName,Value}</c> to the test case information list + A required variable can also be given a default value to + be used if the variable is not found in any configuration file. To specify + a default value, add a tuple on the form + <c>{default_config,ConfigVariableName,Value}</c> to the test case information list (the position in the list is irrelevant). </p> <p><em>Examples:</em></p> - + <pre> - testcase1() -> + testcase1() -> [{require, ftp}, {default_config, ftp, [{ftp, "my_ftp_host"}, {username, "aladdin"}, {password, "sesame"}]}}].</pre> <pre> - testcase2() -> + testcase2() -> [{require, unix_telnet, unix}, {require, {unix, [telnet, username, password]}}, {default_config, unix, [{telnet, "my_telnet_host"}, @@ -369,16 +372,16 @@ <p>For more information about <c>require</c>, see section <seeguide marker="config_file_chapter#require_config_data"> Requiring and Reading Configuration Data</seeguide> - in section External Configuration Data and function + in section External Configuration Data and function <seemfa marker="ct#require/1"><c>ct:require/1/2</c></seemfa>.</p> <note><p>Specifying a default value for a required variable can result in a test case always getting executed. This might not be a desired behavior.</p> - </note> - + </note> + <p>If <c>timetrap</c> or <c>require</c>, or both, is not set specifically for a particular test case, default values specified by function - <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> + <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> are used. </p> @@ -395,7 +398,7 @@ {require,interfaces}, {userdata, [{description,"System Upgrade: RpuAddition Normal RebootNode"}, - {fts,"http://someserver.ericsson.se/test_doc4711.pdf"}]} + {fts,"http://someserver.ericsson.se/test_doc4711.pdf"}]} ].</pre> </section> @@ -404,23 +407,23 @@ <marker id="suite"></marker> <title>Test Suite Information Function</title> - <p>Function <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> - can, for example, be used in a test suite module to set a default - <c>timetrap</c> value and to <c>require</c> external configuration data. + <p>Function <seemfa marker="ct_suite#Module:suite/0"><c>suite/0</c></seemfa> + can, for example, be used in a test suite module to set a default + <c>timetrap</c> value and to <c>require</c> external configuration data. If a test case, or a group information function also specifies any of the information tags, it - overrides the default values set by <c>suite/0</c>. For details, - see + overrides the default values set by <c>suite/0</c>. For details, + see <seeguide marker="#info_function">Test Case Information Function</seeguide> and <seeguide marker="#test_case_groups">Test Case Groups</seeguide>. </p> - + <p>The following options can also be specified with the suite information list:</p> <list type="bulleted"> - <item><c>stylesheet</c>, + <item><c>stylesheet</c>, see <seeguide marker="run_test_chapter#html_stylesheet">HTML Style Sheets</seeguide></item> - <item><c>userdata</c>, + <item><c>userdata</c>, see <seeguide marker="#info_function">Test Case Information Function</seeguide></item> - <item><c>silent_connections</c>, + <item><c>silent_connections</c>, see <seeguide marker="run_test_chapter#silent_connections">Silent Connections</seeguide></item> </list> @@ -435,16 +438,16 @@ {userdata,[{info,"This suite tests database transactions."}]}, {silent_connections,[telnet]}, {stylesheet,"db_testing.css"} - ].</pre> - + ].</pre> + </section> <section> <marker id="test_case_groups"></marker> <title>Test Case Groups</title> - <p>A test case group is a set of test cases sharing configuration + <p>A test case group is a set of test cases sharing configuration functions and execution properties. Test case groups are defined by - function + function <seemfa marker="ct_suite#Module:groups/0"><c>groups/0</c></seemfa> according to the following syntax:</p> <pre> @@ -461,9 +464,9 @@ TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre> <p><c>GroupName</c> is the name of the group and must be unique within - the test suite module. Groups can be nested, by including a group definition - within the <c>GroupsAndTestCases</c> list of another group. - <c>Properties</c> is the list of execution + the test suite module. Groups can be nested, by including a group definition + within the <c>GroupsAndTestCases</c> list of another group. + <c>Properties</c> is the list of execution properties for the group. The possible values are as follows:</p> <pre> Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] @@ -484,24 +487,24 @@ <tag><c>shuffle</c></tag> <item><p>The cases in the group are executed in random order.</p></item> <tag><c>repeat, repeat_until_*</c></tag> - <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the + <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the group a given number of times, or until any, or all, cases fail or succeed.</p></item> </taglist> - + <p><em>Example:</em></p> <pre> groups() -> [{group1, [parallel], [test1a,test1b]}, {group2, [shuffle,sequence], [test2a,test2b,test2c]}].</pre> <p>To specify in which order groups are to be executed (also with respect - to test cases that are not part of any group), add tuples on the form + to test cases that are not part of any group), add tuples on the form <c>{group,GroupName}</c> to the <c>all/0</c> list.</p> <p><em>Example:</em></p> <pre> all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre> - <p>Execution properties with a group tuple in - <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. + <p>Execution properties with a group tuple in + <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. These properties override those specified in the group definition (see <c>groups/0</c> earlier). This way, the same set of tests can be run, but with different properties, without having to make copies of the group @@ -544,7 +547,7 @@ {tests3, [{repeat,2}], [t3a,t3b,t3c]}]. all() -> - [{group, tests1, default, + [{group, tests1, default, [{tests2, default, [{tests3, [parallel,{repeat,100}]}]}]}].</pre> @@ -561,33 +564,33 @@ execution is immediately stopped and the remaining cases are skipped.</p> <p>Before execution of a group begins, the configuration function - <seemfa marker="ct_suite#Module:init_per_group/2"><c>init_per_group(GroupName, Config)</c></seemfa> - is called. The list of tuples returned from this function is passed to the - test cases in the usual manner by argument <c>Config</c>. - <c>init_per_group/2</c> is meant to be used for initializations common + <seemfa marker="ct_suite#Module:init_per_group/2"><c>init_per_group(GroupName, Config)</c></seemfa> + is called. The list of tuples returned from this function is passed to the + test cases in the usual manner by argument <c>Config</c>. + <c>init_per_group/2</c> is meant to be used for initializations common for the test cases in the group. After execution of the group is finished, function - <seemfa marker="ct_suite#Module:end_per_group/2"><c>end_per_group(GroupName, Config)</c></seemfa> - is called. This function is meant to be used for cleaning up after + <seemfa marker="ct_suite#Module:end_per_group/2"><c>end_per_group(GroupName, Config)</c></seemfa> + is called. This function is meant to be used for cleaning up after <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p> <p>Whenever a group is executed, if <c>init_per_group</c> and <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls dummy functions (with the same names) instead. Output generated by hook functions are saved to the log files for these dummies. - For more information, see section + For more information, see section <seeguide marker="ct_hooks_chapter#manipulating">Manipulating Tests</seeguide> in section Common Test Hooks. </p> <note><p><c>init_per_testcase/2</c> and <c>end_per_testcase/2</c> - are always called for each individual test case, no matter if the case + are always called for each individual test case, no matter if the case belongs to a group or not.</p></note> - <p>The properties for a group are always printed in the top of the HTML log + <p>The properties for a group are always printed in the top of the HTML log for <c>init_per_group/2</c>. The total execution time for a group is included at the bottom of the log for <c>end_per_group/2</c>.</p> - <p>Test case groups can be nested so sets of groups can be + <p>Test case groups can be nested so sets of groups can be configured with the same <c>init_per_group/2</c> and <c>end_per_group/2</c> functions. Nested groups can be defined by including a group definition, or a group name reference, in the test case list of another group.</p> @@ -602,7 +605,7 @@ {group5, [sequence], [test5a,test5b,test5c]}].</pre> <p>In the previous example, if <c>all/0</c> returns group name references - in the order <c>[{group,group1},{group,group3}]</c>, the order of the + in the order <c>[{group,group1},{group,group3}]</c>, the order of the configuration functions and test cases becomes the following (notice that <c>init_per_testcase/2</c> and <c>end_per_testcase/2:</c> are also always called, but not included in this example for simplification):</p> @@ -613,7 +616,7 @@ test2a(Config2), test2b(Config2) end_per_group(group2, Config2) test1b(Config1) - end_per_group(group1, Config1) + end_per_group(group1, Config1) init_per_group(group3, Config) -> Config3 init_per_group(group4, Config3) -> Config4 test4a(Config4), test4b(Config4) (**) @@ -626,36 +629,36 @@ <p>(*) The order of test case <c>test1a</c>, <c>test1b</c>, and <c>group2</c> is undefined, as <c>group1</c> has a shuffle property.</p> <p>(**) These cases are not executed in order, but in parallel.</p> - <p>Properties are not inherited from top-level groups to nested - subgroups. For instance, in the previous example, the test cases in <c>group2</c> + <p>Properties are not inherited from top-level groups to nested + subgroups. For instance, in the previous example, the test cases in <c>group2</c> are not executed in random order (which is the property of <c>group1</c>).</p> </section> <section> <title>Parallel Property and Nested Groups</title> <p>If a group has a parallel property, its test cases are spawned - simultaneously and get executed in parallel. However, a test case is not + simultaneously and get executed in parallel. However, a test case is not allowed to execute in parallel with <c>end_per_group/2</c>, which means that the time to execute a parallel group is equal to the execution time of the slowest test case in the group. A negative side effect of running test cases in parallel is that the HTML summary pages - are not updated with links to the individual test case logs until function + are not updated with links to the individual test case logs until function <c>end_per_group/2</c> for the group has finished.</p> - <p>A group nested under a parallel group starts executing in parallel - with previous (parallel) test cases (no matter what properties the nested - group has). However, as test cases are never executed in parallel with - <c>init_per_group/2</c> or <c>end_per_group/2</c> of the same group, it is - only after a nested group has finished that remaining parallel cases + <p>A group nested under a parallel group starts executing in parallel + with previous (parallel) test cases (no matter what properties the nested + group has). However, as test cases are never executed in parallel with + <c>init_per_group/2</c> or <c>end_per_group/2</c> of the same group, it is + only after a nested group has finished that remaining parallel cases in the previous group become spawned.</p> </section> <section> <title>Parallel Test Cases and I/O</title> - <p>A parallel test case has a private I/O server as its group leader. + <p>A parallel test case has a private I/O server as its group leader. (For a description of the group leader concept, see <seeapp marker="erts:index">ERTS</seeapp>). - The central I/O server process, which handles the output from + The central I/O server process, which handles the output from regular test cases and configuration functions, does not respond to I/O messages during execution of parallel groups. This is important to understand to avoid certain traps, like the following:</p> @@ -676,13 +679,13 @@ (specified by an integer) or indefinitely (specified by <c>forever</c>). The repetition can also be stopped too early if any or all cases fail or succeed, that is, if any of the properties <c>repeat_until_any_fail</c>, - <c>repeat_until_any_ok</c>, <c>repeat_until_all_fail</c>, or + <c>repeat_until_any_ok</c>, <c>repeat_until_all_fail</c>, or <c>repeat_until_all_ok</c> is used. If the basic <c>repeat</c> - property is used, status of test cases is irrelevant for the repeat + property is used, status of test cases is irrelevant for the repeat operation.</p> - + <p>The status of a subgroup can be returned (<c>ok</c> or - <c>failed</c>), to affect the execution of the group on the level above. + <c>failed</c>), to affect the execution of the group on the level above. This is accomplished by, in <c>end_per_group/2</c>, looking up the value of <c>tc_group_properties</c> in the <c>Config</c> list and checking the result of the test cases in the group. If status <c>failed</c> is to be @@ -692,9 +695,9 @@ group is to be repeated or not (unless the basic <c>repeat</c> property is used).</p> - <p>The value of <c>tc_group_properties</c> is a list of status tuples, + <p>The value of <c>tc_group_properties</c> is a list of status tuples, each with the key <c>ok</c>, <c>skipped</c>, and <c>failed</c>. The - value of a status tuple is a list with names of test cases + value of a status tuple is a list with names of test cases that have been executed with the corresponding status as result.</p> <p>The following is an example of how to return the status from a group:</p> @@ -702,7 +705,7 @@ end_per_group(_Group, Config) -> Status = ?config(tc_group_result, Config), case proplists:get_value(failed, Status) of - [] -> % no failed cases + [] -> % no failed cases {return_group_result,ok}; _Failed -> % one or more failed {return_group_result,failed} @@ -721,27 +724,27 @@ case lists:member({group_result,group2}, Failed) of true -> {return_group_result,failed}; - false -> + false -> {return_group_result,ok} - end; + end; ...</pre> - <note><p>When a test case group is repeated, the configuration - functions <c>init_per_group/2</c> and <c>end_per_group/2</c> are + <note><p>When a test case group is repeated, the configuration + functions <c>init_per_group/2</c> and <c>end_per_group/2</c> are also always called with each repetition.</p></note> </section> <section> <title>Shuffled Test Case Order</title> <p>The order in which test cases in a group are executed is under normal - circumstances the same as the order specified in the test case list + circumstances the same as the order specified in the test case list in the group definition. With property <c>shuffle</c> set, however, <c>Common Test</c> instead executes the test cases in random order.</p> <p>You can provide a seed value (a tuple of three integers) with the shuffle property <c>{shuffle,Seed}</c>. This way, the same shuffling order can be created every time the group is executed. If no seed value - is specified, <c>Common Test</c> creates a "random" seed for the shuffling operation + is specified, <c>Common Test</c> creates a "random" seed for the shuffling operation (using the return value of <c>erlang:timestamp/0</c>). The seed value is always printed to the <c>init_per_group/2</c> log file so that it can be used to recreate the same execution order in a subsequent test run.</p> @@ -750,9 +753,9 @@ reset between turns.</p></note> <p>If a subgroup is specified in a group with a <c>shuffle</c> property, - the execution order of this subgroup in relation to the test cases + the execution order of this subgroup in relation to the test cases (and other subgroups) in the group, is random. The order of the - test cases in the subgroup is however not random (unless the + test cases in the subgroup is however not random (unless the subgroup has a <c>shuffle</c> property).</p> </section> @@ -770,10 +773,10 @@ group(connection_tests) -> [{require,login_data}, {timetrap,1000}].</pre> - + <p>The group information properties override those set with the suite information function, and can in turn be overridden by test - case information properties. For a list of valid information properties + case information properties. For a list of valid information properties and more general information, see the <seeguide marker="#info_function">Test Case Information Function</seeguide>. </p> @@ -784,10 +787,10 @@ <p>Information functions can also be used for functions <c>init_per_suite</c>, <c>end_per_suite</c>, <c>init_per_group</c>, and <c>end_per_group</c>, and they work the same way as with the - <seeguide marker="#info_function">Test Case Information Function</seeguide>. - This is useful, for example, for setting timetraps and requiring - external configuration data relevant only for the configuration - function in question (without affecting properties set for groups + <seeguide marker="#info_function">Test Case Information Function</seeguide>. + This is useful, for example, for setting timetraps and requiring + external configuration data relevant only for the configuration + function in question (without affecting properties set for groups and test cases in the suite).</p> <p>The information function <c>init/end_per_suite()</c> is called for @@ -797,7 +800,7 @@ cannot be used with <c>init/end_per_testcase(TestCase, Config)</c>, as these configuration functions execute on the test case process and use the same properties as the test case (that is, the properties - set by the test case information function, <c>TestCase()</c>). For a list + set by the test case information function, <c>TestCase()</c>). For a list of valid information properties and more general information, see the <seeguide marker="#info_function">Test Case Information Function</seeguide>. </p> @@ -807,9 +810,9 @@ <marker id="data_priv_dir"></marker> <title>Data and Private Directories</title> - <p>In the data directory, <c>data_dir</c>, the test module has - its own files needed for the testing. The name of <c>data_dir</c> - is the the name of the test suite followed by <c>"_data"</c>. + <p>In the data directory, <c>data_dir</c>, the test module has + its own files needed for the testing. The name of <c>data_dir</c> + is the the name of the test suite followed by <c>"_data"</c>. For example, <c>"some_path/foo_SUITE.beam"</c> has the data directory <c>"some_path/foo_SUITE_data/"</c>. Use this directory for portability, that is, to avoid hardcoding directory names in your suite. As the data @@ -829,11 +832,11 @@ Especially if the same test cases are executed multiple times during a test run (that is, if they belong to a test case group with property <c>repeat</c>) and there is a risk that files in the private directory get - overwritten. Under these circumstances, <c>Common Test</c> can be + overwritten. Under these circumstances, <c>Common Test</c> can be configured to create one dedicated private directory per test case and execution instead. This is accomplished with the flag/option <c>create_priv_dir</c> (to be used with the - <seecom marker="ct_run"><c>ct_run</c></seecom> program, the + <seecom marker="ct_run"><c>ct_run</c></seecom> program, the <seemfa marker="ct#run_test/1"><c>ct:run_test/1</c></seemfa> function, or as test specification term). There are three possible values for this option as follows: @@ -848,19 +851,19 @@ one private directory created per test run. The two latter values tell <c>Common Test</c> to generate a unique test directory name per test case and execution. If the auto version is used, <em>all</em> - private directories are created automatically. This can become very - inefficient for test runs with many test cases or repetitions, or both. - Therefore, if the manual version is used instead, the test case must tell + private directories are created automatically. This can become very + inefficient for test runs with many test cases or repetitions, or both. + Therefore, if the manual version is used instead, the test case must tell <c>Common Test</c> to create <c>priv_dir</c> when it needs it. - It does this by calling the function + It does this by calling the function <seemfa marker="ct#make_priv_dir/0"><c>ct:make_priv_dir/0</c></seemfa>. </p> <note><p>Do not depend on the current working directory for - reading and writing data files, as this is not portable. All - scratch files are to be written in the <c>priv_dir</c> and all - data files are to be located in <c>data_dir</c>. Also, - the <c>Common Test</c> server sets the current working directory to + reading and writing data files, as this is not portable. All + scratch files are to be written in the <c>priv_dir</c> and all + data files are to be located in <c>data_dir</c>. Also, + the <c>Common Test</c> server sets the current working directory to the test case log directory at the start of every case. </p></note> @@ -871,12 +874,12 @@ <p>Each test case is executed by a dedicated Erlang process. The process is spawned when the test case starts, and terminated when - the test case is finished. The configuration functions - <c>init_per_testcase</c> and <c>end_per_testcase</c> execute on the + the test case is finished. The configuration functions + <c>init_per_testcase</c> and <c>end_per_testcase</c> execute on the same process as the test case. </p> - <p>The configuration functions <c>init_per_suite</c> and + <p>The configuration functions <c>init_per_suite</c> and <c>end_per_suite</c> execute, like test cases, on dedicated Erlang processes. </p> @@ -898,30 +901,30 @@ for a subgroup, it overrides that of its higher level groups. Timetrap values set by individual test cases (by the test case information function) override both group- and suite- level timetraps.</p> - + <p>A timetrap can also be set or reset dynamically during the - execution of a test case, or configuration function. + execution of a test case, or configuration function. This is done by calling - <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>. - This function cancels the current timetrap and starts a new one + <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>. + This function cancels the current timetrap and starts a new one (that stays active until time-out, or end of the current function).</p> - + <p>Timetrap values can be extended with a multiplier value specified at startup with option <c>multiply_timetraps</c>. It is also possible to let the test server decide to scale up timetrap time-out values - automatically. That is, if tools such as <c>cover</c> or <c>trace</c> - are running during the test. This feature is disabled by default and + automatically. That is, if tools such as <c>cover</c> or <c>trace</c> + are running during the test. This feature is disabled by default and can be enabled with start option <c>scale_timetraps</c>.</p> - + <p>If a test case needs to suspend itself for a time that also gets multipled by <c>multiply_timetraps</c> (and possibly also scaled up if - <c>scale_timetraps</c> is enabled), the function + <c>scale_timetraps</c> is enabled), the function <seemfa marker="ct#sleep/1"><c>ct:sleep/1</c></seemfa> can be used (instead of, for example, <c>timer:sleep/1</c>).</p> - - <p>A function (<c>fun/0</c> or <c>{Mod,Func,Args}</c> (MFA) tuple) can be - specified as timetrap value in the suite-, group- and test case information - function, and as argument to function + + <p>A function (<c>fun/0</c> or <c>{Mod,Func,Args}</c> (MFA) tuple) can be + specified as timetrap value in the suite-, group- and test case information + function, and as argument to function <seemfa marker="ct#timetrap/1"><c>ct:timetrap/1</c></seemfa>.</p> <p><em>Examples:</em></p> @@ -936,12 +939,12 @@ </list> <p>Before execution of the timetrap function (which is performed on a parallel, dedicated timetrap process), <c>Common Test</c> cancels - any previously set timer for the test case or configuration function. + any previously set timer for the test case or configuration function. When the timetrap function returns, the time-out is triggered, <em>unless</em> the return value is a valid timetrap time, such as an integer, or a <c>{SecMinOrHourTag,Time}</c> tuple (for details, see module - <seeerl marker="common_test">common_test</seeerl>). If a time value - is returned, a new timetrap is started to generate a time-out after + <seeerl marker="common_test">common_test</seeerl>). If a time value + is returned, a new timetrap is started to generate a time-out after the specified time.</p> <p>The user timetrap function can return a time value after a delay. @@ -952,19 +955,19 @@ <section> <marker id="logging"></marker> <title>Logging - Categories and Verbosity Levels</title> - <p><c>Common Test</c> provides the following three main functions for + <p><c>Common Test</c> provides the following three main functions for printing strings:</p> <list type="bulleted"> <item><c>ct:log(Category, Importance, Format, FormatArgs, Opts)</c></item> <item><c>ct:print(Category, Importance, Format, FormatArgs)</c></item> <item><c>ct:pal(Category, Importance, Format, FormatArgs)</c></item> </list> - <p>The <seemfa marker="ct#log/1"><c>log/1,2,3,4,5</c></seemfa> function - prints a string to the test case log file. - The <seemfa marker="ct#print/1"><c>print/1,2,3,4</c></seemfa> function + <p>The <seemfa marker="ct#log/1"><c>log/1,2,3,4,5</c></seemfa> function + prints a string to the test case log file. + The <seemfa marker="ct#print/1"><c>print/1,2,3,4</c></seemfa> function prints the string to screen. - The <seemfa marker="ct#pal/1"><c>pal/1,2,3,4</c></seemfa> function - prints the same string both to file and screen. The functions are described + The <seemfa marker="ct#pal/1"><c>pal/1,2,3,4</c></seemfa> function + prints the same string both to file and screen. The functions are described in module <seeerl marker="ct">ct</seeerl>. </p> @@ -984,7 +987,7 @@ exist in the <c>ct.hrl</c> header file. The default importance level, <c>?STD_IMPORTANCE</c> (used if argument <c>Importance</c> is not provided), is 50. This is also the importance used for standard I/O, - for example, from printouts made with <c>io:format/2</c>, + for example, from printouts made with <c>io:format/2</c>, <c>io:put_chars/1</c>, and so on.</p> <p><c>Importance</c> is compared to a verbosity level set by the @@ -1046,11 +1049,11 @@ ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</pre> ct:log(?INFO, "Info report: ~p", [Info]) ct:pal(?ERROR, "Error report: ~p", [Error])</pre> - + <p>The functions <seemfa marker="ct#set_verbosity/2"><c>ct:set_verbosity/2</c></seemfa> and <seemfa marker="ct#get_verbosity/1"><c>ct:get_verbosity/1</c></seemfa> may be used to modify and read verbosity levels during test execution.</p> - + <p>The arguments <c>Format</c> and <c>FormatArgs</c> in <c>ct:log/print/pal</c> are always passed on to the STDLIB function <c>io:format/3</c> (For details, see the <seeerl marker="stdlib:io"><c>io</c></seeerl> manual page).</p> @@ -1064,7 +1067,7 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> <p>How categories can be mapped to CSS tags is documented in section <seeguide marker="run_test_chapter#html_stylesheet">HTML Style Sheets</seeguide> in section Running Tests and Analyzing Results.</p> - + <p>Common Test will escape special HTML characters (<, > and &) in printouts to the log file made with <c>ct:pal/4</c> and <c>io:format/2</c>. In order to print strings with HTML tags to the log, use the <c>ct:log/3,4,5</c> function. The character @@ -1078,55 +1081,55 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> supported in test specifications).</p> <p>For more information about log files, see section - <seeguide marker="run_test_chapter#log_files">Log Files</seeguide> + <seeguide marker="run_test_chapter#log_files">Log Files</seeguide> in section Running Tests and Analyzing Results.</p> </section> - + <section> <title>Illegal Dependencies</title> <p>Even though it is highly efficient to write test suites with the <c>Common Test</c> framework, mistakes can be made, - mainly because of illegal dependencies. Some of the - more frequent mistakes from our own experience with running the + mainly because of illegal dependencies. Some of the + more frequent mistakes from our own experience with running the Erlang/OTP test suites follows:</p> <list type="bulleted"> <item><p>Depending on current directory, and writing there:</p> - + <p>This is a common error in test suites. It is assumed that the current directory is the same as the author used as current directory when the test case was developed. Many test cases even try to write scratch files to this directory. Instead - <c>data_dir</c> and <c>priv_dir</c> are to be used to locate + <c>data_dir</c> and <c>priv_dir</c> are to be used to locate data and for writing scratch files. </p> </item> <item><p>Depending on execution order:</p> - - <p>During development of test suites, make no assumptions on the - execution order of the test cases or suites. For example, a test - case must not assume that a server it depends on is already + + <p>During development of test suites, make no assumptions on the + execution order of the test cases or suites. For example, a test + case must not assume that a server it depends on is already started by a previous test case. Reasons for this follows: </p> <list type="bulleted"> <item>The user/operator can specify the order at will, and maybe - a different execution order is sometimes more relevant or + a different execution order is sometimes more relevant or efficient.</item> - <item>If the user specifies a whole directory of test suites - for the test, the execution order of the suites depends on - how the files are listed by the operating system, which varies + <item>If the user specifies a whole directory of test suites + for the test, the execution order of the suites depends on + how the files are listed by the operating system, which varies between systems.</item> - <item>If a user wants to run only a subset of a test suite, - there is no way one test case could successfully depend on + <item>If a user wants to run only a subset of a test suite, + there is no way one test case could successfully depend on another.</item> </list> </item> <item><p>Depending on Unix:</p> - - <p>Running Unix commands through <c>os:cmd</c> are likely + + <p>Running Unix commands through <c>os:cmd</c> are likely not to work on non-Unix platforms. </p> </item> @@ -1134,36 +1137,36 @@ ct:pal(?ERROR, "Error report: ~p", [Error])</pre> <item><p>Nested test cases:</p> <p>Starting a test case from another not only tests the same - thing twice, but also makes it harder to follow what is being + thing twice, but also makes it harder to follow what is being tested. Also, if the called test case fails for some reason, so do the caller. This way, one error gives cause to several error reports, which is to be avoided. </p> - <p>Functionality common for many test case functions can be - implemented in common help functions. If these functions are - useful for test cases across suites, put the help functions + <p>Functionality common for many test case functions can be + implemented in common help functions. If these functions are + useful for test cases across suites, put the help functions into common help modules. </p> </item> - + <item><p>Failure to crash or exit when things go wrong:</p> - + <p>Making requests without checking that the return value indicates success can be OK if the test case fails later, but it is never acceptable just to print an error - message (into the log file) and return successfully. Such test - cases do harm, as they create a false sense of security when + message (into the log file) and return successfully. Such test + cases do harm, as they create a false sense of security when overviewing the test results. </p> </item> <item><p>Messing up for subsequent test cases:</p> - + <p>Test cases are to restore as much of the execution environment as possible, so that subsequent test cases - do not crash because of their execution order. - The function - <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase</c></seemfa> + do not crash because of their execution order. + The function + <seemfa marker="ct_suite#Module:end_per_testcase/2"><c>end_per_testcase</c></seemfa> is suitable for this. </p> </item> diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 1994d75518..d1e8e2e8ba 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.19.1 +COMMON_TEST_VSN = 1.20 diff --git a/lib/compiler/doc/src/Makefile b/lib/compiler/doc/src/Makefile index e1c445662c..39c00935cb 100644 --- a/lib/compiler/doc/src/Makefile +++ b/lib/compiler/doc/src/Makefile @@ -36,6 +36,9 @@ EDOC_REF3_FILES = cerl.xml cerl_trees.xml cerl_clauses.xml XML_PART_FILES = internal.xml XML_NOTES_FILES = notes.xml +XML_INTERNAL_FILES = beam_ssa.xml + +XML_GEN_FILES = $(XML_INTERNAL_FILES:%=$(XMLDIR)/%) BOOK_FILES = book.xml @@ -43,4 +46,7 @@ XML_FILES = \ $(BOOK_FILES) $(XML_NOTES_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) +$(XMLDIR)/%.xml: ../../internal_doc/%.md $(ERL_TOP)/make/emd2exml + $(ERL_TOP)/make/emd2exml $< $@ + include $(ERL_TOP)/make/doc.mk diff --git a/lib/compiler/doc/src/internal.xml b/lib/compiler/doc/src/internal.xml index f24b363c1c..4167f43e95 100644 --- a/lib/compiler/doc/src/internal.xml +++ b/lib/compiler/doc/src/internal.xml @@ -34,5 +34,6 @@ <xi:include href="cerl.xml"/> <xi:include href="cerl_trees.xml"/> <xi:include href="cerl_clauses.xml"/> + <xi:include href="beam_ssa.xml"/> </internal> diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 4938c6920a..0d8fafb523 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,28 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.6.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug in the type optimization pass that could + yield incorrect values or cause the wrong clauses to be + executed.</p> + <p> + Own Id: OTP-17073</p> + </item> + <item> + <p>Fixed a bug in the validator that could cause it to + reject valid code.</p> + <p> + Own Id: OTP-17126 Aux Id: ERL-1471 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.6.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/internal_doc/beam_ssa.md b/lib/compiler/internal_doc/beam_ssa.md new file mode 100644 index 0000000000..29ad019194 --- /dev/null +++ b/lib/compiler/internal_doc/beam_ssa.md @@ -0,0 +1,43 @@ +Invariants on the Structure and Format of BEAM SSA +================================================== + +Function Calls +-------------- + +All function calls not in a tail call position must be followed by a +succeeded:body-instruction unless one of the following exceptions +apply: + +* The function call can statically be proven to always fail. + +* The function call is to the `erlang`-module and can statically be + proven to always succeed or fail. + +Variable Naming +--------------- + +A variable name in BEAM SSA is either an atom, a non-negative integer +or a tuple: `atom() | non_neg_integer() | {atom() | non_neg_integer(), +non_neg_integer()}`. In order to generate fresh unused variable names, +all compiler transforms maintain a counter, the `cnt`-field in the +`opt_st`-record, which is incremented each time a new variable or +label is created. In the following description the value of the +`cnt`-field is called `Cnt`. + +Due to peculiarities in the BEAM SSA code generator, a compiler +transformation unfortunately cannot just use the `cnt`-value directly +as a fresh name. There are three basic strategies for creating fresh +variable names which can by used by a compiler pass: + +1) A name can be derived from an existing name of the form `V :: + atom() | non_neg_integer()` by selecting an atom, which is unique to + the compiler pass, to form a new name `{A, V}`. The same `A` cannot + be used by strategy 3) below. + +2) A name can be derived from an existing name of the form `V :: + non_neg_integer()` by combining it with the `cnt`-field into `{V, + Cnt}`. + +3) A fresh name can be created by selecting an atom `A`, which is + unique to the compiler pass, to form the new name `{A, Cnt}`. The + same `A` cannot be used by strategy 1) above. diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 564ca27c95..2fd2655eb3 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -2912,7 +2912,7 @@ format_error({nomatch,{shadow,Line}}) -> format_error({nomatch,shadow}) -> "this clause cannot match because a previous clause always matches"; format_error({nomatch,guard}) -> - "this caluse cannot match because its guard evaluates to 'false'"; + "this clause cannot match because its guard evaluates to 'false'"; format_error({nomatch,{bit_syntax_truncated,Signess,Val,Sz}}) -> S = case Signess of signed -> "a 'signed'"; diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 421b5ac2a1..c628ece88c 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.6.6 +COMPILER_VSN = 7.6.7 diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index aea2139351..304e263f88 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,54 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix minor memory leaks in crypto ENGINE and robustify the + code.</p> + <p> + Own Id: OTP-17212</p> + </item> + <item> + <p> + The otp_test_engine no longer fails if NO_EC* is set in + the OpenSSL configuration.</p> + <p> + Own Id: OTP-17256 Aux Id: PR-4580, GH-4573 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Various address sanitizer support.</p> + <p> + Own Id: OTP-16959 Aux Id: PR-2965 </p> + </item> + <item> + <p> + EVP is now disabled for OpenSSL cryptolib versions up to + and including 1.0.2</p> + <p> + Own Id: OTP-17116 Aux Id: PR-2972 </p> + </item> + <item> + <p> + Warning for unused C function removed</p> + <p> + Own Id: OTP-17145 Aux Id: OTP-17105, PR-2872 </p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.8.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 0fb42b3993..1aaf84ba37 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.8.3 +CRYPTO_VSN = 4.9 diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 7667ab9459..eac56be316 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 4.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Correct handling of PLTs in the GUI.</p> + <p> + Own Id: OTP-17091</p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 4.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/dialyzer/test/r9c_SUITE_data/results/inets b/lib/dialyzer/test/r9c_SUITE_data/results/inets index 91da02a2bc..d098c29841 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/results/inets +++ b/lib/dialyzer/test/r9c_SUITE_data/results/inets @@ -1,6 +1,6 @@ ftp.erl:1243:5: The pattern {'ok', {N, Bytes}} can never match the type 'eof' | {'error',atom() | {'no_translation','unicode','latin1'}} | {'ok',binary() | string()} -ftp.erl:640:2: The pattern {'closed', _Why} can never match the type 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'trans_neg_compl' | 'trans_no_space' | {'error' | 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'pos_prel' | 'trans_neg_compl' | 'trans_no_space',atom() | [any()] | {'invalid_server_response',[any(),...]}} +ftp.erl:640:2: The pattern {'closed', _Why} can never match the type 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'trans_neg_compl' | 'trans_no_space' | {'error' | 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'pos_prel' | 'trans_neg_compl' | 'trans_no_space',atom() | [any()] | {'invalid_server_response',[any(),...]} | {'timeout',binary()}} http.erl:117:3: The pattern {'error', Reason} can never match the type #req_headers{connection::[45 | 97 | 101 | 105 | 107 | 108 | 112 | 118,...],content_length::[48,...],other::[{_,_}]} http.erl:138:1: Function close_session/2 will never be called http_lib.erl:286:21: The call http_lib:close('ip_comm' | {'ssl',_},any()) will never return since it differs in the 1st argument from the success typing arguments: ('http' | 'https',any()) diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 3e7cebb1c9..803d121b53 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 4.3 +DIALYZER_VSN = 4.3.1 diff --git a/lib/eldap/doc/src/notes.xml b/lib/eldap/doc/src/notes.xml index 946db5b93d..667c0075cd 100644 --- a/lib/eldap/doc/src/notes.xml +++ b/lib/eldap/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Eldap application.</p> +<section><title>Eldap 1.2.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add ability to specify size limit on ldap requests</p> + <p> + Own Id: OTP-17166 Aux Id: PR-2904 </p> + </item> + </list> + </section> + +</section> + <section><title>Eldap 1.2.8</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/eldap/vsn.mk b/lib/eldap/vsn.mk index 8f969e6945..f0b9745f08 100644 --- a/lib/eldap/vsn.mk +++ b/lib/eldap/vsn.mk @@ -1 +1 @@ -ELDAP_VSN = 1.2.8 +ELDAP_VSN = 1.2.9 diff --git a/lib/erl_docgen/doc/src/inline_tags.xml b/lib/erl_docgen/doc/src/inline_tags.xml index d56eac41ca..2be4fa6755 100644 --- a/lib/erl_docgen/doc/src/inline_tags.xml +++ b/lib/erl_docgen/doc/src/inline_tags.xml @@ -122,8 +122,8 @@ <p>Points to an Erlang module or a custom <seeguide marker="#markerTAG">marker</seeguide> within a module. Example:</p> <pre><![CDATA[ -<seeerl marker="stdlib:string">string(3)</seemfa>, -<seeerl marker="stdlib:string#oldapi">Old API in string</seemfa> +<seeerl marker="stdlib:string">string(3)</seeerl>, +<seeerl marker="stdlib:string#oldapi">Old API in string</seeerl> ]]></pre> results in: <seeerl marker="stdlib:string">string(3)</seeerl>,<seeerl marker="stdlib:string#oldapi">Old API in string</seeerl>. </item> diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index 41aba1dbe9..e4dd926a62 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -919,18 +919,26 @@ <xsl:with-param name="chapnum" select="$chapnum"/> </xsl:call-template> </xsl:if> - <xsl:if test="(local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)"> - <!-- .../internal or .../internal/chapter --> - <xsl:call-template name="menu.internal.ug"> - <xsl:with-param name="chapnum" select="$chapnum"/> - </xsl:call-template> - </xsl:if> - <xsl:if test="(local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal)"> - <!-- .../internal,.../internal/erlref, .../internal/comref or .../internal/cref or .../internal/fileref or .../internal/appref --> - <xsl:call-template name="menu.internal.ref"> - <xsl:with-param name="curModule" select="$curModule"/> - </xsl:call-template> - </xsl:if> + + <xsl:choose> + <!-- Ugly hack to avoid two menus when we have internal documentation covering both modules and chapters --> + <xsl:when test="((local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)) and ((local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal))"> + <xsl:call-template name="menu.internal.ug_ref"> + </xsl:call-template> + </xsl:when> + <xsl:when test="(local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)"> + <!-- .../internal or .../internal/chapter --> + <xsl:call-template name="menu.internal.ug"> + <xsl:with-param name="chapnum" select="$chapnum"/> + </xsl:call-template> + </xsl:when> + <xsl:when test="(local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal)"> + <!-- .../internal,.../internal/erlref, .../internal/comref or .../internal/cref or .../internal/fileref or .../internal/appref --> + <xsl:call-template name="menu.internal.ref"> + <xsl:with-param name="curModule" select="$curModule"/> + </xsl:call-template> + </xsl:when> + </xsl:choose> <xsl:if test="(local-name() = 'application') or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::application)"> <!-- .../application,.../application/erlref, .../application/comref or .../application/cref or .../application/fileref or .../application/appref --> <xsl:call-template name="menu.ref"> @@ -1069,7 +1077,9 @@ </xsl:for-each> <!-- xsl:value-of select="$partnum"/>.<xsl:value-of select="$chapnum"/>.<xsl:value-of select="$sectnum"/>.<xsl:number/ --> <xsl:call-template name="title_link"> - <xsl:with-param name="title" select="title"/> + <xsl:with-param name="title"> + <xsl:value-of select="title"/> + </xsl:with-param> </xsl:call-template> </h4> <xsl:apply-templates> @@ -1102,7 +1112,9 @@ <xsl:call-template name="marker-before-title"/> </xsl:for-each> <xsl:call-template name="title_link"> - <xsl:with-param name="title" select="title"/> + <xsl:with-param name="title"> + <xsl:value-of select="title"/> + </xsl:with-param> </xsl:call-template> </h3> <div class="REFBODY rb-3"> @@ -1479,6 +1491,39 @@ </div> </xsl:template> + <!-- Menu.internal.chapter combined when we have both modules and free-form chapters --> + <xsl:template name="menu.internal.ug_ref"> + <div id="leftnav"> + <div class="leftnav-tube"> + + <xsl:call-template name="erlang_logo"/> + <p class="section-title"><xsl:value-of select="/book/header/title"/></p> + <p class="section-subtitle">Internal Documentation</p> + <p class="section-version">Version <xsl:value-of select="$appver"/></p> + + <xsl:call-template name="menu_top"/> + + <xsl:call-template name="menu_middle"/> + + <h3>Chapters</h3> + + <ul class="flipMenu" imagepath="{$topdocdir}/js/flipmenu"> + <xsl:call-template name="menu.chapter"> + <xsl:with-param name="entries" select="/book/internals/internal/chapter[header/title]"/> + </xsl:call-template> + </ul> + + <h3>Modules</h3> + + <ul class="flipMenu"> + <xsl:call-template name="menu.ref2"> + <xsl:with-param name="entries" select="/book/internals/internal/erlref[module]|/book/internals/internal/cref[lib]|/book/internals/internal/comref[com]|/book/internals/internal/fileref[file]|/book/internals/internal/appref[app]"/> + <!--xsl:with-param name="genFuncMenu" select="true"/--> + </xsl:call-template> + </ul> + </div> + </div> + </xsl:template> <!--Users Guide --> diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index ac2255b3d4..fede6606e2 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -2186,15 +2186,13 @@ static int send_status(ei_socket_callbacks *cbs, void *ctx, if (err) { EI_TRACE_ERR2("send_status","-> SEND_STATUS socket write failed: %s (%d)", estr(err), err); - if (buf != dbuf) - free(buf); EI_CONN_SAVE_ERRNO__(err); ret = -1; } else { + EI_TRACE_CONN1("send_status","-> SEND_STATUS (%s)",status); ret = 0; } - EI_TRACE_CONN1("send_status","-> SEND_STATUS (%s)",status); done: if (buf != dbuf) free(buf); @@ -2364,8 +2362,6 @@ static int send_name(ei_cnode *ec, err = EIO; if (err) { EI_TRACE_ERR0("send_name", "SEND_NAME -> socket write failed"); - if (buf != dbuf) - free(buf); EI_CONN_SAVE_ERRNO__(err); ret = -1; } @@ -2438,8 +2434,6 @@ static int send_challenge(ei_cnode *ec, err = EIO; if (err) { EI_TRACE_ERR0("send_challenge", "-> SEND_CHALLENGE socket write failed"); - if (buf != dbuf) - free(buf); EI_CONN_SAVE_ERRNO__(err); ret = -1; } @@ -2614,8 +2608,6 @@ static int send_complement(ei_cnode *ec, err = EIO; if (err) { EI_TRACE_ERR0("send_name", "SEND_NAME -> socket write failed"); - if (buf != dbuf) - free(buf); EI_CONN_SAVE_ERRNO__(err); ret = -1; } diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index 002a069a92..71f3765f21 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -95,6 +95,7 @@ start(Options) -> init(Options) -> XMLDir = proplists:get_value(dir, Options, ?XMLDIR), + ensure_xmldir(XMLDir), St = #state{verbose = proplists:get_bool(verbose, Options), xmldir = XMLDir, testsuites = []}, @@ -255,6 +256,19 @@ add_testcase_to_testsuite({error, Exception}, TestCaseTmp, TestSuite) -> testcases = [TestCase|TestSuite#testsuite.testcases] } end. +ensure_xmldir(XMLDir) -> + Steps = [ + fun filelib:ensure_dir/1, + fun file:make_dir/1], + lists:foldl(fun ensure_xmldir/2, XMLDir, Steps). + +ensure_xmldir(Fun, XMLDir) -> + case Fun(XMLDir) of + ok -> XMLDir; + {error, eexist} -> XMLDir; + {error, _Reason} = Error -> throw(Error) + end. + %% ---------------------------------------------------------------------------- %% Write a report to the XML directory. %% This function opens the report file, calls write_report_to/2 and closes the file. diff --git a/lib/eunit/test/eunit_SUITE.erl b/lib/eunit/test/eunit_SUITE.erl index e55091e8c1..df754e46c5 100644 --- a/lib/eunit/test/eunit_SUITE.erl +++ b/lib/eunit/test/eunit_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, app_test/1,appup_test/1,eunit_test/1,surefire_utf8_test/1,surefire_latin_test/1, - surefire_c0_test/1]). + surefire_c0_test/1, surefire_ensure_dir_test/1]). -include_lib("common_test/include/ct.hrl"). @@ -30,7 +30,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [app_test, appup_test, eunit_test, surefire_utf8_test, surefire_latin_test, - surefire_c0_test]. + surefire_c0_test, surefire_ensure_dir_test]. groups() -> []. @@ -76,6 +76,11 @@ surefire_c0_test(Config) when is_list(Config) -> true = lists:member($\t, Chars), ok. +surefire_ensure_dir_test(Config) when is_list(Config) -> + XMLDir = filename:join(proplists:get_value(priv_dir, Config), "c1"), + ok = eunit:test(tc0, [{report,{eunit_surefire,[{dir,XMLDir}]}}]), + ok = file:del_dir_r(XMLDir). + check_surefire(Module) -> File = "TEST-"++atom_to_list(Module)++".xml", file:delete(File), diff --git a/lib/jinterface/doc/src/notes.xml b/lib/jinterface/doc/src/notes.xml index fca0b46878..8603433fbd 100644 --- a/lib/jinterface/doc/src/notes.xml +++ b/lib/jinterface/doc/src/notes.xml @@ -31,6 +31,31 @@ </header> <p>This document describes the changes made to the Jinterface application.</p> +<section><title>Jinterface 1.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A <seeguide + marker="erts:erl_dist_protocol#new_link_protocol">new + link protocol</seeguide> has been introduced which + prevents links from ending up in an inconsistent state + where one participant considers itself linked while the + other doesn't. This bug has always existed in the + distributed case, but has since OTP 21 also existed in + the node local case since the distributed link protocol + then was adopted also for node local links. The bug + could, however, only trigger if both participants + operated on the link simultaneously.</p> + <p> + Own Id: OTP-17127</p> + </item> + </list> + </section> + +</section> + <section><title>Jinterface 1.11</title> <section><title>Improvements and New Features</title> diff --git a/lib/jinterface/vsn.mk b/lib/jinterface/vsn.mk index f253cf7431..5209c63af3 100644 --- a/lib/jinterface/vsn.mk +++ b/lib/jinterface/vsn.mk @@ -1 +1 @@ -JINTERFACE_VSN = 1.11 +JINTERFACE_VSN = 1.11.1 diff --git a/lib/kernel/doc/src/gen_tcp.xml b/lib/kernel/doc/src/gen_tcp.xml index 661784cab0..4a2767fb3c 100644 --- a/lib/kernel/doc/src/gen_tcp.xml +++ b/lib/kernel/doc/src/gen_tcp.xml @@ -472,6 +472,8 @@ do_recv(Sock, Bs) -> <p>There is no <c>send</c> call with a time-out option, use socket option <c>send_timeout</c> if time-outs are desired. See section <seeerl marker="#examples">Examples</seeerl>.</p> + <p>The return value <c>{error, {timeout, RestData}}</c> can + only be returned when <c>inet_backend = socket</c>. </p> <marker id="non_blocking_send"></marker> <note> <p>Non-blocking send.</p> @@ -488,6 +490,12 @@ do_recv(Sock, Bs) -> send timeout (as specified by the <c>send_timeout</c> option) expires (the function can hang even when using 'inet' backend if the internal buffers are full). </p> + <p>If this happens when using <c>packet =/= raw</c>, we have a partial + package written. A new package therefor <em>must not</em> be written + at this point, as there is no way for the peer to distinguish this + from the data portion of the current package. Instead, set package + to raw, send the rest data (as raw data) and then set package to + the wanted package type again. </p> </note> </desc> </func> diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index e209aacbb8..91cdf465cb 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -686,6 +686,16 @@ get_tcpi_sacked(Sock) -> </func> <func> + <name name="info" arity="1" since="@OTP-17203@"/> + <fsummary>Retreive miscellaneous information about a socket</fsummary> + <desc> + <p> + Produces a term containg miscellaneous information about a socket. + </p> + </desc> + </func> + + <func> <name name="ntoa" arity="1" since="OTP R16B02"/> <fsummary>Convert IPv6/IPV4 address to ASCII.</fsummary> <desc> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 9bfb2df9f7..479c9699b5 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,60 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 7.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The range check for compression pointers in DNS encoding + was faulty, which caused incorrect label compression + encoding for very large DNS messages; larger than about + 16 kBytes, such as AXFR responses. This more than 11 year + old bug has now been corrected.</p> + <p> + Own Id: OTP-13641 Aux Id: PR-2959 </p> + </item> + <item> + <p> + Fix of internal links in the <c>erpc</c> documentation.</p> + <p> + Own Id: OTP-17202 Aux Id: PR-4516 </p> + </item> + <item> + <p> + Fix bug where complex seq_trace tokens (that is lists, + tuples, maps etc) could becomes corrupted by the GC. The + bug was introduced in OTP-21.</p> + <p> + Own Id: OTP-17209 Aux Id: PR-3039 </p> + </item> + <item> + <p>When running Xref in the <c>modules</c> mode, the + Debugger application would show up as a depency for the + Kernel applications.</p> + <p> + Own Id: OTP-17223 Aux Id: GH-4546, PR-4554 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + <seeerl marker="erl_epmd"><c>erl_epmd</c></seeerl> (the + epmd client) will now try to reconnect to the local EPMD + if the connection is broken.</p> + <p> + Own Id: OTP-17178 Aux Id: PR-3003 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 7.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -616,6 +670,22 @@ </section> +<section><title>Kernel 6.5.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>When running Xref in the <c>modules</c> mode, the + Debugger application would show up as a depency for the + Kernel applications.</p> + <p> + Own Id: OTP-17223 Aux Id: GH-4546, PR-4554 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.5.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index b44144d88a..f0204802e3 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -88,7 +88,6 @@ nodelay | packet | packet_size | - pktoptions | priority | {raw, Protocol :: non_neg_integer(), @@ -296,7 +295,8 @@ close(S) -> -spec send(Socket, Packet) -> ok | {error, Reason} when Socket :: socket(), Packet :: iodata(), - Reason :: closed | inet:posix(). + Reason :: closed | {timeout, RestData} | inet:posix(), + RestData :: binary(). send({'$inet', GenTcpMod, _} = S, Packet) when is_atom(GenTcpMod) -> GenTcpMod:?FUNCTION_NAME(S, Packet); diff --git a/lib/kernel/src/gen_tcp_socket.erl b/lib/kernel/src/gen_tcp_socket.erl index 40822a5931..3b6315e3a3 100644 --- a/lib/kernel/src/gen_tcp_socket.erl +++ b/lib/kernel/src/gen_tcp_socket.erl @@ -125,6 +125,7 @@ connect_lookup(Address, Port, Opts, Timer) -> connect_open(Addrs, Domain, ConnectOpts, Opts, Fd, Timer, BindAddr) -> %% ?DBG({Addrs, Domain, ConnectOpts, Opts, Fd, Timer, BindAddr}), + %% %% The {netns, File} option is passed in Fd by inet:connect_options/2. %% The {debug, Bool} option is passed in Opts since it is @@ -142,10 +143,12 @@ connect_open(Addrs, Domain, ConnectOpts, Opts, Fd, Timer, BindAddr) -> end, {SocketOpts, StartOpts} = setopts_split(socket, Opts), + %% ?DBG([{socket, SocketOpts}, {start, StartOpts}]), case start_server( - Domain, ExtraOpts, - [{timeout, inet:timeout(Timer)} | start_opts(StartOpts)]) + Domain, + [{timeout, inet:timeout(Timer)} | start_opts(StartOpts)], + ExtraOpts) of {ok, Server} -> {Setopts, _} = @@ -214,9 +217,12 @@ call_bind(Server, BindAddr) -> listen(Port, Opts) -> %% ?DBG({Port, Opts}), {EinvalOpts, Opts_1} = setopts_split(einval, Opts), + %% ?DBG([{opts1, Opts_1}]), EinvalOpts =:= [] orelse exit(badarg), {Mod, Opts_2} = inet:tcp_module(Opts_1), + %% ?DBG([{mod, Mod}, {opts2, Opts_2}]), {StartOpts, Opts_3} = setopts_split(start, Opts_2), + %% ?DBG([{start_opts, StartOpts}, {opts3, Opts_3}]), case Mod:getserv(Port) of {ok, TP} -> case inet:listen_options([{port, TP} | Opts_3], Mod) of @@ -233,6 +239,7 @@ listen(Port, Opts) -> Domain = domain(Mod), %% ?DBG({Domain, BindIP}), BindAddr = bind_addr(Domain, BindIP, BindPort), + %% ?DBG([{listen_opts, ListenOpts}, {backlog, Backlog}]), listen_open( Domain, ListenOpts, StartOpts, Fd, Backlog, BindAddr) end; @@ -243,7 +250,12 @@ listen(Port, Opts) -> %% Helpers ------- listen_open(Domain, ListenOpts, Opts, Fd, Backlog, BindAddr) -> - %% ?DBG({Domain, ListenOpts, Opts, Fd, Backlog, BindAddr}), + %% ?DBG([{domain, Domain}, + %% {listen_opts, ListenOpts}, + %% {opts, Opts}, + %% {fd, Fd}, + %% {backlog, Backlog}, + %% {bind_addr, BindAddr}]), ExtraOpts = if @@ -251,20 +263,24 @@ listen_open(Domain, ListenOpts, Opts, Fd, Backlog, BindAddr) -> is_integer(Fd) -> #{fd => Fd}; %% This is an **ugly** hack. %% inet:connect_options/2 has the bad taste to use this - %% for [{netns,NS}] if that option is used... + %% for [{netns, NS}] if that option is used... %% Which is never called here, but just to keep it the same %% we use the same for listen! is_list(Fd) -> #{netns => Fd} end, {SocketOpts, StartOpts0} = setopts_split(socket, Opts), + %% ?DBG([{socket_opts, SocketOpts}, {start_opts0, StartOpts0}]), StartOpts = [{timeout, infinity} | start_opts(StartOpts0)], - case start_server(Domain, ExtraOpts, StartOpts) of + %% ?DBG([{start_opts, StartOpts}]), + case start_server(Domain, StartOpts, ExtraOpts) of {ok, Server} -> + %% ?DBG([{server_started, Server}]), {Setopts, _} = setopts_split( #{socket => [], server_read => [], server_write => []}, ListenOpts), + %% ?DBG([{setopts, Setopts}]), ErrRef = make_ref(), try ok(ErrRef, @@ -326,6 +342,7 @@ send(?module_socket(Server, Socket), Data) -> Packet =:= 2; Packet =:= 4 -> Size = iolist_size(Data), + %% ?DBG([{packet, Packet}, {data_size, Size}]), Header = <<?header(Packet, Size)>>, Result = socket_send(Socket, [Header, Data], SendTimeout), @@ -342,7 +359,20 @@ send(?module_socket(Server, Socket), Data) -> end. %% send_result(Server, Meta, Result) -> + %% ?DBG([{meta, Meta}, {send_result, Result}]), case Result of + %% We should really return the rest data rather then just ignoring it. + %% In the case we get a timeout when sending and 'send_timeout_close' + %% is set true, we close the connection and only returns the timeout. + %% Otherwise we return the full error. + {error, {timeout = Reason, RestData}} = E when is_binary(RestData) -> + case maps:get(send_timeout_close, Meta) of + true -> + close_server(Server), + {error, Reason}; + false -> + E + end; {error, Reason} -> %% To handle RestData we would have to pass %% all writes through a single process that buffers @@ -355,7 +385,7 @@ send_result(Server, Meta, Result) -> case Reason of econnreset -> case maps:get(show_econnreset, Meta) of - true -> {error, econnreset}; + true -> {error, econnreset}; false -> {error, closed} end; timeout -> @@ -456,29 +486,39 @@ peername(?module_socket(_Server, Socket)) -> getstat(?module_socket(Server, _Socket), What) when is_list(What) -> call(Server, {getstat, What}). + %% ------------------------------------------------------------------------- -info(?module_socket(_Server, Socket)) -> - socket:info(Socket). +info(?module_socket(Server, _Socket)) -> + call(Server, info). + %%% ======================================================================== %%% Socket glue code %%% -compile({inline, [socket_send/3]}). -socket_send(Socket, Opts, Timeout) -> - Result = socket:send(Socket, Opts, Timeout), +socket_send(Socket, Data, Timeout) -> + Result = socket:send(Socket, Data, Timeout), case Result of + {error, {timeout = _Reason, RestData}} = E when is_binary(RestData) -> + %% This is better then closing the socket for every timeout + %% We need to do something about this! + %% ?DBG({timeout, byte_size(RestData)}), + %% {error, Reason}; + E; {error, {_Reason, RestData}} when is_binary(RestData) -> - %% To handle RestData we would have to pass + %% To properly handle RestData we would have to pass %% all writes through a single process that buffers %% the write data, which would be a bottleneck %% %% Since send data may have been lost, and there is no room %% in this API to inform the caller, we at least close %% the socket in the write direction + %% ?DBG({_Reason, byte_size(RestData)}), {error, econnreset}; {error, Reason} -> + %% ?DBG(Reason), {error, case Reason of epipe -> econnreset; @@ -488,6 +528,7 @@ socket_send(Socket, Opts, Timeout) -> %% Can not happen for stream socket, but that %% does not show in the type spec %% - make believe a fatal connection error + %% ?DBG({ok, byte_size(RestData)}), {error, econnreset}; ok -> ok @@ -795,8 +836,8 @@ socket_inherit_opts() -> -compile({inline, [server_read_write_opts/0]}). server_read_write_opts() -> %% Common for read and write side - #{packet => raw, - packet_size => 16#4000000, % 64 MByte + #{packet => raw, + packet_size => 16#4000000, % 64 MByte show_econnreset => false}. -compile({inline, [server_read_opts/0]}). server_read_opts() -> @@ -807,9 +848,9 @@ server_read_opts() -> header => 0, deliver => term, start_opts => [], % Just to make it settable + line_delimiter => $\n, %% XXX not implemented yet - exit_on_close => true, - line_delimiter => $\n}, + exit_on_close => true}, server_read_write_opts()). -compile({inline, [server_write_opts/0]}). server_write_opts() -> @@ -837,9 +878,9 @@ meta(D) -> maps:with(maps:keys(server_write_opts()), D). %% State Machine Engine Call Interface %% Start for connect or listen - create a socket -start_server(Domain, ExtraOpts, StartOpts) -> +start_server(Domain, StartOpts, ExtraOpts) -> Owner = self(), - Arg = {open, Domain, ExtraOpts, Owner}, + Arg = {open, Domain, ExtraOpts, Owner}, case gen_statem:start(?MODULE, Arg, StartOpts) of {ok, Server} -> {ok, Server}; {error, _} = Error -> Error @@ -912,26 +953,34 @@ callback_mode() -> handle_event_function. init({open, Domain, ExtraOpts, Owner}) -> %% Listen or Connect %% + + %% ?DBG([{init, open}, + %% {domain, Domain}, {extraopts, ExtraOpts}, {owner, Owner}]), + process_flag(trap_exit, true), - OwnerMon = monitor(process, Owner), - Proto = if (Domain =:= local) -> default; true -> tcp end, - Opts = #{}, % #{debug => true}, - case socket_open(Domain, Proto, ExtraOpts, Opts) of + OwnerMon = monitor(process, Owner), + Proto = if (Domain =:= local) -> default; true -> tcp end, + Extra = #{}, % #{debug => true}, + case socket_open(Domain, Proto, ExtraOpts, Extra) of {ok, Socket} -> D = server_opts(), - ok = socket:setopt(Socket, {otp,iow}, true), - ok = socket:setopt(Socket, {otp,meta}, meta(D)), + ok = socket:setopt(Socket, {otp,iow}, true), + ok = socket:setopt(Socket, {otp,meta}, meta(D)), + %% ok = socket:setopt(Socket, {otp,rcvbuf}, {8, 8*1024}), P = #params{ socket = Socket, owner = Owner, owner_mon = OwnerMon}, {ok, connect, {P, D#{type => undefined, buffer => <<>>}}}; - {error, Reason} -> {stop, {shutdown, Reason}} + {error, Reason} -> + %% ?DBG({open_failed, Reason}), + {stop, {shutdown, Reason}} end; init({prepare, D, Owner}) -> %% Accept %% + %% ?DBG([{init, prepare}, {d, D}, {owner, Owner}]), process_flag(trap_exit, true), OwnerMon = monitor(process, Owner), P = @@ -944,21 +993,26 @@ init(Arg) -> error(badarg, [Arg]). -socket_open(Domain, Proto, #{fd := FD} = _Extra, Opts0) +socket_open(Domain, Proto, #{fd := FD} = _ExtraOpts, Extra) when is_integer(FD) andalso (FD > 0) -> - Opts = Opts0#{dup => false, + Opts = Extra#{dup => false, domain => Domain, type => stream, protocol => Proto}, + %% ?DBG([{fd, FD}, {opts, Opts}]), socket:open(FD, Opts); -socket_open(Domain, Proto, #{netns := NS} = Extra, Opts0) when is_list(NS) -> - Opts = maps:merge(Opts0, Extra), +socket_open(Domain, Proto, #{netns := NS} = ExtraOpts, Extra) + when is_list(NS) -> + Opts = maps:merge(Extra, ExtraOpts), + %% ?DBG([{netns, NS}, {opts, Opts}]), socket:open(Domain, stream, Proto, Opts); -socket_open(Domain, Proto, _Extra, Opts) -> - socket:open(Domain, stream, Proto, Opts). +socket_open(Domain, Proto, _ExtraOpts, Extra) -> + %% ?DBG([{opts, Extra}]), + socket:open(Domain, stream, Proto, Extra). -terminate(_Reason, State, P_D) -> +terminate(_Reason, State, {_P, _} = P_D) -> + %% ?DBG({_P#params.socket, State, _Reason}), case State of #controlling_process{state = OldState} -> terminate(OldState, P_D); @@ -967,6 +1021,7 @@ terminate(_Reason, State, P_D) -> end. %% terminate(State, {#params{socket = Socket} = P, D}) -> + %% ?DBG({Socket, State}), case State of 'closed' -> ok; 'closed_read' -> @@ -1047,13 +1102,16 @@ handle_event( handle_event( info, ?socket_counter_wrap(Socket, Counter), 'connected' = _State, {#params{socket = Socket} = P, D}) -> + %% ?DBG([{state, _State}, {counter, Counter}]), {keep_state, {P, wrap_counter(Counter, D)}}; handle_event( info, ?socket_counter_wrap(Socket, Counter), #recv{} = _State, {#params{socket = Socket} = P, D}) -> + %% ?DBG([{state, _State}, {counter, Counter}]), {keep_state, {P, wrap_counter(Counter, D)}}; handle_event( info, ?socket_counter_wrap(_Socket, _Counter), _State, _P_D) -> + %% ?DBG([{state, _State}, {counter, _Counter}]), {keep_state_and_data, [postpone]}; @@ -1102,6 +1160,7 @@ handle_event( %% Call: close/0 handle_event({call, From}, close, State, {P, D} = P_D) -> + %% ?DBG({P#params.socket, State}), case State of 'closed_read' -> {next_state, 'closed', P_D, @@ -1129,10 +1188,21 @@ handle_event({call, From}, {getopts, Opts}, State, {P, D}) -> %% Call: setopts/1 handle_event({call, From}, {setopts, Opts}, State, {P, D}) -> - %% ?DBG({Opts, State, D}), + %% ?DBG([{opts, Opts}, {state, State}, {d, D}]), {Result, D_1} = state_setopts(P, D, State, Opts), - %% ?DBG({Result, D_1}), - ok = socket:setopt(P#params.socket, {otp,meta}, meta(D_1)), + %% ?DBG([{result, Result}, {d1, D_1}]), + case Result of + {error, einval} -> + %% If we get this error, either the options where crap or + %% the socket is in a "bad state" (maybe its closed). + %% So, if that is the case we accept that we may not be + %% able to update the meta data. + _ = socket:setopt(P#params.socket, {otp,meta}, meta(D_1)), + ok; + _ -> + %% We should really handle this better. stop_and_reply? + ok = socket:setopt(P#params.socket, {otp,meta}, meta(D_1)) + end, Reply = {reply, From, Result}, case State of 'connected' -> @@ -1156,6 +1226,18 @@ handle_event({call, From}, {getstat, What}, State, {P, D}) -> [{reply, From, {ok, Result}}]} end; +%% Call: info/1 +handle_event({call, From}, info, State, {P, D}) -> + case State of + 'closed' -> + {keep_state_and_data, + [{reply, From, #{}}]}; + _ -> + {D_1, Result} = handle_info(P#params.socket, D), + {keep_state, {P, D_1}, + [{reply, From, Result}]} + end; + %% State: 'closed' - what is not handled above handle_event(Type, Content, 'closed' = State, P_D) -> handle_closed(Type, Content, State, P_D); @@ -1163,7 +1245,7 @@ handle_event(Type, Content, 'closed' = State, P_D) -> %% Call: shutdown/1 handle_event({call, From}, {shutdown, How} = _SHUTDOWN, State, {P, D}) -> - %% ?DBG({handle_event, call, _SHUTDOWN, State}), + %% ?DBG({P#params.socket, _SHUTDOWN, State}), case State of 'closed_read' when (How =:= read) -> %% ?DBG('already closed-read'), @@ -1181,7 +1263,7 @@ handle_event({call, From}, {shutdown, How} = _SHUTDOWN, State, {P, D}) -> {keep_state_and_data, [{reply, From, SRes}]}; {NextState, SRes} -> - %% ?DBG({'shutdown result', SRes, NextState}), + %% ?DBG({P#params.socket, 'shutdown result', SRes, NextState}), next_state( P, cleanup_close_read(P, D#{active := false}, State, closed), @@ -1326,6 +1408,7 @@ handle_event(Type, Content, #connect{} = State, P_D) -> %% Call: recv/2 - last part handle_event( {call, From}, {recv, Length, Timeout}, State, {P, D}) -> + %% ?DBG([recv, {length, Length}, {timeout, Timeout}, {state, State}]), case State of 'connected' -> handle_recv_start(P, D, From, Length, Timeout); @@ -1343,6 +1426,7 @@ handle_event( #recv{info = ?select_info(SelectRef)}, {#params{socket = Socket} = P, D}) -> %% + %% ?DBG([info, {socket, Socket}, {ref, SelectRef}]), handle_recv(P, D, []); %% handle_event( @@ -1350,12 +1434,14 @@ handle_event( #recv{info = ?select_info(SelectRef)}, {#params{socket = Socket} = P, D}) -> %% + %% ?DBG({abort, Reason}), handle_connected(P, cleanup_recv_reply(P, D, [], Reason)); %% %% Timeout on recv in non-active mode handle_event( {timeout, recv}, recv, #recv{} = State, {P, D}) -> %% + %% ?DBG({timeout, recv}), handle_connected(P, cleanup_recv(P, D, State, timeout)); %% Catch-all @@ -1435,6 +1521,7 @@ handle_closed(Type, Content, State, {P, _D}) -> handle_connect( #params{socket = Socket} = P, D, From, Addr, Timeout) -> %% + %% ?DBG([{d, D}, {addr, Addr}]), case socket:connect(Socket, Addr, nowait) of ok -> handle_connected( @@ -1486,6 +1573,7 @@ handle_connected(P, {D, ActionsR}) -> handle_connected(P, D, ActionsR). %% handle_connected(P, D, ActionsR) -> + %% ?DBG([{p, P}, {d, D}, {actions_r, ActionsR}]), case D of #{active := false} -> {next_state, 'connected', @@ -1500,6 +1588,7 @@ handle_recv_start( when Packet =:= raw, 0 < Length; Packet =:= 0, 0 < Length -> Size = iolist_size(Buffer), + %% ?DBG([{packet, Packet}, {length, Length}, {buf_sz, Size}]), if Length =< Size -> {Data, NewBuffer} = @@ -1517,11 +1606,13 @@ handle_recv_start( [{{timeout, recv}, Timeout, recv}]) end; handle_recv_start(P, D, From, _Length, Timeout) -> + %% ?DBG([{p, P}, {d, D}]), handle_recv( P, D#{recv_length => 0, recv_from => From}, [{{timeout, recv}, Timeout, recv}]). handle_recv(P, #{packet := Packet, recv_length := Length} = D, ActionsR) -> + %% ?DBG([{packet, Packet}, {recv_length, Length}]), if 0 < Length -> handle_recv_length(P, D, ActionsR, Length); @@ -1538,14 +1629,19 @@ handle_recv(P, #{packet := Packet, recv_length := Length} = D, ActionsR) -> handle_recv_peek(P, D, ActionsR, Packet) -> %% Peek Packet bytes + %% ?DBG({packet, Packet}), case D of #{buffer := Buffer} when is_list(Buffer) -> + %% ?DBG('buffer is list - condence'), Data = condense_buffer(Buffer), handle_recv_peek(P, D#{buffer := Data}, ActionsR, Packet); #{buffer := <<Data:Packet/binary, _Rest/binary>>} -> + %% ?DBG('buffer contains header'), handle_recv_peek(P, D, ActionsR, Packet, Data); #{buffer := <<ShortData/binary>>} -> N = Packet - byte_size(ShortData), + %% ?DBG({'buffer does not contain complete header', + %% Packet, N, byte_size(ShortData)}), case socket_recv_peek(P#params.socket, N) of {ok, <<FinalData/binary>>} -> handle_recv_peek( @@ -1571,10 +1667,13 @@ handle_recv_peek(P, D, ActionsR, Packet) -> handle_recv_peek(P, D, ActionsR, Packet, Data) -> <<?header(Packet, N)>> = Data, #{packet_size := PacketSize} = D, + %% ?DBG({'packet size', Packet, N, PacketSize}), if 0 < PacketSize, PacketSize < N -> + %% ?DBG({emsgsize}), handle_recv_error(P, D, ActionsR, emsgsize); true -> + %% ?DBG({'read a message'}), handle_recv_length(P, D, ActionsR, Packet + N) end. @@ -1594,6 +1693,7 @@ handle_recv_length(P, #{buffer := Buffer} = D, ActionsR, Length) -> %% is the last argument binary and D#{buffer} is not updated %% handle_recv_length(P, D, ActionsR, Length, Buffer) when 0 < Length -> + %5 ?DBG('try socket recv'), case socket_recv(P#params.socket, Length) of {ok, <<Data/binary>>} -> handle_recv_deliver( @@ -1612,22 +1712,28 @@ handle_recv_length(P, D, ActionsR, Length, Buffer) when 0 < Length -> reverse(ActionsR)}; {error, {Reason, <<Data/binary>>}} -> %% Error before all data + %% ?DBG({'recv error w rest-data', Reason, byte_size(Data)}), handle_recv_error( P, D#{buffer := [Data | Buffer]}, ActionsR, Reason); {error, Reason} -> + %% ?DBG({'recv error wo rest-data', Reason}), handle_recv_error(P, D#{buffer := Buffer}, ActionsR, Reason) end; handle_recv_length(P, D, ActionsR, _0, Buffer) -> + %% ?DBG({byte_size(Buffer)}), case Buffer of <<>> -> %% We should not need to update the buffer field here %% since the only way to get here with empty Buffer %% is when Buffer comes from the buffer field Socket = P#params.socket, + %% ?DBG({'try read some more', byte_size(Buffer)}), case socket_recv(Socket, 0) of {ok, <<Data/binary>>} -> + %% ?DBG({'got some', byte_size(Data)}), handle_recv_deliver(P, D, ActionsR, Data); {ok, {Data, SelectInfo}} -> + %% ?DBG({'got another select with data', byte_size(Data)}), case socket:cancel(Socket, SelectInfo) of ok -> handle_recv_deliver(P, D, ActionsR, Data); @@ -1635,13 +1741,16 @@ handle_recv_length(P, D, ActionsR, _0, Buffer) -> handle_recv_error(P, D, ActionsR, Reason, Data) end; {select, SelectInfo} -> + %% ?DBG({'got another select', SelectInfo}), {next_state, #recv{info = SelectInfo}, {P, D}, reverse(ActionsR)}; {error, {Reason, <<Data/binary>>}} -> + %% ?DBG({'error with data', Reason, byte_size(Data)}), handle_recv_error(P, D, ActionsR, Reason, Data); {error, Reason} -> + %% ?DBG({'error', Reason}), handle_recv_error(P, D, ActionsR, Reason) end; <<Data/binary>> -> @@ -1651,13 +1760,23 @@ handle_recv_length(P, D, ActionsR, _0, Buffer) -> handle_recv_deliver(P, D#{buffer := <<>>}, ActionsR, Data) end. -handle_recv_decode(P, #{packet_size := PacketSize} = D, ActionsR, Data) -> - case - erlang:decode_packet( - decode_packet(D), Data, - [{packet_size, PacketSize}, - {line_length, PacketSize}]) - of +handle_recv_decode(P, + #{packet := line, + line_delimiter := LineDelimiter, + packet_size := PacketSize} = D, + ActionsR, Data) -> + DecodeOpts = [{line_delimiter, LineDelimiter}, + {line_length, PacketSize}], + handle_recv_decode(P, D, + ActionsR, Data, DecodeOpts); +handle_recv_decode(P, D, ActionsR, Data) -> + handle_recv_decode(P, D, ActionsR, Data, []). + +handle_recv_decode(P, #{packet_size := PacketSize} = D, + ActionsR, Data, DecocdeOpts0) -> + %% ?DBG([{packet_sz, PacketSize}, {decode_opts0, DecocdeOpts0}]), + DecodeOpts = [{packet_size, PacketSize}|DecocdeOpts0], + case erlang:decode_packet(decode_packet(D), Data, DecodeOpts) of {ok, Decoded, Rest} -> %% is_list(Buffer) -> try to decode first %% is_binary(Buffer) -> get more data first @@ -1714,17 +1833,21 @@ handle_recv_error_decode( handle_recv_more(P, D, ActionsR, BufferedData) -> case socket_recv(P#params.socket, 0) of {ok, <<MoreData/binary>>} -> - Data = catbin(BufferedData, MoreData), + %% ?DBG([{more_data_sz, byte_size(MoreData)}]), + Data = catbin(BufferedData, MoreData), handle_recv_decode(P, D, ActionsR, Data); {select, SelectInfo} -> + %% ?DBG([{select_info, SelectInfo}]), {next_state, #recv{info = SelectInfo}, {P, D#{buffer := BufferedData}}, reverse(ActionsR)}; {error, {Reason, <<MoreData/binary>>}} -> + %% ?DBG({P#params.socket, error, Reason, byte_size(MoreData)}), Data = catbin(BufferedData, MoreData), handle_recv_error_decode(P, D, ActionsR, Reason, Data); {error, Reason} -> + %% ?DBG({P#params.socket, error, Reason}), handle_recv_error( P, D#{buffer := BufferedData}, ActionsR, Reason) end. @@ -1740,7 +1863,7 @@ handle_recv_error(P, D, ActionsR, Reason, Data) -> handle_recv_error(P, D_1, ActionsR_1, Reason). %% handle_recv_error(P, D, ActionsR, Reason) -> -%%% ?DBG(Reason), + %% ?DBG({P#params.socket, Reason}), {D_1, ActionsR_1} = cleanup_recv_reply(P, D#{buffer := <<>>}, ActionsR, Reason), case Reason of @@ -1762,6 +1885,7 @@ next_state(P, {D, ActionsR}, State, Actions) -> {next_state, State, {P, D}, reverse(ActionsR, Actions)}. cleanup_close_read(P, D, State, Reason) -> + %% ?DBG({P#params.socket, State, Reason}), case State of #accept{ info = SelectInfo, from = From, listen_socket = ListenSocket} -> @@ -1777,6 +1901,7 @@ cleanup_close_read(P, D, State, Reason) -> end. cleanup_recv(P, D, State, Reason) -> + %% ?DBG({P#params.socket, State, Reason}), case State of #recv{info = SelectInfo} -> socket_cancel(P#params.socket, SelectInfo), @@ -1787,26 +1912,37 @@ cleanup_recv(P, D, State, Reason) -> cleanup_recv_reply( P, #{show_econnreset := ShowEconnreset} = D, ActionsR, Reason) -> + %% ?DBG({ShowEconnreset, Reason}), case D of #{active := false} -> ok; #{active := _} -> ModuleSocket = module_socket(P), Owner = P#params.owner, -%%% ?DBG({ModuleSocket, Reason}), + %% ?DBG({ModuleSocket, Reason}), case Reason of timeout -> + %% ?DBG({P#params.socket, 'timeout'}), Owner ! {tcp_error, ModuleSocket, Reason}, ok; + closed when (ShowEconnreset =:= true) -> + %% ?DBG({P#params.socket, 'closed'}), + %% Time to bug-compatible with the inet-driver... + Owner ! {tcp_error, ModuleSocket, econnreset}, + Owner ! {tcp_closed, ModuleSocket}, + ok; closed -> + %% ?DBG({P#params.socket, 'closed'}), Owner ! {tcp_closed, ModuleSocket}, ok; emsgsize -> Owner ! {tcp_error, ModuleSocket, Reason}, ok; - econnreset when ShowEconnreset =:= false -> + econnreset when (ShowEconnreset =:= false) -> + %% ?DBG({P#params.socket, 'do not show econnreset'}), Owner ! {tcp_closed, ModuleSocket}, ok; _ -> + %% ?DBG({P#params.socket, 'show econnreset'}), Owner ! {tcp_error, ModuleSocket, Reason}, Owner ! {tcp_closed, ModuleSocket}, ok @@ -1818,6 +1954,7 @@ cleanup_recv_reply( Reason_1 = case Reason of econnreset when ShowEconnreset =:= false -> closed; + closed when ShowEconnreset =:= true -> econnreset; _ -> Reason end, [{reply, From, {error, Reason_1}}, @@ -1849,6 +1986,9 @@ recv_data_deliver( packet := Packet} = D, ActionsR, Data) -> %% + %% ?DBG([{owner, Owner}, + %% {mode, Mode}, + %% {header, Header}, {deliver, Deliver}, {packet, Packet}]), DeliverData = deliver_data(Data, Mode, Header, Packet), case D of #{recv_from := From} -> @@ -1861,6 +2001,7 @@ recv_data_deliver( {recv_stop(next_packet(D_1, Packet, Data)), ActionsR}; #{active := Active} -> + %% ?DBG({active, Active}), ModuleSocket = module_socket(P), Owner ! case Deliver of @@ -1869,6 +2010,7 @@ recv_data_deliver( port -> {ModuleSocket, {data, DeliverData}} end, + %% ?DBG('package delivered'), case Active of true -> {recv_start(next_packet(D, Packet, Data)), @@ -1982,7 +2124,7 @@ tag(Packet) -> state_setopts(_P, D, _State, []) -> {ok, D}; state_setopts(P, D, State, [Opt | Opts]) -> - %% ?DBG({entry, State, Opt}), + %% ?DBG([{state, State}, {opt, Opt}]), Opt_1 = conv_setopt(Opt), %% ?DBG({'option converted', Opt_1}), case setopt_categories(Opt_1) of @@ -2048,6 +2190,7 @@ state_setopts_server(P, D, State, Opts, {Tag, Value}) -> {{error, einval}, D} end; _ -> + %% ?DBG([{tag, Tag}, {value, Value}]), state_setopts(P, D#{Tag => Value}, State, Opts) end. @@ -2132,7 +2275,22 @@ state_getopts(P, D, State, [Tag | Tags], Acc) -> {error, einval} end. - +handle_info(Socket, D) -> + %% Read counters + Counters_1 = socket_info_counters(Socket), + %% Check for recent wraps + {D_1, Wrapped} = receive_counter_wrap(Socket, D, []), + %% + %% Assumption: a counter that we just now got a wrap message from + %% will not wrap again before we read the updated value + %% + %% Update wrapped counters + Info = #{counters := Counters_2} = socket:info(Socket), + Counters_3 = maps:merge(Counters_1, maps:with(Wrapped, Counters_2)), + %% Go ahead with wrap updated counters + Counters_4 = maps:from_list(getstat_what(D_1, Counters_3)), + {D_1, Info#{counters => Counters_4}}. + getstat(Socket, D, What) -> %% Read counters Counters_1 = socket_info_counters(Socket), @@ -2148,6 +2306,9 @@ getstat(Socket, D, What) -> %% Go ahead with wrap updated counters {D_1, getstat_what(What, D_1, Counters_3)}. +getstat_what(D, C) -> + getstat_what(inet:stats(), D, C). + getstat_what([], _D, _C) -> []; getstat_what([Tag | What], D, C) -> Val = @@ -2190,8 +2351,9 @@ socket_info_counters(Socket) -> receive_counter_wrap(Socket, D, Wrapped) -> receive ?socket_counter_wrap(Socket, Counter) -> + %% ?DBG([{counter, Counter}]), receive_counter_wrap( - Socket, wrap_counter(Counter, D) , [Counter | Wrapped]) + Socket, wrap_counter(Counter, D), [Counter | Wrapped]) after 0 -> {D, Wrapped} end. diff --git a/lib/kernel/src/group_history.erl b/lib/kernel/src/group_history.erl index f50060cefa..b2cc05f5fd 100644 --- a/lib/kernel/src/group_history.erl +++ b/lib/kernel/src/group_history.erl @@ -20,6 +20,8 @@ -module(group_history). -export([load/0, add/1]). +-include_lib("kernel/include/logger.hrl"). + %% Make a minimal size that should encompass set of lines and then make %% a file rotation for N files of this size. -define(DEFAULT_HISTORY_FILE, "erlang-shell-log"). @@ -66,7 +68,11 @@ load() -> upgrade_version(?LOG_NAME, Version), load(); {error, {badarg, size}} -> - open_new_log(?LOG_NAME); + try open_new_log(?LOG_NAME) + catch exit:_ -> + %% Same reason as comment in catch below + [] + end; {error, Reason} -> handle_open_error(Reason), disable_history(), @@ -79,33 +85,55 @@ load() -> disabled -> []; Provider -> - Provider:load() + try Provider:load() of + History when is_list(History) -> + History; + Error -> + show_custom_provider_faulty_load_return(Provider, Error), + disable_history(), + [] + catch E:R:ST -> + show_custom_provider_crash(Provider, E, R, ST), + disable_history(), + [] + end end. %% @doc adds a log line to the erlang history log, if configured to do so. -spec add(iodata()) -> ok. -add(Line) -> add(Line, history_status()). - -add(Line, enabled) -> +add(Line) -> case lists:member(Line, to_drop()) of false -> - case disk_log:log(?LOG_NAME, Line) of - ok -> - ok; - {error, no_such_log} -> - _ = open_log(), % a wild attempt we hope works! - disk_log:log(?LOG_NAME, Line); - {error, _Other} -> - % just ignore, we're too late - ok - end; + add(Line, history_status()); true -> ok + end. + +add(Line, enabled) -> + case disk_log:log(?LOG_NAME, Line) of + ok -> + ok; + {error, no_such_log} -> + _ = open_log(), % a wild attempt we hope works! + disk_log:log(?LOG_NAME, Line); + {error, _Other} -> + % just ignore, we're too late + ok end; add(_Line, disabled) -> ok; add(Line, Provider) -> - lists:member(Line, to_drop()) orelse Provider:add(Line). + try Provider:add(Line) of + ok -> + ok; + Error -> + show_custom_provider_faulty_add_return(Provider, Error), + ok + catch E:R:ST -> + show_custom_provider_crash(Provider, E, R, ST), + disable_history(), + ok + end. %%%%%%%%%%%%%%% %%% PRIVATE %%% @@ -215,6 +243,9 @@ read_full_log(Name) -> {error, no_such_log} -> show_unexpected_close_warning(), []; + {error, Reason} -> + show_invalid_chunk_warning(Name, Reason), + []; eof -> []; {Cont, Logs} -> @@ -237,6 +268,9 @@ read_full_log(Name, Cont) -> {error, no_such_log} -> show_unexpected_close_warning(), []; + {error, Reason} -> + show_invalid_chunk_warning(Name, Reason), + []; eof -> []; {NextCont, Logs} -> @@ -261,13 +295,11 @@ handle_open_error({invalid_header, Term}) -> show('$#erlang-history-invalid-header', "Shell history expects to be able to use the log files " "which currently have unknown headers (~p) and may belong to " - "another mechanism. History logging will be " - "disabled.~n", + "another mechanism.~n", [Term]); handle_open_error({file_error, FileName, Reason}) -> show('$#erlang-history-file-error', - "Error handling File ~ts. Reason: ~p~n" - "History logging will be disabled.~n", + "Error handling file ~ts. Reason: ~p~n", [FileName, Reason]); handle_open_error(Err) -> show_unexpected_warning({disk_log, open, 1}, Err). @@ -310,14 +342,14 @@ resize_log(Name, _OldSize, NewSize) -> case disk_log:change_size(Name, NewSize) of ok -> show('$#erlang-history-resize-result', - "ok~n", []); + "resized the log history file~n", []); {error, {new_size_too_small, _, _}} -> % cannot happen show('$#erlang-history-resize-result', - "failed (new size is too small)~n", []), + "failed to resize the log history file (new size is too small)~n", []), disable_history(); {error, Reason} -> show('$#erlang-history-resize-result', - "failed (~p)~n", [Reason]), + "failed to resize the log history file (~p)~n", [Reason]), disable_history() end, _ = disk_log:close(?LOG_NAME), @@ -360,12 +392,17 @@ show_rename_warning() -> "name will keep being used for this session.~n", []). +show_invalid_chunk_warning(Name, Reason) -> + show('$#erlang-history-invalid-chunk-warn', + "Invalid chunk in the file ~ts.~n" + "Some entries may have been lost.~nReason ~p.", + [proplists:get_value(file,disk_log:info(Name)), Reason]). + show_invalid_file_warning(FileName) -> show('$#erlang-history-invalid-file', "Shell history expects to be able to use the file ~ts " "which currently exists and is not a file usable for " - "history logging purposes. History logging will be " - "disabled.~n", [FileName]). + "history logging purposes.~n", [FileName]). show_unexpected_warning({M,F,A}, Term) -> show('$#erlang-history-unexpected-return', @@ -383,10 +420,29 @@ show_size_warning(_Current, _New) -> "The configured log history file size is different from " "the size of the log file on disk.~n", []). +show_custom_provider_crash(Provider, Class, Reason, StackTrace) -> + show('$#erlang-history-custom-crash', + "The configured custom shell_history provider '~p' crashed. ~n" + "Did you mean to write 'enabled'?~n" + "~ts~n", + [Provider, erl_error:format_exception(Class, Reason, StackTrace)]). + +show_custom_provider_faulty_load_return(Provider, Return) -> + show('$#erlang-history-custom-return', + "The configured custom shell_history provider ~p:load/0 did not return a list.~n" + "It returned ~p.~n", + [Provider, Return]). + +show_custom_provider_faulty_add_return(Provider, Return) -> + show('$#erlang-history-custom-return', + "The configured custom shell_history provider ~p:add/1 did not return ok.~n" + "It returned ~p.~n", + [Provider, Return]). + show(Key, Format, Args) -> case get(Key) of undefined -> - io:format(standard_error, Format, Args), + ?LOG_ERROR(Format, Args), put(Key, true), ok; true -> diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index 6ae5351652..4af31b86cc 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -31,6 +31,7 @@ getif/1, getif/0, getiflist/0, getiflist/1, ifget/3, ifget/2, ifset/3, ifset/2, getstat/1, getstat/2, + info/1, ip/1, stats/0, options/0, pushf/3, popf/1, close/1, gethostname/0, gethostname/1, parse_ipv4_address/1, parse_ipv6_address/1, parse_ipv4strict_address/1, @@ -602,6 +603,63 @@ gethostbyaddr(Address,Timeout) -> gethostbyaddr_tm(Address,Timer) -> gethostbyaddr_tm(Address, Timer, inet_db:res_option(lookup)). + +-spec info(Socket) -> Info when + Socket :: socket(), + Info :: term(). + +info({'$inet', GenSocketMod, _} = Socket) + when is_atom(GenSocketMod) -> + GenSocketMod:?FUNCTION_NAME(Socket); +info(Socket) -> + PortInfo = port_info(Socket), + States = + case prim_inet:getstatus(Socket) of + {ok, State0} -> + State0; + _ -> + #{} + end, + Stats = + case getstat(Socket) of + {ok, Stats0} -> + maps:from_list(Stats0); + _ -> + [] + end, + PortInfo#{counters => Stats, + states => States}. + +port_info(P) when is_port(P) -> + PI0 = port_info(erlang:port_info(P), + [connected, links, input, output]) ++ + [erlang:port_info(P, memory), + erlang:port_info(P, monitors)], + PI = pi_replace([{connected, owner}], PI0), + maps:from_list(PI); +port_info(_) -> + #{}. + +port_info(PI, Items) -> + port_info(PI, Items, []). + +port_info(_PI, [], Acc) -> + Acc; +port_info(PI, [Item | Items], Acc) -> + Val = proplists:get_value(Item, PI), + port_info(PI, Items, [{Item, Val} | Acc]). + +pi_replace([], Items) -> + Items; +pi_replace([{Key1, Key2}|Keys], Items) -> + case lists:keysearch(Key1, 1, Items) of + {value, {Key1, Value}} -> + Items2 = lists:keyreplace(Key1, 1, Items, {Key2, Value}), + pi_replace(Keys, Items2); + false -> + pi_replace(Keys, Items) + end. + -spec ip(Ip :: ip_address() | string() | atom()) -> {'ok', ip_address()} | {'error', posix()}. @@ -781,11 +839,13 @@ stats() -> [recv_oct, recv_cnt, recv_max, recv_avg, recv_dvi, send_oct, send_cnt, send_max, send_avg, send_pend]. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Available options for tcp:connect %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% connect_options() -> - [tos, tclass, priority, reuseaddr, keepalive, linger, sndbuf, recbuf, nodelay, + [tos, tclass, priority, reuseaddr, keepalive, linger, nodelay, + sndbuf, recbuf, recvtos, recvtclass, ttl, recvttl, header, active, packet, packet_size, buffer, mode, deliver, line_delimiter, exit_on_close, high_watermark, low_watermark, high_msgq_watermark, diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index a8490ecefa..796bec68b3 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -52,7 +52,8 @@ {<<"^7\\.1$">>,[restart_new_emulator]}, {<<"^7\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^7\\.2$">>,[restart_new_emulator]}, - {<<"^7\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^7\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^7\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^6\\.0$">>,[restart_new_emulator]}, {<<"^6\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -77,4 +78,5 @@ {<<"^7\\.1$">>,[restart_new_emulator]}, {<<"^7\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^7\\.2$">>,[restart_new_emulator]}, - {<<"^7\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^7\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^7\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index 74fa6744ee..07b7b0ef6e 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1805,7 +1805,7 @@ start_protos_listen(Name, Host, Node, [Proto | Ps], Ls, CleanHalt) -> catch error:undef -> proto_error(CleanHalt, Proto, "not supported"), start_protos_listen(Name, Host, Node, Ps, Ls, CleanHalt); - error:Reason -> + _:Reason -> register_error(CleanHalt, Proto, Reason), start_protos_listen(Name, Host, Node, Ps, Ls, CleanHalt) end; diff --git a/lib/kernel/src/pg.erl b/lib/kernel/src/pg.erl index 97741cc703..573d2e6953 100644 --- a/lib/kernel/src/pg.erl +++ b/lib/kernel/src/pg.erl @@ -281,19 +281,7 @@ handle_info({leave, Peer, PidOrPids, Groups}, #state{scope = Scope, nodes = Node case maps:get(Peer, Nodes, []) of {MRef, RemoteMap} -> _ = leave_remote(Scope, PidOrPids, Groups), - NewRemoteMap = lists:foldl( - fun (Group, Acc) -> - case maps:get(Group, Acc) of - PidOrPids -> - maps:remove(Group, Acc); - [PidOrPids] -> - maps:remove(Group, Acc); - Existing when is_pid(PidOrPids) -> - Acc#{Group => lists:delete(PidOrPids, Existing)}; - Existing -> - Acc#{Group => Existing-- PidOrPids} - end - end, RemoteMap, Groups), + NewRemoteMap = leave_update_remote_map(PidOrPids, RemoteMap, Groups), {noreply, State#state{nodes = Nodes#{Peer => {MRef, NewRemoteMap}}}}; [] -> %% Handle race condition: remote node disconnected, but scope process @@ -306,8 +294,8 @@ handle_info({leave, Peer, PidOrPids, Groups}, #state{scope = Scope, nodes = Node end; %% we're being discovered, let's exchange! -handle_info({discover, Peer}, #state{scope = Scope, nodes = Nodes} = State) -> - gen_server:cast(Peer, {sync, self(), all_local_pids(Scope)}), +handle_info({discover, Peer}, #state{nodes = Nodes, monitors = Monitors} = State) -> + gen_server:cast(Peer, {sync, self(), all_local_pids(Monitors)}), %% do we know who is looking for us? case maps:is_key(Peer, Nodes) of true -> @@ -524,9 +512,33 @@ leave_remote(Scope, Pids, Groups) -> end || Group <- Groups]. -all_local_pids(Scope) -> - %% selector: ets:fun2ms(fun({N,_,L}) when L =/=[] -> {N,L}end). - ets:select(Scope, [{{'$1','_','$2'},[{'=/=','$2',[]}],[{{'$1','$2'}}]}]). +leave_update_remote_map(Pid, RemoteMap, Groups) when is_pid(Pid) -> + leave_update_remote_map([Pid], RemoteMap, Groups); +leave_update_remote_map(Pids, RemoteMap, Groups) -> + lists:foldl( + fun (Group, Acc) -> + case maps:get(Group, Acc) -- Pids of + [] -> + maps:remove(Group, Acc); + Remaining -> + Acc#{Group => Remaining} + end + end, RemoteMap, Groups). + +all_local_pids(Monitors) -> + maps:to_list(maps:fold( + fun(Pid, {_Ref, Groups}, Acc) -> + lists:foldl( + fun(Group, Acc1) -> + Acc1#{Group => [Pid | maps:get(Group, Acc1, [])]} + end, + Acc, + Groups + ) + end, + #{}, + Monitors + )). %% Works as gen_server:abcast(), but accepts a list of processes %% instead of nodes list. diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 6d4163e744..8edec7c850 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -25,6 +25,8 @@ -export([interfaces/1]). +-include_lib("kernel/include/logger.hrl"). + -define(OP_PUTC,0). -define(OP_MOVE,1). -define(OP_INSC,2). @@ -128,8 +130,12 @@ server1(Iport, Oport, Shell) -> %% are running using "-sname undefined". _ = net_kernel:start([undefined, shortnames]), NodeName = append_hostname(Node, net_kernel:nodename()), - true = net_kernel:connect_node(NodeName), - NodeName; + case net_kernel:connect_node(NodeName) of + true -> + NodeName; + _Else -> + ?LOG_ERROR("Could not connect to ~p",[Node]) + end; true -> append_hostname(Node, node()) end, diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index b3267a590f..62909d324d 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2020. All Rights Reserved. +# Copyright Ericsson AB 1997-2021. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,6 +41,8 @@ SOCKET_MODULES = \ MODULES= \ kernel_test_lib \ + kernel_test_global_sys_monitor \ + kernel_test_sys_monitor \ erpc_SUITE \ rpc_SUITE \ pdict_SUITE \ diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 56c0ca9f9c..5d305868a1 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -173,6 +173,9 @@ init_per_suite(Config0) -> ?P("init_per_suite -> end when " "~n Config: ~p", [Config1]), + %% We need a monitor on this node also + kernel_test_sys_monitor:start(), + Config1 end. @@ -183,10 +186,13 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " - "~n Nodes: ~p", [erlang:nodes()]), + "~n Nodes: ~p", [erlang:nodes()]), Config1. @@ -240,10 +246,40 @@ init_per_testcase(Func, Config) dbg:tpl(gen_tcp, cx), Config; init_per_testcase(_Func, Config) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + kernel_test_global_sys_monitor:reset_events(), + + ?P("init_per_testcase -> done when" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), Config. -end_per_testcase(_Func, _Config) -> - dbg:stop(). +end_per_testcase(Func, _Config) + when Func =:= undefined -> % Insert your testcase name here + dbg:stop(); +end_per_testcase(_Func, Config) -> + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + ?P("system events during test: " + "~n ~p", [kernel_test_global_sys_monitor:events()]), + + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), + ok. %%% gen_tcp:accept/1,2 @@ -321,15 +357,29 @@ do_recv_delim(Config) -> [{active,false}, {packet,line}, {line_delimiter,$X}], {ok, Client} = gen_tcp:connect(localhost, Port, Opts), {ok, A} = gen_tcp:accept(L), + ?P("send the data"), ok = gen_tcp:send(A, "abcXefgX"), + %% Why do we need a timeout? %% Sure, normally there would be no delay, %% but this testcase has nothing to do with timeouts? ?P("read the first chunk"), - {ok, "abcX"} = gen_tcp:recv(Client, 0), %, 200), + {ok, "abcX"} = gen_tcp:recv(Client, 0), % 200), ?P("read the second chunk"), - {ok, "efgX"} = gen_tcp:recv(Client, 0), %, 200), + {ok, "efgX"} = gen_tcp:recv(Client, 0), % 200), + + ?P("set active = 2"), + ok = inet:setopts(Client, [{active,2}]), + + ?P("send the data again"), + ok = gen_tcp:send(A, "abcXefgX"), + + ?P("await the first chunk"), + receive {tcp, Client, "abcX"} -> ?P("received first chunck") end, + ?P("await the second chunk"), + receive {tcp, Client, "efgX"} -> ?P("received second chunck") end, + ?P("cleanup"), ok = gen_tcp:close(Client), ok = gen_tcp:close(A), @@ -1151,6 +1201,13 @@ delete_local_filenames() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +pi(Item) -> + {Item, Val} = process_info(self(), Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + connect_failed_str(Reason) -> ?F("Connect failed: ~w", [Reason]). diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 859d0055c9..83c96e6c46 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -63,16 +63,43 @@ not_owner/1, passive_sockets_server/3, priority_server/2, - oct_acceptor/1, + %% oct_acceptor/1, otp_3924_sender/5, otp_7731_server/2, zombie_server/3, do_iter_max_socks/3]). init_per_testcase(_Func, Config) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + kernel_test_global_sys_monitor:reset_events(), + + ?P("init_per_testcase -> done when" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), Config. -end_per_testcase(_Func, _Config) -> +end_per_testcase(_Func, Config) -> + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + ?P("system events during test: " + "~n ~p", [kernel_test_global_sys_monitor:events()]), + + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), ok. suite() -> @@ -138,11 +165,17 @@ all_cases() -> accept_timeouts_in_order5, accept_timeouts_in_order6, accept_timeouts_in_order7, accept_timeouts_mixed, killing_acceptor, killing_multi_acceptors, - killing_multi_acceptors2, several_accepts_in_one_go, accept_system_limit, - active_once_closed, send_timeout, send_timeout_active, otp_7731, + killing_multi_acceptors2, + several_accepts_in_one_go, accept_system_limit, + active_once_closed, + send_timeout, send_timeout_active, + otp_7731, wrapping_oct, - zombie_sockets, otp_7816, otp_8102, otp_9389, - otp_12242, delay_send_error, bidirectional_traffic + zombie_sockets, + otp_7816, + otp_8102, otp_9389, + otp_12242, delay_send_error, + bidirectional_traffic ]. @@ -161,6 +194,9 @@ init_per_suite(Config0) -> ?P("init_per_suite -> end when " "~n Config: ~p", [Config1]), + %% We need a monitor on this node also + kernel_test_sys_monitor:start(), + Config1 end. @@ -170,10 +206,13 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " - "~n Nodes: ~p", [erlang:nodes()]), + "~n Nodes: ~p", [erlang:nodes()]), Config1. @@ -288,10 +327,10 @@ do_default_options(Config) -> do_delay_on_other_node(XArgs, Function) -> Dir = filename:dirname(code:which(?MODULE)), - {ok,Node} = test_server:start_node(?UNIQ_NODE_NAME, slave, - [{args,"-pa " ++ Dir ++ " " ++ XArgs}]), + {ok, Node} = ?START_SLAVE_NODE(?UNIQ_NODE_NAME, + "-pa " ++ Dir ++ " " ++ XArgs), Res = rpc:call(Node,erlang,apply,[Function,[]]), - test_server:stop_node(Node), + ?STOP_NODE(Node), Res. do_delay_send_1(Config) -> @@ -652,7 +691,7 @@ otp_3924_1(Config, MaxDelay) -> repeat(10, fun(N) -> ok = otp_3924(Config, MaxDelay, Node, Data, DataLen, N) end), - test_server:stop_node(Node), + ?STOP_NODE(Node), ok. otp_3924(Config, MaxDelay, Node, Data, DataLen, N) -> @@ -822,14 +861,13 @@ iter_max_socks(Config) when is_list(Config) -> end, %% Run on a different node in order to limit the effect if this test fails. Dir = filename:dirname(code:which(?MODULE)), - {ok, Node} = test_server:start_node(test_iter_max_socks,slave, - [{args,"+Q 2048 -pa " ++ Dir}]), + {ok, Node} = ?START_SLAVE_NODE(test_iter_max_socks, "+Q 2048 -pa " ++ Dir), %% L = rpc:call(Node,?MODULE,do_iter_max_socks,[N, initalize]), L = iter_max_socks_run(Node, fun() -> exit(do_iter_max_socks(Config, Tries, initalize)) end), - test_server:stop_node(Node), + ?STOP_NODE(Node), io:format("Result: ~p", [L]), all_equal(L), @@ -933,11 +971,11 @@ do_accept(Config, L, Port) -> start_node(Name) -> Pa = filename:dirname(code:which(?MODULE)), - test_server:start_node(Name, slave, [{args, "-pa " ++ Pa}]). + ?START_SLAVE_NODE(Name, "-pa " ++ Pa). start_remote(Name) -> Pa = filename:dirname(code:which(?MODULE)), - test_server:start_node(Name, slave, [{remote, true}, {args, "-pa " ++ Pa}]). + ?START_SLAVE_NODE(Name, "-pa " ++ Pa, [{remote, true}]). %% Tests that when 'the other side' on a passive socket closes, the %% connecting side can still read until the end of data. @@ -1672,42 +1710,51 @@ do_econnreset_after_async_send_active_once(Config) -> ok. craasao_pre(Config) -> - ?P("create listen socket with active = false"), + ?P("[pre] create listen socket with active = false"), {ok, L} = ?LISTEN(Config, 0, [{active, false}, {recbuf, 4096}]), + ?P("[pre] listen socket: " + "~n ~p", [L]), {ok, Port} = inet:port(L), - ?P("create connect socket (~w)", [Port]), + ?P("[pre] create connect socket (~w)", [Port]), Client = case ?CONNECT(Config, localhost, Port, [{active, false}, {sndbuf, 4096}, {show_econnreset, true}]) of {ok, CSock} -> + ?P("[pre] connect socket created:" + "~n ~p", [CSock]), CSock; {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, - ?P("create accept socket"), + ?P("[pre] create accept socket"), {ok, Server} = gen_tcp:accept(L), - ?P("close listen socket"), + ?P("[pre] close listen socket"), ok = gen_tcp:close(L), {Client, Server, craasao_mk_payload(), "Whatever"}. craasao_populate(Client, Server, CPayload, SPayload) when is_port(Client) andalso is_port(Server) -> - ?P("[inet] client send data"), + ?P("[populate,inet] client send data when" + "~n Client: ~p" + "~n Server: ~p", [Client, Server]), ok = gen_tcp:send(Client, CPayload), - ?P("[inet] verify client socket queue size"), + ?P("[populate,inet] verify client socket queue size"), {OS, _} = os:type(), case erlang:port_info(Client, queue_size) of {queue_size, N} when N > 0 -> ok; {queue_size, 0} when OS =:= win32 -> ok; {queue_size, 0} = T -> ct:fail(T) end, - ?P("[inet] server send data"), + ?P("[populate,inet] server send data"), ok = gen_tcp:send(Server, SPayload), - ?P("[inet] sleep some"), + ?P("[populate,inet] sleep some"), ok = ct:sleep(20), {ok, undefined}; craasao_populate(Client, Server, CPayload, SPayload) -> + ?P("[populate,socket] entry when" + "~n Client: ~p" + "~n Server: ~p", [Client, Server]), Sender = spawn_link(fun() -> craasao_populate_sender(Client, CPayload) end), @@ -1715,69 +1762,87 @@ craasao_populate(Client, Server, CPayload, SPayload) -> {'EXIT', Sender, Reason} -> {error, {unexpected_exit, Sender, Reason}} after 2000 -> - ?P("[server] send something"), + ?P("[populate,socket] send something"), ok = gen_tcp:send(Server, SPayload), {ok, Sender} end. craasao_populate_sender(Client, Payload) -> - ?P("[socket] send payload (expect failure)"), + ?P("[populate,sender] send payload (expect failure)"), {error, econnreset} = gen_tcp:send(Client, Payload), - ?P("[socket] payload send failure (as expected)"), + ?P("[populate,sender] payload send failure (as expected)"), exit(normal). craasao_mk_payload() -> list_to_binary(lists:duplicate(1024 * 1024, $.)). craasao_verify(Client, _Sender, _Payload) when is_port(Client) -> - ?P("ensure no 'unexpected messages' received"), + ?P("[verify] ensure no 'unexpected messages' received"), ok = receive Msg -> {unexpected_msg, Msg} after 0 -> ok end, - ?P("set client socket option active: once"), + ?P("[verify] set client socket option active: once"), ok = inet:setopts(Client, [{active, once}]), - ?P("client expect econnreset"), + ?P("[verify] expect client econnreset"), receive {tcp_error, Client, econnreset} -> - ?P("client received econnreset -> expect socket close message"), + ?P("[verify] received client econnreset -> " + "expect client closed message"), receive {tcp_closed, Client} -> - ?P("[connect] received socket close message - done"), + ?P("[verify] " + "received expected client closed message - done"), ok; Other1 -> - ?P("client received unexpected message (expected closed): " + ?P("[verify] client received unexpected message " + "(expected closed): " "~n ~p", [Other1]), ct:fail({unexpected, tcp_closed, Other1}) end; Other2 -> - ?P("client received unexpected message (expected error): " + ?P("[verify] client received unexpected message (expected error): " "~n Unexpected: ~p" "~n Flushed: ~p", [Other2, craasao_flush()]), ct:fail({unexpected, tcp_error, Other2}) end; craasao_verify(Client, Sender, Payload) when is_pid(Sender) -> + ?P("[verify] begin with" + "~n Client: ~p" + "~n Sender: ~p", [Client, Sender]), craasao_verify_sender(Sender), - ?P("ensure no 'unexpected messages' received"), + ?P("[verify] ensure no 'unexpected messages' received"), ok = receive Msg -> {unexpected_msg, Msg} after 0 -> ok end, - ?P("set client socket option active (1): once"), + ?P("[verify] set client socket option active once (1)"), ok = inet:setopts(Client, [{active, once}]), - ?P("client expect (server) data"), + ?P("[verify] client expect (server) data"), receive {tcp, Client, Payload} -> - ?P("client received expected data"), + ?P("[verify] client received expected data"), ok end, - ?P("set client socket option active (2): once"), + ?P("[verify] set client socket option active once (2)"), ok = inet:setopts(Client, [{active, once}]), - ?P("client expect closed"), + ?P("[verify] await client econnreset"), receive - {tcp_closed, Client} -> - ?P("client received expected closed message"); - Other3 -> - ?P("client received unexpected message (expected closed): " + {tcp_error, Client, econnreset} -> + ?P("[verify] client received expected econnreset -> " + "expect socket close message"), + receive + {tcp_closed, Client} -> + ?P("[verify] client received expected closed message"), + ok; + Other1 -> + ?P("[verify] client received unexpected message " + "(expected closed): " + "~n Unexpected: ~p" + "~n Flushed: ~p", [Other1, craasao_flush()]), + ct:fail({unexpected, tcp_closed, Other1}) + end; + Other2 -> + ?P("[verify] client received unexpected message (expected error): " "~n Unexpected: ~p" - "~n Flushed: ~p", [Other3, craasao_flush()]), - ct:fail({unexpected, tcp_closed, Other3}) + "~n Flushed: ~p", [Other2, craasao_flush()]), + ct:fail({unexpected, tcp_error, Other2}) after 10000 -> - ?P("client received unexpected timeout (expected error): " + ?P("[verify] client received unexpected timeout (expected error): " "~n Flushed: ~p", [craasao_flush()]), ct:fail({unexpected_timeout, tcp_error}) end. @@ -1795,6 +1860,7 @@ craasao_verify_sender(Sender) when is_pid(Sender) -> ?P("await sender termination"), receive {'EXIT', Sender, normal} -> + ?P("expected normal sender termination received"), ok; {'EXIT', Sender, Reason} -> ?P("unexpected sender termination: " @@ -1853,7 +1919,7 @@ do_econnreset_after_async_send_passive(Config) -> ?P("sleep some"), ok = ct:sleep(20), - ?P("verify 1 (default)"), + ?P("verify 2 (default)"), ?line ok = craasp_verify(Client2, true, SPayload), ?P("cleanup 2 (default)"), @@ -1957,23 +2023,36 @@ craasp_verify(Client, EConnReset, _Payload) ok -> {error, unexpected_success} end; -craasp_verify(Client, _EConnReset, Payload) -> - ?P("[client] attempt receive and expect success"), +craasp_verify(Client, EConnReset, Payload) + when (EConnReset =:= default) orelse is_boolean(EConnReset) -> + ?P("[client] attempt first recv and expect success"), case gen_tcp:recv(Client, 0) of {ok, Payload} -> - ?P("[client] attempt receive and expect failure (closed)"), + ?P("[client] attempt second recv and expect failure (~w)", + [EConnReset]), case gen_tcp:recv(Client, 0) of - {error, closed} -> - ok; + {error, closed} when (EConnReset =:= default) -> + ?P("[client] expected failure (closed)"), + ok; + {error, econnreset} when (EConnReset =:= true) -> + ?P("[client] expected failure (econnreset)"), + ok; {error, Reason2} -> - {error, {unexpected_error2, Reason2}}; + ?P("[client] unexpected failure:" + "~n EConnReset: ~p" + "~n Reason: ~p", [EConnReset, Reason2]), + {error, {unexpected_error2, EConnReset, Reason2}}; {ok, _} -> + ?P("[client] unexpected success"), {error, unexpected_recv2} end; {ok, _} -> + ?P("[client] unexpected first recv success (wrong payload)"), {error, unexpected_recv1}; - {error, Reason2} -> - {error, {unexpected_error1, Reason2}} + {error, Reason1} -> + ?P("[client] unexpected first recv failure: " + "~n Reason: ~p", [Reason1]), + {error, {unexpected_error1, Reason1}} end. craasp_cleanup(Client, Sender) -> @@ -2168,11 +2247,27 @@ lzs_pre(Config) -> {sndbuf, 4096}]) of {ok, CSock} -> CSock; - {error, eaddrnotavail = Reason} -> - ?SKIPT(connect_failed_str(Reason)) + {error, eaddrnotavail = CReason} -> + ?SKIPT(connect_failed_str(CReason)) end, ?P("accept"), {ok, Server} = gen_tcp:accept(Listen), + %% On *some* platforms, the linger is inherited, + %% but not all so do not assume... + ?P("check if linger option was inherited"), + case inet:getopts(Server, [linger]) of + {ok, [{linger, {true, 0}}]} -> + ?P("linger option was inherited"), + ok; + {ok, [{linger, {true, TO}}]} -> + ?P("linger option was *not* inherited: ~p", [TO]), + ok = inet:setopts(Server, [{linger, {true, 0}}]), + ok; + {error, SReason} -> + ?P("FAILED reading linger option: ~p", [SReason]), + ok = inet:setopts(Server, [{linger, {true, 0}}]), + ok + end, ?P("close listen socket"), ok = gen_tcp:close(Listen), {OS, Client, Server}. @@ -2267,7 +2362,10 @@ do_busy_send(Config) -> fun() -> ?P("[server] create listen socket"), case ?LISTEN(Config, 0, [{active,false},binary, - {reuseaddr,true},{packet,0}]) of + {reuseaddr,true}, + {packet,0}, + {recbuf,4096}, + {sndbuf,4096}]) of {ok, L} -> ?P("[server] listen socket created"), {ok, Port} = inet:port(L), @@ -2275,7 +2373,7 @@ do_busy_send(Config) -> ?P("[server] await continue"), receive {Master, continue} -> - ?P("[server] continue"), + ?P("[server] received continue"), busy_send_srv(L, Master, Msg) end; {error, Reason} -> @@ -2288,9 +2386,13 @@ do_busy_send(Config) -> ListenPort = receive {'EXIT', Server, {skip, _} = SKIP} -> + ?P("[master] server issued skip: " + "~n ~p", [SKIP]), throw(SKIP); {'EXIT', Server, Reason} -> + ?P("[master] server crashed: " + "~n ~p", [Reason]), exit({server, Reason}); {Server, listen_port, LP} -> @@ -2308,7 +2410,11 @@ do_busy_send(Config) -> end, ?P("[client] connect to ~w", [Port]), case ?CONNECT(Config, "localhost", Port, - [{active,false},binary,{packet,0}]) of + [{active,false}, + binary, + {packet,0}, + {recbuf,4096}, + {sndbuf,4096}]) of {ok, Socket} -> Master ! {self(), connected}, ?P("[client] connected - await recv"), @@ -2347,15 +2453,15 @@ busy_send_loop(Server, Client, N) -> ?P("[master] send timeout: server send queue full (N = ~w+1)", [N]), %% Send queue full, sender blocked %% -> stop sender and release client - ?P("Send timeout, time to receive..."), Server ! {self(), close}, Client ! {self(), recv, N+1}, + ?P("[master] await server 'send'..."), receive {Server, send} -> busy_send_2(Server, Client, N+1) after 10000 -> %% If this happens, see busy_send_srv - ?P("UNEXPECTED: server send timeout"), + ?P("[master] UNEXPECTED: server send timeout"), ct:fail({timeout,{server,not_send,flush([])}}) end end. @@ -2363,24 +2469,27 @@ busy_send_loop(Server, Client, N) -> busy_send_2(Server, Client, _N) -> %% Master %% + ?P("[master] await (server) closed"), receive {Server, [closed]} -> - ?P("[master] received expected (server) closed"), + ?P("[master] received expected (server) closed - " + "await client closed"), receive {Client, [0,{error,closed}]} -> ?P("[master] received expected (client) closed"), ok end after 10000 -> - ?P("UNEXPECTED: server closed timeout"), + ?P("[master] UNEXPECTED: server closed timeout"), ct:fail({timeout, {server, not_closed, flush([])}}) end. busy_send_srv(L, Master, Msg) -> %% Server %% Sometimes this accept does not return, do not really know why - %% but is causes the timeout error in busy_send_loop to be + %% but it causes the timeout error in busy_send_loop to be %% triggered. Only happens on OS X Leopard?!? + ?P("[server] try accept"), case gen_tcp:accept(L) of {ok, Socket} -> ?P("[server] accepted"), @@ -2412,6 +2521,8 @@ busy_send_client_loop(Socket, Master, Msg, N) -> {ok, Msg} -> busy_send_client_loop(Socket, Master, Msg, N-1); Other -> + ?P("[client] recv response: " + "~n ~p", [Other]), Master ! {self(), flush([Other,N])} end. @@ -2436,13 +2547,17 @@ do_busy_disconnect_passive2(Config, MuchoData, N) -> ?P("[passive,~w] *** prepare server *** ", [N]), {_Server, S} = busy_disconnect_prepare_server(Config, [{active,false}]), ?P("[passive,~w] server prepared - start sending", [N]), - busy_disconnect_passive_send(S, MuchoData). + {_OSFam, OSName} = os:type(), + busy_disconnect_passive_send(S, MuchoData, OSName). -busy_disconnect_passive_send(S, Data) -> +busy_disconnect_passive_send(S, Data, OS) -> case gen_tcp:send(S, Data) of ok -> - busy_disconnect_passive_send(S, Data); + busy_disconnect_passive_send(S, Data, OS); {error, closed} -> + ok; + {error, eprototype = Reason} when (OS =:= darwin) -> + ?P("send failed with ~w", [Reason]), ok end. @@ -2469,21 +2584,32 @@ do_busy_disconnect_active2(Config, MuchoData, N) -> busy_disconnect_active_send(Server, S, MuchoData). busy_disconnect_active_send(Server, S, Data) -> + {_OSFam, OSName} = os:type(), + busy_disconnect_active_send(Server, S, Data, 1, OSName). + +busy_disconnect_active_send(Server, S, Data, Iter, OS) -> case gen_tcp:send(S, Data) of ok -> - busy_disconnect_active_send(Server, S, Data); + busy_disconnect_active_send(Server, S, Data, Iter+1, OS); {error, closed} -> - ?P("[active-sender] send failed with closed - await tcp-closed"), + ?P("[active-sender,~w] send failed with closed - await tcp-closed", + [Iter]), busy_disconnect_active_send_await_closed(Server, S); + + {error, eprototype = Reason} when (OS =:= darwin) -> + ?P("[active-sender,~w] send failed with ~w - await tcp-closed", + [Iter, Reason]), + busy_disconnect_active_send_await_closed(Server, S); + {error, einval = Reason} -> - ?P("[active-sender] UNEXPECTED send failure:" - "~n ~p", [Reason]), + ?P("[active-sender,~w] UNEXPECTED send failure:" + "~n ~p", [Iter, Reason]), ?SKIPT(send_failed_str(Reason)); {error, Reason} -> - ?P("[active-sender] UNEXPECTED send failure:" - "~n ~p", [Reason]), - ct:fail({unexpected_send_result, Reason}) + ?P("[active-sender,~w] UNEXPECTED send failure:" + "~n ~p", [Iter, Reason]), + ct:fail({unexpected_send_result, Reason, Iter}) end. busy_disconnect_active_send_await_closed(Server, S) -> @@ -3078,11 +3204,26 @@ test_prio_udp() -> %% Tests the so_priority and ip_tos options on sockets when applicable. so_priority(Config) when is_list(Config) -> - ?TC_TRY(so_priority, fun() -> do_so_priority(Config) end). + ?TC_TRY(so_priority, + %% Normally we should have the condition funm here, + %% but since we are (currently) not platform independant, + %% we can't...check in the test case instead. + fun() -> do_so_priority(Config) end). do_so_priority(Config) -> ?P("create listen socket"), - {ok,L} = ?LISTEN(Config, 0, [{active,false}]), + L = case ?LISTEN(Config, 0, [{active,false}]) of + {ok, LSock} when not is_port(LSock) -> + case socket:is_supported(options, socket, priority) of + true -> + LSock; + false -> + (catch gen_tcp:close(LSock)), + ?SKIPT("Option 'priority' not supported") + end; + {ok, LSock} when is_port(LSock) -> + LSock + end, ?P("set opts on listen socket: prio to 1"), ok = inet:setopts(L,[{priority,1}]), ?P("verify prio"), @@ -3184,9 +3325,9 @@ recvtclass(Config) -> %% platforms - change {unix,_} to false? %% pktoptions is not supported for IPv4 -recvtos_ok({unix,netbsd}, _OSVer) -> false; +recvtos_ok({unix,netbsd}, _OSVer) -> false; recvtos_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); -recvtos_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); +recvtos_ok({unix,darwin}, _OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); %% Using the option returns einval, so it is not implemented. recvtos_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); recvtos_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); @@ -3197,9 +3338,9 @@ recvtos_ok({unix,_}, _) -> true; recvtos_ok(_, _) -> false. %% pktoptions is not supported for IPv4 -recvttl_ok({unix,netbsd}, _OSVer) -> false; +recvttl_ok({unix,netbsd}, _OSVer) -> false; recvttl_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); -recvttl_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); +recvttl_ok({unix,darwin}, _OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); %% Using the option returns einval, so it is not implemented. recvttl_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); recvttl_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); @@ -3210,9 +3351,9 @@ recvttl_ok({unix,_}, _) -> true; recvttl_ok(_, _) -> false. %% pktoptions is not supported for IPv6 -recvtclass_ok({unix,netbsd}, _OSVer) -> false; -recvtclass_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); -recvtclass_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); +recvtclass_ok({unix,netbsd}, _OSVer) -> false; +recvtclass_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer,{6,9,0}); +recvtclass_ok({unix,darwin}, _OSVer) -> false; % not semver_lt(OSVer,{19,6,0}); recvtclass_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); %% Using the option returns einval, so it is not implemented. recvtclass_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); @@ -3764,7 +3905,7 @@ do_accept_timeouts_in_order7(Config) -> %% mixed with successful timeouts. accept_timeouts_mixed(Config) when is_list(Config) -> ?TC_TRY(accept_timeouts_mixed, - fun() -> do_accept_timeouts_mixed(Config) end). + fun() -> do_accept_timeouts_mixed(Config) end). do_accept_timeouts_mixed(Config) -> ?P("create listen socket"), @@ -3860,6 +4001,9 @@ do_killing_acceptor(Config) -> ok. do_killing_acceptor_inet(LS) -> + ?P("validate state - listening"), + validate_acceptor_state(LS, [listen], []), + ?P("create acceptor process"), Pid = spawn( fun() -> @@ -3868,35 +4012,26 @@ do_killing_acceptor_inet(LS) -> ?P("sleep some"), receive after 100 -> ok end, - ?P("get status for listen socket"), - {ok,L1} = prim_inet:getstatus(LS), - ?P("verify listen socket accepting"), - true = lists:member(accepting, L1), + + ?P("validate state - accepting"), + validate_acceptor_state(LS, [accepting], []), + ?P("kill acceptor"), - exit(Pid,kill), + exit(Pid, kill), ?P("sleep some"), receive after 100 -> ok end, - ?P("get status for listen socket"), - {ok,L2} = prim_inet:getstatus(LS), - ?P("verify listen socket *not* accepting"), - false = lists:member(accepting, L2), + + ?P("validate state - listening"), + validate_acceptor_state(LS, [listen], [accepting]), + ?P("cleanup"), (catch gen_tcp:close(LS)), ok. do_killing_acceptor_socket(LS) -> ?P("validate state - listening"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 0}, - num_acceptors := 0} -> - ?P("pre validated"), - ok; - INFO0 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO0]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 0, [listening], []), ?P("create acceptor process"), Pid = spawn( @@ -3907,39 +4042,113 @@ do_killing_acceptor_socket(LS) -> ct:sleep(100), ?P("validate state - accepting"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 1}, - num_acceptors := 1} -> - ?P("validated"), - ok; - INFO1 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO1]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 1, [accepting], []), ?P("kill acceptor"), - exit(Pid,kill), + exit(Pid, kill), ?P("sleep some"), ct:sleep(100), ?P("validate state - listening"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 1}, - num_acceptors := 0} -> - ?P("post validated"), - ok; - INFO2 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO2]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 0, [listening], [accepting]), ?P("cleanup"), (catch gen_tcp:close(LS)), ok. +validate_acceptor_state(LS, ExpStates, ExpNotStates) when is_port(LS) -> + case inet:info(LS) of + #{states := States} -> + + ?P("try validate state when: " + "~n Exp States: ~p" + "~n Exp Not States: ~p" + "~n States: ~p", [ExpStates, ExpNotStates, States]), + + %% State *shall* contain ExpStates => States1 =/= States + States1 = States -- ExpStates, + + %% State shall *not* contain ExpNotStates => States2 =:= States + States2 = States -- ExpNotStates, + + if + (States1 =/= States) andalso + (States2 =:= States) -> + ?P("validated: " + "~n Expected States: ~p" + "~n Expected Not States: ~p", + [ExpStates, ExpNotStates]), + ok; + true -> + ?P("invalid states: " + "~n Expected States: ~p" + "~n Expected Not States: ~p" + "~n States: ~p", + [ExpStates, ExpNotStates, States]), + ct:fail("Invalid state(s)") + end; + + InvalidInfo -> + ?P("invalid info: " + "~n Expected States: ~p" + "~n Expected Not States: ~p" + "~n Invalid Info: ~p", + [ExpStates, ExpNotStates, InvalidInfo]), + ct:fail("Invalid state") + end. + +validate_acceptor_state(LS, ExpNumAcc, ExpState, ExpNotState) -> + case inet:info(LS) of + #{num_acceptors := ExpNumAcc, rstates := States} -> + ?P("try validate state when: " + "~n Expected State: ~p" + "~n Expected Not State: ~p" + "~n RStates: ~p", [ExpState, ExpNotState, States]), + + %% States *shall* contain ExpState => States1 =/= States + States1 = States -- ExpState, + + %% States shall *not* contain ExpNotState => States2 =:= States + States2 = States -- ExpNotState, + + if + (States1 =/= States) andalso + (States2 =:= States) -> + ?P("validated: " + "~n Expected States: ~p" + "~n Expected Not States: ~p", + [ExpState, ExpNotState]), + ok; + true -> + ?P("invalid states: " + "~n Expected State: ~p" + "~n Expected Not State: ~p" + "~n States: ~p", + [ExpState, ExpNotState, States]), + ct:fail("Invalid state(s)") + end; + #{num_acceptors := NumAcc, rstates := RStates, wstates := WStates} -> + ?P("invalid state: " + "~n Expected Num Acceptors: ~w" + "~n Num Acceptors: ~w" + "~n Expected State: ~p" + "~n Expected Not State: ~p" + "~n RStates: ~p" + "~n WStates: ~p", + [ExpNumAcc, NumAcc, ExpState, RStates, WStates]), + ct:fail("Invalid state"); + + InvalidInfo -> + ?P("invalid info: " + "~n Expected Num Acceptors: ~w" + "~n Expected State: ~p" + "~n Expected Not State: ~p" + "~n Invalid Info: ~p", + [ExpNumAcc, ExpNumAcc, ExpState, InvalidInfo]), + ct:fail("Invalid state") + end. + %% Check that multi acceptors behaves as expected when killed. killing_multi_acceptors(Config) when is_list(Config) -> @@ -3960,6 +4169,9 @@ do_killing_multi_acceptors(Config) -> ok. do_killing_multi_acceptors_inet(LS) -> + ?P("validate state - listen"), + validate_acceptor_state(LS, [listen], []), + Parent = self(), F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, F2 = mktmofun(1000,Parent,LS), @@ -3973,8 +4185,7 @@ do_killing_multi_acceptors_inet(LS) -> end, ?P("validate state - accepting"), - {ok, L1} = prim_inet:getstatus(LS), - true = lists:member(accepting, L1), + validate_acceptor_state(LS, [accepting], []), ?P("kill first acceptor"), exit(Pid, kill), @@ -3984,31 +4195,21 @@ do_killing_multi_acceptors_inet(LS) -> end, ?P("validate state - still accepting"), - {ok, L2} = prim_inet:getstatus(LS), - true = lists:member(accepting, L2), + validate_acceptor_state(LS, [accepting], []), ?P("await second acceptor exit - timeout"), - ok = ?EXPECT_ACCEPTS([{Pid2,{error,timeout}}],1,1000), + ok = ?EXPECT_ACCEPTS([{Pid2, {error,timeout}}], 1, 1000), ?P("validate state - *not* accepting"), - {ok, L3} = prim_inet:getstatus(LS), - false = lists:member(accepting, L3), + validate_acceptor_state(LS, [listen], [accepting]), ?P("cleanup"), (catch gen_tcp:close(LS)), ok. do_killing_multi_acceptors_socket(LS) -> - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 0}, - num_acceptors := 0} -> - ?P("pre validated"), - ok; - INFO0 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO0]), - ct:fail("wrong number of acceptors") - end, + ?P("validate state - listening"), + validate_acceptor_state(LS, 0, [listening], []), Parent = self(), F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, @@ -4022,16 +4223,7 @@ do_killing_multi_acceptors_socket(LS) -> ct:sleep(100), ?P("validate state - accepting"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 1}, - num_acceptors := 2} -> - ?P("validated"), - ok; - INFO1 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO1]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 2, [accepting], []), ?P("kill first acceptor"), exit(Pid, kill), @@ -4039,34 +4231,13 @@ do_killing_multi_acceptors_socket(LS) -> ct:sleep(100), ?P("validate state - (still) accepting"), - case gen_tcp_socket:info(LS) of - %% No new accept call after the previous 'current acceptor' - %% terminated, just a new select, so change in the counters. - #{counters := #{acc_tries := 1}, - num_acceptors := 1} -> - ?P("validated:1"), - ok; - INFO2 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO2]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 1, [accepting], []), ?P("await second acceptor exit - timeout"), ok = ?EXPECT_ACCEPTS([{Pid2,{error,timeout}}],1,1000), ?P("validate state - listening"), - case gen_tcp_socket:info(LS) of - %% Still no new accept calls - #{counters := #{acc_tries := 1}, - num_acceptors := 0} -> - ?P("post validated"), - ok; - INFO3 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO3]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 0, [listening], [accepting]), ?P("cleanup"), (catch gen_tcp:close(LS)), @@ -4092,6 +4263,9 @@ do_killing_multi_acceptors2(Config) -> ok. do_killing_multi_acceptors2_inet(Config, LS) -> + ?P("validate state - listen"), + validate_acceptor_state(LS, [listen], []), + Parent = self(), ?P("get port number for listen socket"), {ok, PortNo} = inet:port(LS), @@ -4108,8 +4282,7 @@ do_killing_multi_acceptors2_inet(Config, LS) -> end, ?P("validate state - accepting"), - {ok, L1} = prim_inet:getstatus(LS), - true = lists:member(accepting, L1), + validate_acceptor_state(LS, [accepting], []), ?P("kill acceptor 1"), exit(Pid,kill), @@ -4130,8 +4303,7 @@ do_killing_multi_acceptors2_inet(Config, LS) -> end, ?P("validate state - listening"), - {ok, L3} = prim_inet:getstatus(LS), - false = lists:member(accepting, L3), + validate_acceptor_state(LS, [listen], [accepting]), ?P("create acceptor 3"), Pid3 = spawn(F2), @@ -4141,8 +4313,7 @@ do_killing_multi_acceptors2_inet(Config, LS) -> end, ?P("validate state - accepting"), - {ok, L4} = prim_inet:getstatus(LS), - true = lists:member(accepting, L4), + validate_acceptor_state(LS, [accepting], []), ?P("connect to port ~p", [PortNo]), ?CONNECT(Config, "localhost", PortNo,[]), @@ -4151,24 +4322,15 @@ do_killing_multi_acceptors2_inet(Config, LS) -> ok = ?EXPECT_ACCEPTS([{Pid3,{ok,CSock}}] when is_port(CSock),1,100), ?P("validate state - listening"), - {ok, L5} = prim_inet:getstatus(LS), - false = lists:member(accepting, L5), + validate_acceptor_state(LS, [listen], [accepting]), ?P("cleanup"), (catch gen_tcp:close(LS)), ok. do_killing_multi_acceptors2_socket(Config, LS) -> - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 0}, - num_acceptors := 0} -> - ?P("pre validated"), - ok; - INFO0 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO0]), - ct:fail("wrong number of acceptors") - end, + ?P("validate state - listening"), + validate_acceptor_state(LS, 0, [listening], []), Parent = self(), ?P("get port number for listen socket"), @@ -4186,16 +4348,7 @@ do_killing_multi_acceptors2_socket(Config, LS) -> ct:sleep(100), ?P("validate state - accepting"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 1}, - num_acceptors := 2} -> - ?P("validated"), - ok; - INFO1 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO1]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 2, [accepting], []), ?P("kill acceptor 1"), exit(Pid,kill), @@ -4204,16 +4357,7 @@ do_killing_multi_acceptors2_socket(Config, LS) -> ct:sleep(100), ?P("validate state - (still) accepting"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 1}, - num_acceptors := 1} -> - ?P("validated"), - ok; - INFO2 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO2]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 1, [accepting], []), ?P("kill acceptor 2"), exit(Pid2, kill), @@ -4223,17 +4367,7 @@ do_killing_multi_acceptors2_socket(Config, LS) -> end, ?P("validate state - listening"), - case gen_tcp_socket:info(LS) of - %% Still no new accept calls - #{counters := #{acc_tries := 1}, - num_acceptors := 0} -> - ?P("validated"), - ok; - INFO3 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO3]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 0, [listening], [accepting]), ?P("create acceptor 3"), Pid3 = spawn(F2), @@ -4242,16 +4376,7 @@ do_killing_multi_acceptors2_socket(Config, LS) -> ct:sleep(100), ?P("validate state - accepting"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_tries := 2}, - num_acceptors := 1} -> - ?P("validated"), - ok; - INFO4 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO4]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 1, [accepting], []), ?P("connect to port ~p", [PortNo]), ?CONNECT(Config, "localhost", PortNo,[]), @@ -4260,17 +4385,7 @@ do_killing_multi_acceptors2_socket(Config, LS) -> ok = ?EXPECT_ACCEPTS([{Pid3,{ok, _CSock}}],1,100), ?P("validate state - listening"), - case gen_tcp_socket:info(LS) of - #{counters := #{acc_success := 1, - acc_tries := 3}, - num_acceptors := 0} -> - ?P("validated"), - ok; - INFO5 -> - ?P("Invalid acceptor info: " - "~n ~p", [INFO5]), - ct:fail("wrong number of acceptors") - end, + validate_acceptor_state(LS, 0, [listening], [accepting]), ?P("cleanup"), (catch gen_tcp:close(LS)), @@ -4285,7 +4400,8 @@ several_accepts_in_one_go(Config) when is_list(Config) -> do_several_accepts_in_one_go(Config) -> ?P("create listen socket"), - LS = case ?LISTEN(Config, 0,[]) of + NumActors = 8, + LS = case ?LISTEN(Config, 0, [{backlog, NumActors}]) of {ok, LSock} -> LSock; {error, eaddrnotavail = Reason} -> @@ -4293,13 +4409,15 @@ do_several_accepts_in_one_go(Config) -> end, Parent = self(), {ok, PortNo} = inet:port(LS), - F1 = fun() -> ?P("acceptor starting"), - Parent ! {accepted,self(),gen_tcp:accept(LS)} + F1 = fun() -> + ?P("acceptor starting"), + Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - F2 = fun() -> ?P("connector starting"), - Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} + F2 = fun() -> + ?P("connector starting"), + Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} end, - Ns = lists:seq(1,8), + Ns = lists:seq(1, NumActors), ?P("start acceptors"), _ = [spawn(F1) || _ <- Ns], ?P("await accept timeouts"), @@ -4307,7 +4425,7 @@ do_several_accepts_in_one_go(Config) -> ?P("start connectors"), _ = [spawn(F2) || _ <- Ns], ?P("await accepts"), - ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],8,15000), + ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],NumActors,15000), ?P("await connects"), ok = ?EXPECT_CONNECTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],1000), ?P("done"), @@ -4354,23 +4472,22 @@ wait_until_accepting(Proc,N) -> %% Check that accept returns {error, system_limit} %% (and not {error, enfile}) when running out of ports. accept_system_limit(Config) when is_list(Config) -> - OldFlag = process_flag(trap_exit, true), - Res = try do_accept_system_limit(Config) - catch - exit:{skip, _} = SKIP -> - SKIP - end, - process_flag(trap_exit, OldFlag), - Res. + Cond = fun() -> + case ?IS_SOCKET_BACKEND(Config) of + true -> + {skip, "Not complient with socket"}; + false -> + ok + end + end, + TC = fun() -> do_accept_system_limit(Config) end, + ?TC_TRY(accept_system_limit, Cond, TC). do_accept_system_limit(Config) -> ?P("create listen socket"), LS = case ?LISTEN(Config, 0, []) of - {ok, LSocket} when is_port(LSocket) -> - LSocket; {ok, LSocket} -> - (catch gen_tcp:close(LSocket)), - ?SKIPT("Non-socket compliant test case"); + LSocket; {error, eaddrnotavail = Reason} -> ?SKIPT(listen_failed_str(Reason)) end, @@ -4438,62 +4555,98 @@ open_ports(L) -> %% Check that active once and tcp_close messages behave as expected. active_once_closed(Config) when is_list(Config) -> - try do_active_once_closed(Config) - catch - throw:{skip, _} = SKIP -> - SKIP - end. + ?TC_TRY(active_once_closed, fun() -> do_active_once_closed(Config) end). do_active_once_closed(Config) -> ?P("stage 1"), (fun() -> + ?P("[stage1] begin setup"), {Loop,A} = setup_closed_ao(Config), + ?P("[stage1] begin send"), Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ok = inet:setopts(A,[{active,once}]), + ?P("[stage1] try set active:once => expect success"), + ok = inet:setopts(A, [{active,once}]), + ?P("[stage1] await socket closed"), ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + ?P("[stage1] try set active:once => expect failure (einval)"), {error,einval} = inet:setopts(A,[{active,once}]), - ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ?P("[stage1] await socket closed - expect timeout"), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ?P("[stage1] done"), + ok end)(), ?P("stage 2"), (fun() -> + ?P("[stage2] begin setup"), {Loop,A} = setup_closed_ao(Config), + ?P("[stage2] begin send"), Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), + ?P("[stage2] try set active:true => expect success"), ok = inet:setopts(A,[{active,true}]), + ?P("[stage2] await socket closed"), ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + ?P("[stage2] try set active:true => expect failure (einval)"), {error,einval} = inet:setopts(A,[{active,true}]), - ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ?P("[stage2] await socket closed - expect timeout"), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ?P("[stage2] done"), + ok end)(), ?P("stage 3"), (fun() -> + ?P("[stage3] begin setup"), {Loop,A} = setup_closed_ao(Config), + ?P("[stage3] begin send"), Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), + ?P("[stage3] try set active:true => expect success"), ok = inet:setopts(A,[{active,true}]), + ?P("[stage3] await socket closed"), ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + ?P("[stage3] try set active:once => expect failure (einval)"), {error,einval} = inet:setopts(A,[{active,once}]), - ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ?P("[stage3] await socket closed - expect timeout"), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ?P("[stage3] done"), + ok end)(), ?P("stage 4"), (fun() -> + ?P("[stage4] begin setup"), {Loop,A} = setup_closed_ao(Config), + ?P("[stage4] begin send"), Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), + ?P("[stage1] try set active:once => expect success"), ok = inet:setopts(A,[{active,once}]), + ?P("[stage4] await socket closed"), ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + ?P("[stage4] try set active:true => expect failure (einval)"), {error,einval} = inet:setopts(A,[{active,true}]), - ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ?P("[stage4] await socket closed - expect timeout"), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ?P("[stage4] done"), + ok end)(), ?P("stage 5"), (fun() -> + ?P("[stage5] begin setup"), {Loop,A} = setup_closed_ao(Config), + ?P("[stage5] begin send"), Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), + ?P("[stage5] try set active:false => expect success"), ok = inet:setopts(A,[{active,false}]), + ?P("[stage5] await socket closed => expect timeout"), ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ?P("[stage5] try set active:once => expect success"), ok = inet:setopts(A,[{active,once}]), - ok = receive {tcp_closed, A} -> ok after 1000 -> error end + ?P("[stage5] await socket closed"), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + ?P("[stage5] done"), + ok end)(), ?P("done"), ok. @@ -4606,36 +4759,49 @@ anc_await_closed_and_down(S, Pid, MRef, Size, Closed, Down) -> %% Test the send_timeout socket option. send_timeout(Config) when is_list(Config) -> + ?TC_TRY(send_timeout, fun() -> do_send_timeout(Config) end). + +do_send_timeout(Config) -> ?P("begin"), Dir = filename:dirname(code:which(?MODULE)), ?P("create (slave) node"), - {ok,RNode} = test_server:start_node(?UNIQ_NODE_NAME, slave, - [{args,"-pa " ++ Dir}]), + {ok, RNode} = ?START_SLAVE_NODE(?UNIQ_NODE_NAME, "-pa " ++ Dir), + + {TslTimeout, SndTimeout, BinData, SndBuf} = + case ?IS_SOCKET_BACKEND(Config) of + true -> + {100, 3000, binary:copy(<<$a:8>>, 10*1024), 5*1024}; + false -> + {1, 1000, binary:copy(<<$a:8>>, 1*1024), 16*1024} + end, %% Basic ?P("basic check wo autoclose"), - send_timeout_basic(Config, false, RNode), + send_timeout_basic(Config, BinData, SndBuf, TslTimeout, SndTimeout, + false, RNode), ?P("basic check w autoclose"), - send_timeout_basic(Config, true, RNode), - - BinData = <<1:10000>>, + send_timeout_basic(Config, BinData, SndBuf, TslTimeout, SndTimeout, + true, RNode), %% Check timeout length. - ?P("check timeout length"), + ?P("spawn sink process (check timeout length)"), Self = self(), {Pid, Mon} = spawn_monitor( fun() -> - A = setup_timeout_sink(Config, RNode, 1000, true), + {A, _} = setup_timeout_sink(Config, + RNode, SndTimeout, + true, SndBuf), Send = fun() -> Res = gen_tcp:send(A, BinData), Self ! Res, Res end, - {{error, timeout}, _} = timeout_sink_loop(Send) + {{error, timeout}, _} = + timeout_sink_loop(Send, TslTimeout) end), Diff = get_max_diff(), - ?P("Max time for send: ~p",[Diff]), - true = (Diff > 500) and (Diff < 1500), + ?P("Max time for send: ~p", [Diff]), + true = (Diff > (SndTimeout - 500)) and (Diff < (SndTimeout + 500)), %% Wait for the process to die. ?P("await (timeout checker) process death"), @@ -4643,80 +4809,78 @@ send_timeout(Config) when is_list(Config) -> %% Check that parallell writers do not hang forever ?P("check parallell writers wo autoclose"), - send_timeout_para(Config, false, RNode), + send_timeout_para(Config, BinData, SndBuf, TslTimeout, SndTimeout, + false, RNode), ?P("check parallell writers w autoclose"), - send_timeout_para(Config, true, RNode), + send_timeout_para(Config, BinData, SndBuf, TslTimeout, SndTimeout, + true, RNode), ?P("stop (slave) node"), - test_server:stop_node(RNode), + ?STOP_NODE(RNode), ?P("done"), ok. -send_timeout_basic(Config, AutoClose, RNode) -> - BinData = <<1:10000>>, - - A = setup_timeout_sink(Config, RNode, 1000, AutoClose), - Send = fun() -> gen_tcp:send(A, BinData) end, - {{error, timeout}, _} = timeout_sink_loop(Send), +send_timeout_basic(Config, BinData, SndBuf, TslTimeout, SndTimeout, + AutoClose, RNode) -> + ?P("[basic] sink"), + {A, Pid} = setup_timeout_sink(Config, RNode, SndTimeout, + AutoClose, SndBuf), + Send = fun() -> gen_tcp:send(A, BinData) end, + {{error, timeout}, _} = timeout_sink_loop(Send, TslTimeout), %% Check that the socket is not busy/closed... - {error,Error} = gen_tcp:send(A, <<"Hej">>), - after_send_timeout(AutoClose, Error), - ok. - -send_timeout_para(Config, AutoClose, RNode) -> - BinData = <<1:10000>>, + ?P("verify socket not busy/closed"), + case gen_tcp:send(A, BinData) of + {error, Reason} -> + ?P("(expected) send failure"), + after_send_timeout(AutoClose, Reason), + (catch gen_tcp:close(A)), + exit(Pid, kill), + ok; + ok -> + %% Note that there is no active reader on the other end, + %% so a 'channel' has been filled, should remain filled. + (catch gen_tcp:close(A)), + exit(Pid, kill), + ct:fail("Unexpected send success") + end. - ?P("[para] sink"), - A = setup_timeout_sink(Config, RNode, 1000, AutoClose), +send_timeout_para(Config, BinData, BufSz, TslTimeout, SndTimeout, + AutoClose, RNode) -> + ?P("[para] sink -> entry with" + "~n size(BinData): ~p" + "~n BufSz: ~p" + "~n SndTimeout: ~p" + "~n AutoClose: ~p", + [byte_size(BinData), BufSz, SndTimeout, AutoClose]), + {A, Pid} = setup_timeout_sink(Config, RNode, SndTimeout, AutoClose, BufSz), Self = self(), SenderFun = fun() -> ?P("[para:sender] start"), Send = fun() -> gen_tcp:send(A, BinData) end, - Self ! {self(), timeout_sink_loop(Send)} + Self ! {self(), timeout_sink_loop(Send, TslTimeout)} end, - Info = fun() -> - {(catch erlang:port_info(A)), - try inet:getopts(A, [send_timeout]) of - {ok, [V2]} -> - V2; - {error, R2} -> - ?F("ERROR: ~p", [R2]); - X2 -> - ?F("UNKNOWN: ~p", [X2]) - catch - C2:E2:S2 -> - ?F("CATCHED: ~p, ~p, ~p", [C2, E2, S2]) - end, - try inet:getstat(A) of - {ok, S3} -> - S3; - {error, R3} -> - ?F("ERROR: ~p", [R3]); - X3 -> - ?F("UNKNOWN: ~p", [X3]) - catch - C3:E3:S3 -> - ?F("CATCHED: ~p, ~p, ~p", [C3, E3, S3]) - end, - try prim_inet:getstatus(A) of - {ok, S4} -> - S4; - {error, R4} -> - ?F("ERROR: ~p", [R4]); - X4 -> - ?F("UNKNOWN: ~p", [X4]) - catch - C4:E4:S4 -> - ?F("CATCHED: ~p, ~p, ~p", [C4, E4, S4]) - end} - end, ?P("[para] spawn process 1 with sender fun"), Snd1 = spawn_link(SenderFun), ?P("[para] spawn process 2 with sender fun"), Snd2 = spawn_link(SenderFun), + SockInfo = fun() -> (catch inet:info(A)) end, + SockTimeout = fun() -> + try inet:getopts(A, [send_timeout]) of + {ok, [V2]} -> + V2; + {error, R2} -> + ?F("ERROR: ~p", [R2]); + X2 -> + ?F("UNKNOWN: ~p", [X2]) + catch + C2:E2:S2 -> + ?F("CATCHED: ~p, ~p, ~p", [C2, E2, S2]) + end + end, + ?P("[para] await sender timeout when" "~n Sender 1: ~p" "~n Sender 2: ~p", [Snd1, Snd2]), @@ -4729,22 +4893,21 @@ send_timeout_para(Config, AutoClose, RNode) -> ?P("[para] timeout received from sender 2 (~p, ~p)", [Snd2, N]), 2 after 20000 -> - {PortStatus1, SockOpts1, SockStat1, SockStatus1} = Info(), + SockInfo1 = SockInfo(), + SockTo1 = SockTimeout(), ?P("[para] UNEXPECTED timeout(1,~w) when:" "~n Sender 1 Info: ~p" "~n Sender 2 Info: ~p" - "~n Port Status: ~p" + "~n Socket Info: ~p" "~n Send Timeout: ~p" - "~n Socket Stats: ~p" - "~n Socket Status: ~p" "~n Message Queue: ~p", [AutoClose, (catch process_info(Snd1)), (catch process_info(Snd2)), - PortStatus1, SockOpts1, SockStat1, SockStatus1, + SockInfo1, SockTo1, flush([])]), - Snd1 ! {info_and_die, Info}, - Snd2 ! {info_and_die, Info}, + Snd1 ! {info_and_die, SockInfo, SockTimeout}, + Snd2 ! {info_and_die, SockInfo, SockTimeout}, ct:sleep(?SECS(1)), exit({timeout, AutoClose}) end, @@ -4762,42 +4925,42 @@ send_timeout_para(Config, AutoClose, RNode) -> after_send_timeout(AutoClose, Error_1) after 10000 -> if (Second =:= 1) -> - {PortStatus21, SockOpts21, SockStat21, SockStatus21} = - Info(), + SockInfo21 = SockInfo(), + SockTo21 = SockTimeout(), ?P("[para] UNEXPECTED timeout(2,~w):" "~n Sender 1 Info: ~p" - "~n Port Status: ~p" + "~n Socket Info: ~p" "~n Send Timeout: ~p" - "~n Socket Stats: ~p" - "~n Socket Status: ~p" "~n Message Queue: ~p", [AutoClose, (catch process_info(Snd1)), - PortStatus21, SockOpts21, SockStat21, SockStatus21, + SockInfo21, SockTo21, flush([])]), - Snd1 ! {info_and_die, Info}; + Snd1 ! {info_and_die, SockInfo, SockTimeout}; true -> - {PortStatus22, SockOpts22, SockStat22, SockStatus22} = - Info(), + SockInfo22 = SockInfo(), + SockTo22 = SockTimeout(), ?P("[para] UNEXPECTED timeout(2,~w):" "~n Sender 2 Info: ~p" - "~n Port Status: ~p" + "~n Socket Info: ~p" "~n Send Timeout: ~p" - "~n Socket Stats: ~p" - "~n Socket Status: ~p" "~n Message Queue: ~p", [AutoClose, (catch process_info(Snd2)), - PortStatus22, SockOpts22, SockStat22, SockStatus22, + SockInfo22, SockTo22, flush([])]), - Snd2 ! {info_and_die, Info} + Snd2 ! {info_and_die, SockInfo, SockTimeout} end, ct:sleep(?SECS(1)), exit({timeout, AutoClose, Second}) end, - {error,Error_2} = gen_tcp:send(A, <<"Hej">>), + {error, Error_2} = gen_tcp:send(A, BinData), after_send_timeout(AutoClose, Error_2), - ?P("[para] done"), + ?P("cleanup - close socket"), + (catch gen_tcp:close(A)), + ?P("cleanup - kill sink"), + exit(Pid, kill), + ?P("[para] done"), ok. @@ -4816,6 +4979,7 @@ get_max_diff(Max) -> Diff = millis() - T1, if Diff > Max -> + ?P("new max send time: ~w", [Diff]), get_max_diff(Diff); true -> get_max_diff(Max) @@ -4824,6 +4988,7 @@ get_max_diff(Max) -> Diff = millis() - T1, if Diff > Max -> + ?P("timeout diff (> prev max send to): ~w", [Diff]), Diff; true -> Max @@ -4834,22 +4999,28 @@ get_max_diff(Max) -> after_send_timeout(AutoClose, Reason) -> case Reason of - timeout when AutoClose =:= false -> ok; - enotconn when AutoClose =:= true -> ok -%%% timeout -> ok; -%%% enotconn when AutoClose -> ok; -%%% closed when AutoClose -> ok + timeout when AutoClose =:= false -> ok; + {timeout, _RestData} when AutoClose =:= false -> ok; + enotconn when AutoClose =:= true -> ok; + closed when AutoClose -> ok; + _ -> + ?P("after_send_timeout -> " + "~n AutoClose: ~w" + "~n Reason: ~p", [AutoClose, Reason]), + exit({after_send_timeout, AutoClose, Reason}) end. %% Test the send_timeout socket option for active sockets. send_timeout_active(Config) when is_list(Config) -> + ?TC_TRY(send_timeout_active, fun() -> do_send_timeout_active(Config) end). + +do_send_timeout_active(Config) -> Dir = filename:dirname(code:which(?MODULE)), - {ok,RNode} = test_server:start_node(?UNIQ_NODE_NAME, slave, - [{args,"-pa " ++ Dir}]), + {ok, RNode} = ?START_SLAVE_NODE(?UNIQ_NODE_NAME, "-pa " ++ Dir), do_send_timeout_active(Config, false, RNode), do_send_timeout_active(Config, true, RNode), - test_server:stop_node(RNode), + ?STOP_NODE(RNode), ok. do_send_timeout_active(Config, AutoClose, RNode) -> @@ -4858,29 +5029,80 @@ do_send_timeout_active(Config, AutoClose, RNode) -> Mad = spawn_link(RNode, fun() -> mad_sender(C) end), ListData = lists:duplicate(1000, $a), F = fun() -> + ?P("[sink action] await data"), receive {tcp, _Sock, _Data} -> + ?P("[sink action] active -> once"), inet:setopts(A, [{active, once}]), + ?P("[sink action] send payload"), Res = gen_tcp:send(A, ListData), Res; - Err -> - io:format("sock closed: ~p~n", [Err]), - Err + Unexpected -> + ?P("[sink action] unexpected message: " + "~n ~p", [Unexpected]), + Unexpected end end, - {{error, timeout}, _} = timeout_sink_loop(F), + {{error, timeout}, _} = timeout_sink_loop(F, 1), unlink(Mad), exit(Mad, kill), flush(), ok. mad_sender(S) -> + put(action, nothing), + put(sent, 0), + put(elapsed, 0), + mad_sender(S, 0). + +mad_sender(S, N) -> U = rand:uniform(1000000), - case gen_tcp:send(S, integer_to_list(U)) of + put(action, send), + Start = erlang:monotonic_time(), + Ret = gen_tcp:send(S, integer_to_list(U)), + Stop = erlang:monotonic_time(), + Elapsed = get(elapsed), + put(elapsed, Elapsed + (Stop - Start)), + put(action, sent), + N2 = N + 1, + put(sent, N2), + case Ret of ok -> - mad_sender(S); - Err -> - Err + mad_sender(S, N + 1); + {error, timeout} = ERROR1 -> + ?P("mad_sender -> send failed: timeout" + "~n Number of sends: ~w" + "~n Elapsed (send) time: ~w msec", + [N2, + erlang:convert_time_unit(get(elapsed), native, millisecond)]), + ERROR1; + {error, {timeout, RestData}} = ERROR2 -> + ?P("mad_sender -> " + "send failed: timeout with ~w bytes of rest data" + "~n Number of sends: ~w" + "~n Elapsed (send) time: ~w msec", + [byte_size(RestData), + N2, + erlang:convert_time_unit(get(elapsed), native, millisecond)]), + ERROR2; + {error, Reason} = ERROR3 -> + ?P("mad_sender -> send failed: " + "~n ~p" + "~n Number of sends: ~w" + "~n Elapsed (send) time: ~w msec", + [Reason, + N2, + erlang:convert_time_unit(get(elapsed), native, millisecond)]), + ERROR3; + ERROR4 -> + ?P("mad_sender -> send failed: " + "~n ~p" + "~n Number of sends: ~w" + "~n Elapsed (send) time: ~w msec", + [ERROR4, + N2, + erlang:convert_time_unit(get(elapsed), native, millisecond)]), + ERROR4 end. flush() -> @@ -4894,8 +5116,7 @@ flush() -> setup_closed_ao(Config) -> Dir = filename:dirname(code:which(?MODULE)), ?P("[setup] start slave node"), - R = case test_server:start_node(?UNIQ_NODE_NAME, slave, - [{args,"-pa " ++ Dir}]) of + R = case ?START_SLAVE_NODE(?UNIQ_NODE_NAME, "-pa " ++ Dir) of {ok, Slave} -> Slave; {error, Reason} -> @@ -4907,7 +5128,7 @@ setup_closed_ao(Config) -> {ok, LSock} -> LSock; {error, eaddrnotavail = LReason} -> - (catch test_server:stop_node(R)), + (catch ?STOP_NODE(R)), ?SKIPT(listen_failed_str(LReason)) end, Fun = fun(F) -> @@ -4933,7 +5154,7 @@ setup_closed_ao(Config) -> {ok, CSock} -> CSock; {error, eaddrnotavail = CReason} -> - (catch test_server:stop_node(R)), + (catch ?STOP_NODE(R)), ?SKIPT(connect_failed_str(CReason)) end, ?P("[setup] accept (local) connection"), @@ -4941,10 +5162,10 @@ setup_closed_ao(Config) -> {ok, ASock} -> ASock; {error, eaddrnotavail = AReason} -> - (catch test_server:stop_node(R)), + (catch ?STOP_NODE(R)), ?SKIPT(accept_failed_str(AReason)) end, - ?P("[setup] send message"), + ?P("[setup] send (local) and receive (remote) message"), gen_tcp:send(A,"Hello"), {ok, "Hello"} = Remote(fun() -> gen_tcp:recv(C,0) end), ?P("[setup] close (remote) connection"), @@ -4953,26 +5174,27 @@ setup_closed_ao(Config) -> {failure, timeout}; (L2,{MA,MB},F2,N) -> case F2() of - MA -> MA; - MB -> MB; - Other -> ?P("~p",[Other]), + MA -> ?P("[setup] action result (MA): ~p", [MA] ), MA; + MB -> ?P("[setup] action result (MB): ~p", [MB] ), MB; + Other -> ?P("[setup] Loop2: ~p",[Other]), receive after 1000 -> ok end, L2(L2,{MA,MB},F2,N-1) end end, Loop = fun(Match2,F3) -> Loop2(Loop2,Match2,F3,10) end, ?P("[setup] stop slave node"), - test_server:stop_node(R), + ?STOP_NODE(R), ?P("[setup] done"), {Loop,A}. -setup_timeout_sink(Config, RNode, Timeout, AutoClose) -> - Host = get_hostname(node()), +setup_timeout_sink(Config, RNode, Timeout, AutoClose, BufSz) -> + Host = get_hostname(node()), ?P("[sink] create listen socket"), {ok, L} = ?LISTEN(Config, 0, [{active, false}, - {packet, 2}, - {send_timeout, Timeout}, - {send_timeout_close, AutoClose}]), + {packet, 4}, + {sndbuf, BufSz}, + {send_timeout, Timeout}, + {send_timeout_close, AutoClose}]), Fun = fun(F) -> receive {From,X} when is_function(X) -> @@ -4991,22 +5213,31 @@ setup_timeout_sink(Config, RNode, Timeout, AutoClose) -> ?P("[sink] connect from remote node (~p)", [RNode]), {ok, C} = Remote(fun() -> ?CONNECT(Config, Host,Port, - [{active,false},{packet,2}]) + [{active, false}, + {packet, 4}, + {sndbuf, BufSz div 2}]) end), ?P("[sink] accept"), {ok, A} = gen_tcp:accept(L), - ?P("[sink] send message"), - gen_tcp:send(A,"Hello"), - ?P("[sink] recv message on remote node (~p)", [RNode]), + ?P("[sink] accepted - send 'check' message"), + gen_tcp:send(A, "Hello"), + ?P("[sink] message sent - recv 'check' message on remote node (~p)", + [RNode]), {ok, "Hello"} = Remote(fun() -> gen_tcp:recv(C,0) end), + ?P("[sink] cleanup"), + (catch gen_tcp:close(L)), ?P("[sink] done"), - A. + {A, Pid}. setup_active_timeout_sink(Config, RNode, Timeout, AutoClose) -> Host = get_hostname(node()), - ListenOpts = [binary,{active,false},{packet,0}, - {nodelay,true},{keepalive,true}, - {send_timeout,Timeout},{send_timeout_close,AutoClose}], + ListenOpts = [binary, + {active, false}, + {packet, 0}, + {nodelay, true}, + {keepalive, true}, + {send_timeout, Timeout}, + {send_timeout_close, AutoClose}], {ok, L} = ?LISTEN(Config, 0, ListenOpts), Fun = fun(F) -> receive @@ -5031,13 +5262,16 @@ setup_active_timeout_sink(Config, RNode, Timeout, AutoClose) -> {ok, "H"++_} = Remote(fun() -> gen_tcp:recv(C, 0) end), {A, C}. -timeout_sink_loop(Action) -> +%% timeout_sink_loop(Action) -> +%% timeout_sink_loop(Action, 1). + +timeout_sink_loop(Action, To) -> put(action, nothing), put(sent, 0), put(elapsed, 0), - timeout_sink_loop(Action, 0). + timeout_sink_loop(Action, To, 0). -timeout_sink_loop(Action, N) -> +timeout_sink_loop(Action, To, N) -> put(action, send), Start = erlang:monotonic_time(), Ret = Action(), @@ -5050,18 +5284,23 @@ timeout_sink_loop(Action, N) -> case Ret of ok -> receive - {info_and_die, Info} -> - {PortStatus, SockOpts, SockStat, SockStatus} = Info(), + {info_and_die, SockInfo, SockTimeout} -> ?P("[sink-loop] info and die: " - "~n Port Status: ~p" - "~n Send Timeout: ~p" - "~n Socket Stats: ~p" - "~n Socket Status: ~p", - [PortStatus, SockOpts, SockStat, SockStatus]), + "~n Socket Info: ~p" + "~n Send Timeout: ~p", + [SockInfo(), SockTimeout()]), exit(normal) - after 1 -> ok - end, - timeout_sink_loop(Action, N+1); + after To -> ok end, + timeout_sink_loop(Action, To, N+1); + {error, {timeout, RestData}} -> + ?P("[sink-loop] action result: " + "~n Number of actions: ~p" + "~n Elapsed time: ~p msec" + "~n Result: timeout with ~w of rest data", + [N2, + erlang:convert_time_unit(get(elapsed), native, millisecond), + byte_size(RestData)]), + {{error, timeout}, N2}; Other -> ?P("[sink-loop] action result: " "~n Number of actions: ~p" @@ -5084,70 +5323,96 @@ has_superfluous_schedulers() -> %% Leaking message from inet_drv {inet_reply,P,ok} %% when a socket sending resumes working after a send_timeout. +%% Should we even bother testing this if 'inet_backend = socket'? otp_7731(Config) when is_list(Config) -> + ?TC_TRY(otp_7731, fun() -> do_otp_7731(Config) end). + +do_otp_7731(Config) when is_list(Config) -> + ?P("[ctrl] create server"), ServerPid = spawn_link(?MODULE, otp_7731_server, [Config, self()]), + ?P("[ctrl] await listening port (number) from server"), receive {ServerPid, ready, PortNum} -> ok end, + ?P("[ctrl] connect to server on port ~w", [PortNum]), {ok, Socket} = ?CONNECT(Config, "localhost", PortNum, [binary, {active, false}, {packet, raw}, {send_timeout, 1000}]), + + ?P("[ctrl] send data"), otp_7731_send(Socket), - ?P("Sending complete..."), + ?P("[ctrl] sending complete - order server to recv"), ServerPid ! {self(), recv}, - receive {ServerPid, ok} -> ok end, - - ?P("Client waiting for leaking messages..."), + ?P("[ctrl] await 'recv complete' from server"), + receive {ServerPid, ok} -> ?P("[ctrl] received 'recv complete'"), ok end, %% Now make sure inet_drv does not leak any internal messages. + ?P("[ctrl] waiting for leaking messages..."), receive Msg -> + ?P("[ctrl] got unexpected message: " + "~n ~p", [Msg]), ct:fail({unexpected, Msg}) after 1000 -> ok end, - ?P("No leaking messages. Done."), - gen_tcp:close(Socket). + ?P("[ctrl] no leaking messages - cleanup"), + (catch gen_tcp:close(Socket)), + ServerPid ! {self(), die}, + ?P("[ctrl] done."), + ok. otp_7731_send(Socket) -> Bin = <<1:10000>>, - io:format("Client sending ~p bytes...\n",[size(Bin)]), + ?P("[client] sending ~p bytes...", [byte_size(Bin)]), case gen_tcp:send(Socket, Bin) of - ok -> otp_7731_send(Socket); - {error,timeout} -> ok + ok -> + otp_7731_send(Socket); + {error, {timeout, RestData}} -> + ?P("[client] send timeout with ~w bytes of rest data", + [byte_size(RestData)]), + ok; + {error, timeout} -> + ?P("[client] send timeout"), + ok end. -otp_7731_server(Config, ClientPid) -> +otp_7731_server(Config, Ctrl) -> + ?P("[server] create listen socket"), {ok, LSocket} = ?LISTEN(Config, 0, [binary, {packet, raw}, {active, false}]), {ok, {_, PortNum}} = inet:sockname(LSocket), - ?P("Listening on ~w with port number ~p", [LSocket, PortNum]), - ClientPid ! {self(), ready, PortNum}, + ?P("[server] listening on port number ~p", [PortNum]), + Ctrl ! {self(), ready, PortNum}, + ?P("[server] accept"), {ok, CSocket} = gen_tcp:accept(LSocket), + ?P("[server] accepted - close listen socket"), gen_tcp:close(LSocket), - ?P("Server got connection, wait for recv order..."), - - receive {ClientPid, recv} -> ok end, + ?P("[server] await recv order"), + receive {Ctrl, recv} -> ok end, - ?P("Server start receiving..."), - + ?P("[server] start receiving..."), otp_7731_recv(CSocket), - ClientPid ! {self(), ok}, + ?P("[server] announce recv done"), + Ctrl ! {self(), ok}, - ?P("Server finished, closing..."), - gen_tcp:close(CSocket). + ?P("[server] finished, (connection) closing..."), + gen_tcp:close(CSocket), + receive {Ctrl, die} -> ok end, + ?P("[server] done"), + exit(normal). otp_7731_recv(Socket) -> case gen_tcp:recv(Socket, 0, 1000) of - {ok, Bin} -> - ?P("Server received ~p bytes", [size(Bin)]), - otp_7731_recv(Socket); - {error,timeout} -> - ?P("Server got receive timeout"), - ok - end. + {ok, Bin} -> + ?P("[server] received ~p bytes", [size(Bin)]), + otp_7731_recv(Socket); + {error, timeout} -> + ?P("[server] receive timeout - done recv"), + ok + end. %% OTP-7615: TCP-ports hanging in CLOSING state when sending large @@ -5266,92 +5531,189 @@ zombie_server_handler(Socket, Bin) -> %% Hanging send on windows when sending iolist with more than 16 binaries. otp_7816(Config) when is_list(Config) -> - Client = self(), - Server = spawn_link(fun()-> otp_7816_server(Config, Client) end), + ?TC_TRY(otp_7816, fun() -> do_otp_7816(Config) end). + +do_otp_7816(Config) -> + Ctrl = self(), + ?P("[ctrl] create server process..."), + Server = spawn_link(fun() -> otp_7816_server(Config, Ctrl) end), + ?P("[ctrl] await server process ready..."), receive {Server, ready, PortNum} -> ok end, + ?P("[ctrl] connect to server..."), {ok, Socket} = ?CONNECT(Config, "localhost", PortNum, [binary, {active, false}, {packet, 4}, {send_timeout, 10}]), + %% We use the undocumented feature that sending can be resumed after %% a send_timeout without any data loss if the peer starts to receive data. %% Unless of course the 7816-bug is in affect, in which case the write event %% for the socket is lost on windows and not all data is sent. - [otp_7816_send(Socket,18,BinSize,Server) || BinSize <- lists:seq(1000, 2000, 123)], + ?P("[ctrl] begin sending..."), + + [otp_7816_ctrl(Socket, 18, BinSize, Server) || + BinSize <- lists:seq(1000, 2000, 123)], - io:format("Sending complete...\n",[]), + ?P("[ctrl] sending complete..."), ok = gen_tcp:close(Socket), Server ! {self(), closed}, {Server, closed} = receive M -> M end. - -otp_7816_send(Socket, BinNr, BinSize, Server) -> - Data = lists:duplicate(BinNr, <<1:(BinSize*8)>>), - SentBytes = otp_7816_send_data(Socket, Data, 0) * BinNr * BinSize, - io:format("Client sent ~p bytes...\n",[SentBytes]), - Server ! {self(),recv,SentBytes}, - {Server, ok} = receive M -> M end. +otp_7816_ctrl(Socket, BinNr, BinSize, Server) -> + ?P("[ctrl] create payload (BinSz: ~w)...", [BinSize]), + Data = lists:duplicate(BinNr, <<1:(BinSize*8)>>), + ?P("[ctrl] socket info prior to start sending: " + "~n ~p", [inet:info(Socket)]), + Ctrl = self(), + Client = spawn_link(fun() -> otp_7816_send_data(Ctrl, Socket, Data) end), + SentBytes = + receive + {Client, continue, Loops} -> + ?P("[ctrl] received continue from client: ~p", [Loops]), + SB = Loops * BinNr * BinSize, + Server ! {self(), recv, SB}, + SB + end, + ct:sleep(1000), + ?P("[ctrl] socket info after sending ~w bytes: " + "~n ~p", [SentBytes, inet:info(Socket)]), + ?P("[ctrl] await server result..."), + ok = receive + {Server, SR} -> + ?P("[ctrl] server result: ~p", [SR]), + SR + end, + ?P("[ctrl] await client termination..."), + ok = receive + {'EXIT', Client, normal} -> + ?P("[ctrl] client normal exit"), + ok; + {'EXIT', Client, CR} -> + ?P("[ctrl] client unexpected exit reason: ~p", [CR]), + CR + end, + ?P("[ctrl] done with BinSz: ~p", [BinSize]), + ok. -otp_7816_send_data(Socket, Data, Loops) -> - io:format("Client sending data...\n",[]), +otp_7816_send_data(Ctrl, Socket, Data) -> + otp_7816_send_data(Ctrl, Socket, Data, 0). + +otp_7816_send_data(Ctrl, Socket, Data, Loops) -> + ?P("[client] sending data (~w bytes, ~w)...", [iolist_size(Data), Loops]), case gen_tcp:send(Socket, Data) of ok -> - otp_7816_send_data(Socket,Data, Loops+1); - {error,timeout} -> - Loops+1 + otp_7816_send_data(Ctrl, Socket, Data, Loops+1); + + {error, timeout} when is_port(Socket) -> + %% For the 'classic' sockets, when the OS buffers are + %% full, the rest data are put into the (inet driver) + %% internal, for later sending. SO, it can be counted + %% as sent. + ?P("[client] send timeout when Loops: ~p (+1)", [Loops]), + Ctrl ! {self(), continue, Loops + 1}, + exit(normal); + + {error, timeout} -> + %% For inet_backend = 'socket' when we nothing of the + %% message was sent. + ?P("[client] send timeout when Loops: ~p", [Loops]), + Ctrl ! {self(), continue, Loops}, + exit(normal); + + {error, {timeout, RestData}} -> + %% A timeout means that **some** of the data was not sent, + %% currently there is no way to know how much. + %% NOTE THAT THIS MEANS THAT WE HAVE A PARTIAL PACKAGE + %% WRITTEN, INCLUDING A HEADER THAT INDUCATES A DATA + %% SIZE THAT IS **NOT** PRESENT!! + %% So for this trest case to work, we need to write the rest. + %% But we cannot do that without first setting the package to raw. + %% + ?P("[client] send timeout" + "~n with ~w bytes of rest data" + "~n when Loops: ~p", [byte_size(RestData), Loops]), + Ctrl ! {self(), continue, Loops + 1}, + ?P("[client] packet to 'raw'..."), + ok = inet:setopts(Socket, [{packet, raw}, {send_timeout, 1000}]), + ?P("[client] send ~w bytes of rest data...", + [byte_size(RestData)]), + ok = gen_tcp:send(Socket, RestData), + ?P("[client] packet (back) to '4'..."), + ok = inet:setopts(Socket, [{packet, 4}, {send_timeout, 10}]), + ?P("[client] done"), + exit(normal) + + end. -otp_7816_server(Config, Client) -> +otp_7816_server(Config, Ctrl) -> + ?P("[server] create listening socket"), {ok, LSocket} = ?LISTEN(Config, 0, [binary, {packet, 4}, {active, false}]), {ok, {_, PortNum}} = inet:sockname(LSocket), - ?P("Listening on ~w with port number ~p", [LSocket, PortNum]), - Client ! {self(), ready, PortNum}, + ?P("[server] listening on ~w with port number ~p", [LSocket, PortNum]), + Ctrl ! {self(), ready, PortNum}, - {ok, CSocket} = gen_tcp:accept(LSocket), - ?P("Server got connection..."), + ?P("[server] accept connection..."), + {ok, ASocket} = gen_tcp:accept(LSocket), + ?P("[server] got connection..."), gen_tcp:close(LSocket), - otp_7816_server_loop(CSocket), - - ?P("Server terminating."). + otp_7816_server_loop(ASocket, Ctrl), + ?P("[server] terminating"). -otp_7816_server_loop(CSocket) -> - ?P("Server waiting for order..."), +otp_7816_server_loop(Socket, Ctrl) -> + ?P("[server] waiting for order..."), receive - {Client, recv, RecvBytes} -> - ?P("Server start receiving..."), + {Ctrl, recv, RecvBytes} -> + ?P("[server] start receiving (~w bytes)...", [RecvBytes]), - ok = otp_7816_recv(CSocket, RecvBytes), + ok = otp_7816_recv(Socket, RecvBytes), - Client ! {self(), ok}, - otp_7816_server_loop(CSocket); + Ctrl ! {self(), ok}, + otp_7816_server_loop(Socket, Ctrl); - {Client, closed} -> - {error, closed} = gen_tcp:recv(CSocket, 0, 1000), - Client ! {self(), closed} + {Ctrl, closed} -> + {error, closed} = gen_tcp:recv(Socket, 0, 1000), + Ctrl ! {self(), closed} end. +otp_7816_recv(Socket, BytesLeft) -> + otp_7816_recv(Socket, BytesLeft, 1). -otp_7816_recv(_, 0) -> - io:format("Server got all.\n",[]), +otp_7816_recv(_, 0, _) -> + ?P("[server] got all data"), ok; -otp_7816_recv(CSocket, BytesLeft) -> - case gen_tcp:recv(CSocket, 0, 1000) of - {ok, Bin} when byte_size(Bin) =< BytesLeft -> - io:format("Server received ~p of ~p bytes.\n",[size(Bin), BytesLeft]), - otp_7816_recv(CSocket, BytesLeft - byte_size(Bin)); - {error,timeout} -> - io:format("Server got receive timeout when expecting more data\n",[]), - error - end. +otp_7816_recv(Socket, BytesLeft, N) -> + ?P("[server] try recv ~w (~p bytes left)", [N, BytesLeft]), + case gen_tcp:recv(Socket, 0, 1000) of + {ok, Bin} when (byte_size(Bin) =< BytesLeft) andalso + (byte_size(Bin) > 0) -> + ?P("[server] received ~p of ~p bytes", + [size(Bin), BytesLeft]), + otp_7816_recv(Socket, BytesLeft - byte_size(Bin), N+1); + {ok, Bin} -> + ?P("[server] received unexpected data (~w): " + "~n Expected: ~p bytes" + "~n Received: ~p bytes" + "~n Socket Info: ~p", + [N, BytesLeft, byte_size(Bin), inet:info(Socket)]), + {error, {unexpected_data, BytesLeft, byte_size(Bin)}}; + {error, timeout} -> + ?P("[server] got receive timeout when expecting more data:" + "~n Socket Info: ~p", [inet:info(Socket)]), + {error, timeout} + end. + + +%% ---------------------------------------------------------------------- %% Receive a packet with a faulty packet header. otp_8102(Config) when is_list(Config) -> @@ -5453,69 +5815,224 @@ otp_9389_loop(S, OrigLinkHdr, State) -> end. wrapping_oct() -> - [{timetrap,{minutes,10}}]. + [{timetrap, {minutes,20}}]. %% Check that 64bit octet counters work. wrapping_oct(Config) when is_list(Config) -> - {ok,Sock} = ?LISTEN(Config, 0,[{active,false},{mode,binary}]), - {ok,Port} = inet:port(Sock), - spawn_link(?MODULE,oct_acceptor,[Sock]), - Res = oct_datapump(Config, Port, 16#10000FFFF), - gen_tcp:close(Sock), + ?TC_TRY(wrapping_oct, fun() -> do_wrapping_oct(Config) end). + +%% {recbuf, 8192}, {sndbuf, 8192} +do_wrapping_oct(Config) -> + ?P("[ctrl] create listen socket"), + Ctrl = self(), + {ok, LSock} = ?LISTEN(Config, 0, [{active,false},{mode,binary}]), + {ok, LPort} = inet:port(LSock), + ?P("[ctrl] spawn acceptor"), + Acceptor = spawn_link(fun() -> oct_acceptor(Ctrl, LSock) end), + ?P("[ctrl] spawn pump"), + Pump = spawn_link(fun() -> + oct_datapump(Ctrl, + Config, LPort, 16#10000FFFF) + end), + ?P("[ctrl] await acceptor socket"), + ASock = wc_await_socket("acceptor", Acceptor), + ?P("[ctrl] await pump socket"), + PSock = wc_await_socket("pump", Pump), + ?P("[ctrl] await completion (from pump)"), + Res = wc_await_completion(Acceptor, ASock, Pump, PSock), + ?P("[ctrl] close listen socket"), + gen_tcp:close(LSock), + ?P("[ctrl] verify result"), ok = Res, + ?P("[ctrl] done"), ok. -oct_datapump(Config, Port, N) -> - {ok,Sock} = ?CONNECT(Config, "localhost", Port, - [{active,false},{mode,binary}]), - oct_pump(Sock,N,binary:copy(<<$a:8>>,100000),0). +wc_await_socket(Tag, Pid) -> + receive + {Pid, socket, AS} -> + ?P("received ~s socket: " + "~n ~p", [Tag, AS]), + AS + end. -oct_pump(S,N,_,_) when N =< 0 -> - gen_tcp:close(S), - ok; -oct_pump(S,N,Bin,Last) -> - case gen_tcp:send(S,Bin) of +wc_await_completion(Acceptor, ASock, Pump, PSock) -> + wc_await_completion(Acceptor, ASock, Pump, PSock, 1, undefined). + +wc_await_completion(undefined = _Acceptor, _ASock, + undefined = _Pump, _PSock, + Loops, Res) -> + ?P("completed after ~w loops", [Loops]), + Res; +wc_await_completion(Acceptor, ASock, Pump, PSock, Loops, CRes) -> + receive + {'EXIT', Pump, Res} -> + ?P("[ctrl] Received pump exit: " + "~n Loops: ~w" + "~n Reason: ~p", [Loops, Res]), + wc_await_completion(Acceptor, ASock, + undefined, PSock, + Loops + 1, Res); + {'EXIT', Acceptor, Res} -> + ?P("[ctrl] Received acceptor exit: " + "~n Loops: ~w" + "~n Reason: ~p", [Loops, Res]), + wc_await_completion(undefined, ASock, + Pump, PSock, + Loops + 1, CRes) + + after 10000 -> + ASockInfo = wc_sock_info(ASock), + AccInfo = wc_proc_info(Acceptor), + PSockInfo = wc_sock_info(PSock), + PumpInfo = wc_proc_info(Pump), + ?P("Info ~w while waiting for clompletion: " + "~n Acceptor Socket Info: ~p" + "~n Acceptor Info: ~p" + "~n Pump Socket Info: ~p" + "~n Pump Info: ~p", + [Loops, ASockInfo, AccInfo, PSockInfo, PumpInfo]), + wc_await_completion(Acceptor, ASock, + Pump, PSock, + Loops + 1, CRes) + end. + +wc_sock_info(S) -> + try inet:info(S) + catch + _:_:_ -> + undefined + end. + +wc_proc_info(P) when is_pid(P) -> + try erlang:process_info(P) + catch + _:_:_ -> + undefined + end; +wc_proc_info(_) -> + undefined. + +%% {recbuf, 16*1024}, {sndbuf, 16*1024} +oct_datapump(Ctrl, Config, Port, N) -> + ?P("[pump] connect to listener"), + {ok, CSock} = ?CONNECT(Config, "localhost", Port, + [{active, false}, {mode, binary}]), + ?P("[pump] announce to ctrl"), + Ctrl ! {self(), socket, CSock}, + {ok, [{sndbuf,SndBuf}]} = inet:getopts(CSock, [sndbuf]), + ?P("[pump] connected - start 'pumping' with" + "~n SndBuf: ~w", [SndBuf]), + put(action, nothing), + put(sent, 0), + put(elapsed, 0), + put(rem_bytes, N), + oct_pump(CSock, N, binary:copy(<<$a:8>>,100000), 0, 0). + +oct_pump(S, N, _, _, _Sent) when N =< 0 -> + ?P("[pump] done"), + (catch gen_tcp:close(S)), + exit(ok); +oct_pump(S, N, Bin, Last, Sent) -> + put(action, send), + Start = erlang:monotonic_time(nanosecond), + Res = gen_tcp:send(S, Bin), + Stop = erlang:monotonic_time(nanosecond), + Elapsed = get(elapsed), + put(elapsed, Elapsed + (Stop - Start)), + put(action, sent), + put(sent, Sent+1), + case Res of ok -> - {ok,Stat}=inet:getstat(S), - {_,R}=lists:keyfind(send_oct,1,Stat), - case (R < Last) of - true -> - io:format("ERROR (output) ~p < ~p~n",[R,Last]), - output_counter_error; - false -> - oct_pump(S,N-byte_size(Bin),Bin,R) + case inet:getstat(S) of + {ok, Stat} -> + {_, R} = lists:keyfind(send_oct, 1, Stat), + case (R < Last) of + true -> + ?P("[pump] send counter error ~p < ~p", [R, Last]), + (catch gen_tcp:close(S)), + exit({error, {output_counter, R, Last, N}}); + false -> + put(rem_bytes, N - byte_size(Bin)), + oct_pump(S, N-byte_size(Bin), Bin, R, Sent+1) + end; + {error, StatReason} -> + ?P("[pump] get stat failed:" + "~n Reason: ~p" + "~n when" + "~n Remaining: ~p" + "~n Last: ~p", [StatReason, N, Last]), + (catch gen_tcp:close(S)), + exit({error, {stat_failure, StatReason, N, Last}}) end; - _ -> - input_counter_error + {error, SendReason} -> + ?P("[pump] send failed:" + "~n Reason: ~p" + "~n when" + "~n Remaining: ~p" + "~n Last: ~p", [SendReason, N, Last]), + (catch gen_tcp:close(S)), + exit({error, {send_failure, SendReason, N, Last}}) end. -oct_acceptor(Sock) -> - {ok,Data} = gen_tcp:accept(Sock), - oct_aloop(Data,0,0). +oct_acceptor(Ctrl, LSock) -> + ?P("[acceptor] accept connection"), + {ok, ASock} = gen_tcp:accept(LSock), + ?P("[acceptor] announce to ctrl"), + Ctrl ! {self(), socket, ASock}, + {ok, [{recbuf,RecBuf}]} = inet:getopts(ASock, [recbuf]), + ?P("[acceptor] connection accepted - start receiving with: " + "~n RecBuf: ~w", [RecBuf]), + put(action, nothing), + put(recv, 0), + put(elapsed, 0), + oct_aloop(ASock, inet:info(ASock), 0, 0). -oct_aloop(S,X,Times) -> - case gen_tcp:recv(S,0) of - {ok,_} -> - {ok,Stat}=inet:getstat(S), - {_,R}=lists:keyfind(recv_oct,1,Stat), - case (R < X) of - true -> - io:format("ERROR ~p < ~p~n",[R,X]), - gen_tcp:close(S), - input_counter_error; - false -> - case Times rem 16#FFFFF of - 0 -> - io:format("Read: ~p~n",[R]); - _ -> - ok - end, - oct_aloop(S,R,Times+1) - end; - _ -> - gen_tcp:close(S), - closed +oct_aloop(S, LastInfo, Received, Times) -> + put(action, recv), + Start = erlang:monotonic_time(), + Res = gen_tcp:recv(S, 0), + Stop = erlang:monotonic_time(), + Elapsed = get(elapsed), + put(elapsed, Elapsed + (Stop - Start)), + put(action, received), + put(recv, Times+1), + case Res of + {ok, _} -> + #{counters := #{recv_oct := R} = _Stat} = Info = inet:info(S), + case (R < Received) of + true -> + ?P("[acceptor] recv counter error:" + "~n Recv Cnt: ~p" + "~n Received: ~p" + "~n Times: ~p" + "~n Info: ~p" + "~n Last Info: ~p", + [R, Received, Times, Info, LastInfo]), + (catch gen_tcp:close(S)), + {error, {output_counter, R, Received, Times}}; + false -> + case Times rem 16#FFFFF of + 0 -> + ?P("[acceptor] read: ~p" + "~n Times: ~w" + "~n Info: ~p", + [R, Times, Info]); + _ -> + ok + end, + oct_aloop(S, Info, R, Times+1) + end; + + {error, RecvReason} -> + ?P("[acceptor] receive failed:" + "~n Reason: ~p" + "~n when" + "~n Received: ~p" + "~n Times: ~p", [RecvReason, Received, Times]), + (catch gen_tcp:close(S)), + ct:sleep(1000), % Just give the 'pump' a chance to get there first + exit(closed) end. ok({ok,V}) -> V. @@ -5563,143 +6080,197 @@ otp_13939(Config) when is_list(Config) -> ct:fail("Server process blocked on send.") end. + +%% No point in running this as the test case with inet_backend = socket +%% as it tests a bug in the inet driver that cannot reproduced with +%% inet_backend = socket. otp_12242(Config) when is_list(Config) -> - case os:type() of - {win32,_} -> - %% Even if we set sndbuf and recbuf to small sizes - %% Windows either happily accepts to send GBytes of data - %% in no time, so the second send below that is supposed - %% to time out just succedes, or the first send that - %% is supposed to fill the inet_drv I/O queue and - %% start waiting for when more data can be sent - %% instead sends all data but suffers a send - %% failure that closes the socket - {skipped,backpressure_broken_on_win32}; - _ -> - %% Find the IPv4 address of an up and running interface - %% that is not loopback nor pointtopoint - {ok,IFList} = inet:getifaddrs(), - ct:pal("IFList ~p~n", [IFList]), - case - lists:flatten( - [lists:filtermap( - fun ({addr,Addr}) when tuple_size(Addr) =:= 4 -> - {true,Addr}; - (_) -> - false - end, Opts) - || {_,Opts} <- IFList, - case lists:keyfind(flags, 1, Opts) of - {_,Flags} -> - lists:member(up, Flags) - andalso - lists:member(running, Flags) - andalso - not lists:member(loopback, Flags) - andalso - not lists:member(pointtopoint, Flags); - false -> - false - end]) - of - [Addr|_] -> - otp_12242(Config, Addr); - Other -> - {skipped,{no_external_address,Other}} - end + Cond = fun() -> + case os:type() of + {win32,_} -> + %% Even if we set sndbuf and recbuf to small sizes + %% Windows either happily accepts to send GBytes of + %% data in no time, so the second send below that + %% is supposed to time out just succedes, or the + %% first send that is supposed to fill the inet_drv + %% I/O queue and start waiting for when more data + %% can be sent instead sends all data but suffers + %% a send failure that closes the socket. + {skip, backpressure_broken_on_win32}; + _ -> + case ?IS_SOCKET_BACKEND(Config) of + true -> + {skip, "Not complient with socket"}; + false -> + ok + end + end + end, + TC = fun() -> do_otp_12242(Config) end, + ?TC_TRY(otp_12242, Cond, TC). + +do_otp_12242(Config) when is_list(Config) -> + %% Find the IPv4 address of an up and running interface + %% that is not loopback nor pointtopoint + {ok, IFList} = inet:getifaddrs(), + ?P("IFList " + "~n ~p", [IFList]), + case + lists:flatten( + [lists:filtermap( + fun ({addr, Addr}) when (tuple_size(Addr) =:= 4) -> + {true, Addr}; + (_) -> + false + end, Opts) + || {_,Opts} <- IFList, + case lists:keyfind(flags, 1, Opts) of + {_,Flags} -> + lists:member(up, Flags) + andalso + lists:member(running, Flags) + andalso + not lists:member(loopback, Flags) + andalso + not lists:member(pointtopoint, Flags); + false -> + false + end]) + of + [Addr|_] -> + otp_12242(Config, Addr); + Other -> + ?SKIPT({no_external_address, Other}) end. + %% -otp_12242(Config, Addr) when tuple_size(Addr) =:= 4 -> +otp_12242(Config, Addr) when (tuple_size(Addr) =:= 4) -> ct:timetrap(30000), - ct:pal("Using address ~p~n", [Addr]), - Bufsize = 16 * 1024, + ?P("Using address ~p", [Addr]), + Bufsize = 16 * 1024, Datasize = 128 * 1024 * 1024, % At least 1 s on GBit interface - Blob = binary:copy(<<$x>>, Datasize), + Blob = binary:copy(<<$x>>, Datasize), LOpts = - [{backlog,4},{reuseaddr,true},{ip,Addr}, - binary,{active,false}, - {recbuf,Bufsize},{sndbuf,Bufsize},{buffer,Bufsize}], + [{backlog, 4}, + {reuseaddr, true}, + {ip, Addr}, + binary, + {active, false}, + {recbuf, Bufsize}, + {sndbuf, Bufsize}, + {buffer, Bufsize}], COpts = - [binary,{active,false},{ip,Addr}, - {linger,{true,1}}, % 1 s - {send_timeout,500}, - {recbuf,Bufsize},{sndbuf,Bufsize},{buffer,Bufsize}], + [binary, + {active, false}, + {ip, Addr}, + {linger, {true, 1}}, % 1 s + {send_timeout, 500}, + {recbuf, Bufsize}, + {sndbuf, Bufsize}, + {buffer, Bufsize}], Dir = filename:dirname(code:which(?MODULE)), - {ok,ListenerNode} = - test_server:start_node( - ?UNIQ_NODE_NAME, slave, [{args,"-pa " ++ Dir}]), + {ok, ListenerNode} = ?START_SLAVE_NODE(?UNIQ_NODE_NAME, "-pa " ++ Dir), Tester = self(), - Listener = - spawn( + ?P("create listener"), + {Listener, ListenerMRef} = + spawn_opt( ListenerNode, fun () -> - {ok,L} = ?LISTEN(Config, 0, LOpts), + ?P("create listen socket"), + {ok,L} = ?LISTEN(Config, 0, LOpts), {ok,LPort} = inet:port(L), - Tester ! {self(),port,LPort}, - {ok,A} = gen_tcp:accept(L), - ok = gen_tcp:close(L), + ?P("inform tester about port number: ~w", [LPort]), + Tester ! {self(), port, LPort}, + ?P("try accept"), + {ok, A} = gen_tcp:accept(L), + ?P("accepted - close listen socket"), + ok = gen_tcp:close(L), + ?P("await close order"), receive - {Tester,stop} -> + {Tester, stop} -> + ?P("close order received - close accepted socket"), ok = gen_tcp:close(A) end - end), - ListenerMref = monitor(process, Listener), + end, [monitor]), + ?P("await listen port"), LPort = receive {Listener,port,P} -> P end, + ?P("try connect to ~w", [LPort]), {ok,C} = ?CONNECT(Config, Addr, LPort, COpts, infinity), - {ok,ReadCOpts} = inet:getopts(C, [recbuf,sndbuf,buffer]), - ct:pal("ReadCOpts ~p~n", [ReadCOpts]), - %% + ?P("connected - get buffers"), + {ok, ReadCOpts} = inet:getopts(C, [recbuf,sndbuf,buffer]), + ?P("connected sockets buffers:" + "~n ~p", [ReadCOpts]), + + otp_12242_2(C, Blob, Datasize), + + ?P("send listener stop"), + Listener ! {Tester, stop}, + ?P("await listener termination"), + wait(ListenerMRef), + ?P("stop listener node"), + ?STOP_NODE(ListenerNode), + ?P("done"), + ok. + + +otp_12242_2(C, Blob, Datasize) when is_port(C) -> %% Fill the buffers - ct:pal("Sending ~p bytes~n", [Datasize]), + ?P("sending ~p bytes", [Datasize]), ok = gen_tcp:send(C, Blob), - ct:pal("Sent ~p bytes~n", [Datasize]), - %% Spawn the Closer, - %% try to ensure that the close call is in progress + ?P("sent ~p bytes", [Datasize]), + + %% Try to ensure that the close call is in progress %% before the owner proceeds with sending - Owner = self(), - {_Closer,CloserMref} = - spawn_opt( - fun () -> - Owner ! {tref, erlang:start_timer(50, Owner, closing)}, - ct:pal("Calling gen_tcp:close(C)~n"), - try gen_tcp:close(C) of - Result -> - ct:pal("gen_tcp:close(C) -> ~p~n", [Result]), - ok = Result - catch - Class:Reason:Stacktrace -> - ct:pal( - "gen_tcp:close(C) >< ~p:~p~n ~p~n", - [Class,Reason,Stacktrace]), - erlang:raise(Class, Reason, Stacktrace) - end - end, [link,monitor]), + CloserMRef = otp_12242_closer(C), + + ?P("await tref"), receive {tref,Tref} -> + ?P("tref received - now await trigger timeout"), receive {timeout,Tref,_} -> ok end, - ct:pal("Sending ~p bytes again~n", [Datasize]), + ?P("trigger timeout received - try send ~p bytes again", + [Datasize]), %% Now should the close be in progress... %% All buffers are full, remote end is not reading, %% and the send timeout is 1 s so this will timeout: - {error,timeout} = gen_tcp:send(C, Blob), - ct:pal("Sending ~p bytes again timed out~n", [Datasize]), - ok = inet:setopts(C, [{send_timeout,10000}]), + {error, timeout} = gen_tcp:send(C, Blob), + ?P("expected timeout - update send_timeout (to 10000)"), + ok = inet:setopts(C, [{send_timeout, 10000}]), %% There is a hidden timeout here. Port close is sampled %% every 5 s by prim_inet:send_recv_reply. %% Linger is 3 s so the Closer will finish this send: - ct:pal("Sending ~p bytes with 10 s timeout~n", [Datasize]), - {error,closed} = gen_tcp:send(C, Blob), - ct:pal("Sending ~p bytes with 10 s timeout was closed~n", - [Datasize]), - normal = wait(CloserMref), - ct:pal("The Closer has exited~n"), - Listener ! {Tester,stop}, - receive {'DOWN',ListenerMref,_,_,_} -> ok end, - ct:pal("The Listener has exited~n"), - test_server:stop_node(ListenerNode), + ?P("try send ~p bytes - expect error (closed)", [Datasize]), + {error, closed} = gen_tcp:send(C, Blob), + ?P("await closer termination"), + normal = wait(CloserMRef), ok end. +otp_12242_closer(C) -> + Owner = self(), + {_Closer, CloserMref} = + spawn_opt( + fun () -> + ?P("[closer] starting"), + Owner ! {tref, erlang:start_timer(50, Owner, closing)}, + ?P("[closer] calling gen_tcp:close(C)"), + try gen_tcp:close(C) of + Result -> + ?P("[closer] gen_tcp:close(C) -> ~p", [Result]), + ok = Result + catch + Class:Reason:Stacktrace -> + ?P("[closer] catched gen_tcp:close(C):" + "~n Error Class: ~p" + "~n Error: ~p" + "~n Stack trace: ~p", + [Class, Reason, Stacktrace]), + erlang:raise(Class, Reason, Stacktrace) + end + end, [link, monitor]), + CloserMref. + + wait(Mref) -> receive {'DOWN',Mref,_,_,Reason} -> Reason end. @@ -5772,73 +6343,70 @@ delay_send_error2(Sock, N) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -send_failed_str(Reason) -> - ?F("Send failed: ~w", [Reason]). - -connect_failed_str(Reason) -> - ?F("Connect failed: ~w", [Reason]). - -listen_failed_str(Reason) -> - ?F("Listen failed: ~w", [Reason]). - -accept_failed_str(Reason) -> - ?F("Accept failed: ~w", [Reason]). - -port_failed_str(Reason) -> - ?F("Port failed: ~w", [Reason]). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -define(ACTIVE_N, 20). -%% 30-second test for gen_tcp in {active, N} mode, ensuring it does not get stuck. +%% 30-second test for gen_tcp in {active, N} mode, +%% ensuring it does not get stuck. %% Verifies that erl_check_io properly handles extra EPOLLIN signals. bidirectional_traffic(Config) when is_list(Config) -> ?TC_TRY(bidirectional_traffic, fun() -> do_bidirectional_traffic(Config) end). -do_bidirectional_traffic(_Config) -> +do_bidirectional_traffic(Config) -> ?P("begin"), Workers = erlang:system_info(schedulers_online) * 2, ?P("Use ~w workers", [Workers]), Payload = crypto:strong_rand_bytes(32), - ?P("create listen socket"), - {ok, LSock} = gen_tcp:listen(0, [binary, {packet, 0}, {active, false}, {reuseaddr, true}]), + {ok, LSock} = ?LISTEN(Config, + 0, + [binary, + {packet, 0}, + {active, false}, + {reuseaddr, true}]), %% get all sockets to know failing ends + ?P("listen socket created: " + "~n ~p", [LSock]), {ok, Port} = inet:port(LSock), ?P("listen socket port number ~w", [Port]), Control = self(), ?P("create ~w receivers", [Workers]), - Receivers = [spawn_link(fun () -> exchange(LSock, Port, Payload, Control) end) || _ <- lists:seq(1, Workers)], + Receivers = [spawn_link(fun () -> + exchange(Config, LSock, Port, Payload, Control) + end) || _ <- lists:seq(1, Workers)], ?P("await the result"), Result = + %% If any of the receivers report, we have an error receive {timeout, Socket, Total} -> ?P("timeout msg for ~p: ~w", [Socket, Total]), {fail, {timeout, Socket, Total}}; - {error, Socket, Reason} -> + {error, Socket, Reason} -> ?P("error msg for ~p: ~p", [Socket, Reason]), {fail, {error, Socket, Reason}} after 30000 -> %% if it does not fail in 30 seconds, it most likely works - ?P("timeout => success?"), + ?P("timeout => success"), ok end, - ?P("terminate receivers"), + ?P("ensure all receivers terminated"), [begin unlink(Rec), exit(Rec, kill) end || Rec <- Receivers], + ?P("close listen socket"), + (catch gen_tcp:close(LSock)), ?P("done"), Result. -exchange(LSock, Port, Payload, Control) -> + +exchange(Config, LSock, Port, Payload, Control) -> %% spin up client - _ClntRcv = spawn( + _ClntRcv = spawn_link( fun () -> ?P("connect"), {ok, Client} = - gen_tcp:connect("localhost", - Port, - [binary, {packet, 0}, {active, ?ACTIVE_N}]), + ?CONNECT(Config, + "localhost", + Port, + [binary, {packet, 0}, {active, ?ACTIVE_N}]), ?P("connected: ~p", [Client]), send_recv_loop(Client, Payload, Control) end), @@ -5856,8 +6424,14 @@ send_recv_loop(Socket, Payload, Control) -> ?P("spawn sender"), _Snd = spawn_link( fun Sender() -> - _ = gen_tcp:send(Socket, Payload), - Sender() + case gen_tcp:send(Socket, Payload) of + ok -> + Sender(); + {error, Reason} -> + ?P("Send failed: " + "~n ~p", [Reason]), + exit({send_failed, Reason}) + end end), ?P("begin recv"), recv(Socket, 0, Control). @@ -5870,32 +6444,47 @@ recv(Socket, Total, Control) -> inet:setopts(Socket, [{active, ?ACTIVE_N}]), recv(Socket, Total, Control); {tcp_closed, Socket} -> - ?P("[recv] closed when total received: ~w", [Total]), - exit(terminate); + ?P("[recv] closed when total received: ~w" + "~n Socket Info: ~p", + [Total, (catch inet:info(Socket))]), + ok; Other -> ?P("[recv] received unexpected when total received: ~w" "~n ~p" "~n Socket: ~p" - "~n Port stat: ~p" - "~n Port status: ~p" - "~n Port Info: ~p", - [Total, Other, - Socket, - (catch inet:getstat(Socket)), - (catch prim_inet:getstatus(Socket)), - (catch erlang:port_info(Socket))]), + "~n Socket Info: ~p", + [Total, Other, Socket, (catch inet:info(Socket))]), Control ! {error, Socket, Other} after 2000 -> - %% no data received in 2 seconds, test failed + %% no data received in 2 seconds => test failed ?P("[recv] received nothing when total received: ~w" "~n Socket: ~p" - "~n Port stat: ~p" - "~n Port status: ~p" - "~n Port Info: ~p", - [Total, - Socket, - (catch inet:getstat(Socket)), - (catch prim_inet:getstatus(Socket)), - (catch erlang:port_info(Socket))]), + "~n Socket Info: ~p", + [Total, Socket, (catch inet:info(Socket))]), Control ! {timeout, Socket, Total} end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pi(Item) -> + {Item, Val} = process_info(self(), Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +send_failed_str(Reason) -> + ?F("Send failed: ~w", [Reason]). + +connect_failed_str(Reason) -> + ?F("Connect failed: ~w", [Reason]). + +listen_failed_str(Reason) -> + ?F("Listen failed: ~w", [Reason]). + +accept_failed_str(Reason) -> + ?F("Accept failed: ~w", [Reason]). + +port_failed_str(Reason) -> + ?F("Port failed: ~w", [Reason]). diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index a52f70933e..5e78ab6f58 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2020. All Rights Reserved. +%% Copyright Ericsson AB 1998-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -100,6 +100,9 @@ init_per_suite(Config0) -> ?P("init_per_suite -> end when " "~n Config: ~p", [Config1]), + %% We need a monitor on this node also + kernel_test_sys_monitor:start(), + Config1 end. @@ -109,10 +112,13 @@ end_per_suite(Config0) -> "~n Config: ~p" "~n Nodes: ~p", [Config0, erlang:nodes()]), + %% Stop the local monitor + kernel_test_sys_monitor:stop(), + Config1 = ?LIB:end_per_suite(Config0), ?P("end_per_suite -> " - "~n Nodes: ~p", [erlang:nodes()]), + "~n Nodes: ~p", [erlang:nodes()]), Config1. @@ -133,15 +139,49 @@ end_per_group(_GroupName, Config) -> Config. -init_per_testcase(read_packets, Config) -> +init_per_testcase(Case, Config0) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config0, erlang:nodes(), pi(links), pi(monitors)]), + + kernel_test_global_sys_monitor:reset_events(), + + Config1 = init_per_testcase2(Case, Config0), + + ?P("init_per_testcase -> done when" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), + Config1. + +init_per_testcase2(read_packets, Config) -> ct:timetrap({minutes, 2}), Config; -init_per_testcase(_Case, Config) -> +init_per_testcase2(_Case, Config) -> Config. -end_per_testcase(_Case, _Config) -> + +end_per_testcase(_Case, Config) -> + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + ?P("system events during test: " + "~n ~p", [kernel_test_global_sys_monitor:events()]), + + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), ok. + %%------------------------------------------------------------- %% Send two packets to a closed port (on some systems this causes the socket %% to be closed). @@ -793,7 +833,7 @@ sendtos_ok({unix,_}, _) -> true; sendtos_ok(_, _) -> false. %% Using the option returns einval, so it is not implemented. -sendttl_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); +sendttl_ok({unix,darwin}, _OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); sendttl_ok({unix,linux}, OSVer) -> not semver_lt(OSVer, {4,0,0}); %% Using the option returns enoprotoopt, so it is not implemented. sendttl_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); @@ -1279,6 +1319,13 @@ get_localaddr([Localhost|Ls]) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +pi(Item) -> + {Item, Val} = process_info(self(), Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% %% Utils %% diff --git a/lib/kernel/test/global_SUITE.erl b/lib/kernel/test/global_SUITE.erl index 22db756d97..689ea09792 100644 --- a/lib/kernel/test/global_SUITE.erl +++ b/lib/kernel/test/global_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2019. All Rights Reserved. +%% Copyright Ericsson AB 1997-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,8 +19,11 @@ %% -module(global_SUITE). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, +-export([all/0, suite/0, groups/0, init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, + names/1, names_hidden/1, locks/1, locks_hidden/1, bad_input/1, names_and_locks/1, lock_die/1, name_die/1, basic_partition/1, basic_name_partition/1, @@ -38,18 +41,77 @@ both_known_1/1, lost_unregister/1, mass_death/1, - garbage_messages/1]). - --export([global_load/3, lock_global/2, lock_global2/2]). - --export([]). --export([init_mass_spawn/1]). + garbage_messages/1, + + ring_line/1 + ]). + +%% Not used +-export([simple_dis/4, + config_dc/4, + w/2, + check_same/2, + check_same/1, + stop/0, + start_nodes_serially/3]). + +-export([lock_global/2, lock_global2/2]). + +-export([ + %% Called via ?RES + resolve_none/3, + resolve_first/3, + resolve_second/3, + bad_resolver/3, + badrpc_resolver/3, + lock_resolver/3, + exit_resolver/3, + disconnect_first/3, + halt_second/3, + init_mass_spawn/1, + + %% Called via rpc_cast + part_2_2/4, + part1/4, + part1_5/4, + part2/10, + part3/10, + crash/1, + single_node/3, + part_ext/4, + isolated_node/4, + mk_part_node/3, + alone/2, + global_load/3, + + %% Called via rpc:call + halt_node/1, + start_proc/0, start_proc/1, + start_proc2/1, + start_proc3/1, + start_proc4/1, + start_proc_basic/1, + start_resolver/2, + + %% Called as a fun (fun ?MODULE:function/x) + fix_basic_name/3, + + %% Called via spawn + init_proc_basic/2, + init_2/0, + p_init/1, p_init/2, + p_init2/2 + + ]). -export([start_tracer/0, stop_tracer/0, get_trace/0]). --compile(export_all). +%% Exports for error_logger handler +-export([init/1, handle_event/2, handle_info/2, handle_call/2, terminate/2]). + -include_lib("common_test/include/ct.hrl"). +-include("kernel_test_lib.hrl"). -define(NODES, [node()|nodes()]). @@ -66,7 +128,8 @@ all() -> case init:get_argument(ring_line) of {ok, _} -> [ring_line]; _ -> - [names, names_hidden, locks, locks_hidden, bad_input, + [ + names, names_hidden, locks, locks_hidden, bad_input, names_and_locks, lock_die, name_die, basic_partition, advanced_partition, basic_name_partition, stress_partition, simple_ring, simple_line, ring, line, @@ -75,7 +138,8 @@ all() -> simple_resolve2, simple_resolve3, leftover_name, re_register_name, name_exit, external_nodes, many_nodes, sync_0, global_groups_change, register_1, both_known_1, - lost_unregister, mass_death, garbage_messages] + lost_unregister, mass_death, garbage_messages + ] end. groups() -> @@ -102,29 +166,59 @@ end_per_suite(_Config) -> -define(nodes_tag, '$global_nodes'). -define(registered, proplists:get_value(registered, Config)). -init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - ok = gen_server:call(global_name_server, high_level_trace_start,infinity), +init_per_testcase(Case, Config0) when is_atom(Case) andalso is_list(Config0) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config0, erlang:nodes(), pi(links), pi(monitors)]), + + ok = gen_server:call(global_name_server, + high_level_trace_start, + infinity), %% Make sure that everything is dead and done. Otherwise there are problems %% on platforms on which it takes a long time to shut down a node. stop_nodes(nodes()), timer:sleep(1000), - [{?TESTCASE, Case}, {registered, registered()} | Config]. + Config1 = [{?TESTCASE, Case}, {registered, registered()} | Config0], + + ?P("init_per_testcase -> done when" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [Config1, erlang:nodes(), pi(links), pi(monitors)]), + + Config1. end_per_testcase(_Case, Config) -> - ct:log("Calling end_per_testcase!",[]), + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + write_high_level_trace(Config), - _ = - gen_server:call(global_name_server, high_level_trace_stop, infinity), + _ = gen_server:call(global_name_server, high_level_trace_stop, infinity), [global:unregister_name(N) || N <- global:registered_names()], InitRegistered = ?registered, Registered = registered(), - [io:format("~s local names: ~p~n", [What, N]) || + + [?P("end_per_testcase -> " + "~n ~s local names: ~p", [What, N]) || {What, N} <- [{"Added", Registered -- InitRegistered}, {"Removed", InitRegistered -- Registered}], N =/= []], + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [erlang:nodes(), pi(links), pi(monitors)]), + ok. %%% General comments: @@ -385,7 +479,7 @@ write_high_level_trace(Config) -> end. write_high_level_trace(Nodes, Config) -> - When = now(), + When = erlang:timestamp(), %% 'info' returns more than the trace, which is nice. Data = [{Node, {info, rpc:call(Node, global, info, [])}} || Node <- Nodes], @@ -419,22 +513,44 @@ lock_global2(Id, Parent) -> %% Kill Pid2 and check that 'test' isn't registered. names(Config) when is_list(Config) -> + ?TC_TRY(names, fun() -> do_names(Config) end). + +do_names(Config) -> + ?P("names -> begin when" + "~n Nodes: ~p" + "~n Names: ~p", + [nodes(), + case net_adm:names() of + {ok, N} -> + N; + _ -> + "-" + end]), Timeout = 30, ct:timetrap({seconds,Timeout}), + ?P("names -> init high level trace"), init_high_level_trace(Timeout), + ?P("names -> init condition"), init_condition(Config), + ?P("names -> get registered names"), OrigNames = global:registered_names(), + ?P("names -> start node cp1"), {ok, Cp1} = start_node(cp1, Config), + ?P("names -> start node cp2"), {ok, Cp2} = start_node(cp2, Config), + ?P("names -> start node cp3"), {ok, Cp3} = start_node(cp3, Config), + ?P("names -> wait for ready net"), wait_for_ready_net(Config), %% start a proc and register it + ?P("names -> start and register process 'test'"), {Pid, yes} = start_proc(test), %% test that it is registered at all nodes + ?P("names -> verify process has been registered on all nodes"), ?UNTIL(begin (Pid =:= global:whereis_name(test)) and (Pid =:= rpc:call(Cp1, global, whereis_name, [test])) and @@ -444,25 +560,32 @@ names(Config) when is_list(Config) -> end), %% try to register the same name + ?P("names -> try register (locally) another 'test' (and expect rejection)"), no = global:register_name(test, self()), + ?P("names -> try register (on cp1) another 'test' (and expect rejection)"), no = rpc:call(Cp1, global, register_name, [test, self()]), %% let process exit, check that it is unregistered automatically + ?P("names -> terminate the 'test' process"), exit_p(Pid), + ?P("names -> verify 'test' process has been automatically unregistered"), ?UNTIL((undefined =:= global:whereis_name(test)) and (undefined =:= rpc:call(Cp1, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp2, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp3, global, whereis_name, [test]))), %% test re_register + ?P("names -> start and register another process 'test'"), {Pid2, yes} = start_proc(test), + ?P("names -> verify process 'test' has been registered"), ?UNTIL(Pid2 =:= rpc:call(Cp3, global, whereis_name, [test])), Pid3 = rpc:call(Cp3, ?MODULE, start_proc2, [test]), ?UNTIL(Pid3 =:= rpc:call(Cp3, global, whereis_name, [test])), Pid3 = global:whereis_name(test), %% test sending + ?P("names -> test sending (from local)"), global:send(test, {ping, self()}), receive {pong, Cp3} -> ok @@ -470,6 +593,7 @@ names(Config) when is_list(Config) -> 2000 -> ct:fail(timeout1) end, + ?P("names -> test sending (from cp1)"), rpc:call(Cp1, global, send, [test, {ping, self()}]), receive {pong, Cp3} -> ok @@ -477,31 +601,43 @@ names(Config) when is_list(Config) -> 2000 -> ct:fail(timeout2) end, + ?P("names -> unregister 'test' process"), _ = global:unregister_name(test), ?UNTIL((undefined =:= global:whereis_name(test)) and (undefined =:= rpc:call(Cp1, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp2, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp3, global, whereis_name, [test]))), + ?P("names -> terminate process 'test'"), exit_p(Pid3), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), %% register a proc + ?P("names -> register a process"), {_Pid6, yes} = rpc:call(Cp3, ?MODULE, start_proc, [test]), + ?P("names -> write high level trace"), write_high_level_trace(Config), %% stop the nodes, and make sure names are released. + ?P("names -> stop node cp1"), stop_node(Cp1), + ?P("names -> stop node cp2"), stop_node(Cp2), + ?P("names -> stop node cp3"), stop_node(Cp3), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), exit_p(Pid2), + ?P("names -> verify not registered"), ?UNTIL(undefined =:= global:whereis_name(test)), init_condition(Config), + + ?P("names -> done"), ok. %% Tests that names on a hidden node doesn't interfere with names on @@ -1164,7 +1300,7 @@ advanced_partition(Config) when is_list(Config) -> %% make cp3-cp6 connected, partitioned from us and cp0-cp2 rpc_cast(Cp3, ?MODULE, part2, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5,Cp6]), + [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), ?UNTIL(is_ready_partition(Config)), %% start different processes in this partition @@ -1266,7 +1402,7 @@ stress_partition(Config) when is_list(Config) -> %% make cp3-cp5 connected, partitioned from us and cp0-cp2 %% cp6 is alone (single node). cp6 pings cp0 and cp3 in 12 secs... rpc_cast(Cp3, ?MODULE, part3, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5,Cp6]), + [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), ?UNTIL(is_ready_partition(Config)), %% start different processes in this partition @@ -1903,9 +2039,9 @@ otp_5737(Config) when is_list(Config) -> {'EXIT', _} = (catch global:set_lock(LockId, Nodes, -1)), {'EXIT', _} = (catch global:set_lock(LockId, Nodes, a)), true = global:set_lock(LockId, Nodes, 0), - Time1 = now(), + Time1 = erlang:timestamp(), false = global:set_lock({?MODULE,not_me}, Nodes, 0), - true = timer:now_diff(now(), Time1) < 5000, + true = timer:now_diff(erlang:timestamp(), Time1) < 5000, _ = global:del_lock(LockId, Nodes), Fun = fun() -> ok end, @@ -2880,7 +3016,7 @@ sync_until() -> sync_until(no_log_file). sync_until(LogFile) -> - Time = ?UNTIL_LOOP - (msec(now()) rem ?UNTIL_LOOP), + Time = ?UNTIL_LOOP - (msec(erlang:timestamp()) rem ?UNTIL_LOOP), catch append_to_file(LogFile, {sync_until, Time}), timer:sleep(Time). @@ -3437,11 +3573,11 @@ is_node_in_part(File, MyPart) -> Rs -> erlang:display({is_node_in_part, resolvers, Rs}), trace_message({node(), is_node_in_part, Rs}), - append_to_file(File, {now(), Known, Nodes, Rs}), + append_to_file(File, {erlang:timestamp(), Known, Nodes, Rs}), false end; _ -> - append_to_file(File, {now(), Known, Nodes}), + append_to_file(File, {erlang:timestamp(), Known, Nodes}), false end. @@ -3497,7 +3633,7 @@ collect_resolves() -> cr(0). cr(Res) -> receive {resolve_called, Name, Node} -> - io:format("resolve called: ~w ~w~n", [Name, Node]), + ?P("resolve called: ~w ~w", [Name, Node]), cr(Res+1) after 0 -> Res @@ -3580,7 +3716,7 @@ loop_2() -> end. msec() -> - msec(now()). + msec(erlang:timestamp()). msec(T) -> element(1,T)*1000000000 + element(2,T)*1000 + element(3,T) div 1000. @@ -3792,7 +3928,7 @@ node_names(Names, Config) -> %% simple_resolve assumes that the node name comes first. node_name(Name, Config) -> U = "_", - {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(now()), + {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(erlang:timestamp()), Date = io_lib:format("~4w_~2..0w_~2..0w__~2..0w_~2..0w_~2..0w", [Y,M,D, H,Min,S]), L = lists:flatten(Date), @@ -3913,13 +4049,13 @@ mass_death(Config) when is_list(Config) -> {H,M,S} = time(), io:format("Started probing: ~.4.0w-~.2.0w-~.2.0w ~.2.0w:~.2.0w:~.2.0w~n", [YYYY,MM,DD,H,M,S]), - wait_mass_death(Nodes, OrigNames, erlang:now(), Config). + wait_mass_death(Nodes, OrigNames, erlang:timestamp(), Config). wait_mass_death(Nodes, OrigNames, Then, Config) -> Names = global:registered_names(), case Names--OrigNames of [] -> - T = now_diff(erlang:now(), Then) div 1000, + T = now_diff(erlang:timestamp(), Then) div 1000, lists:foreach( fun (Node) -> stop_node(Node) @@ -4128,17 +4264,68 @@ garbage_messages(Config) when is_list(Config) -> ok. wait_for_ready_net(Config) -> - wait_for_ready_net(?NODES, Config). + {Pid, MRef} = spawn_monitor(fun() -> + wait_for_ready_net(?NODES, Config) + end), + wait_for_ready_net_loop(Pid, MRef). + +wait_for_ready_net_loop(Pid, MRef) -> + receive + {'DOWN', MRef, process, Pid, Info} -> + ?P("wait-for-ready-net process terminated: " + "~n ~p", [Info]), + ok; + + {'EXIT', ParentPid, {timetrap_timeout, _Timeout, _Stack}} -> + ?P("wait-for-ready-net -> received timetrap timeout:" + "~n Regarding: ~p" + "~n Waiter: ~p" + "~n Current Location: ~p" + "~n Mesages: ~p", + [ParentPid, Pid, pi(Pid, current_location), pi(Pid, messages)]), + ct:fail("Timeout waiting for ready network") + + end. wait_for_ready_net(Nodes0, Config) -> Nodes = lists:sort(Nodes0), - io:format("wait_for_ready_net ~p~n", [Nodes]), + ?P("wait_for_ready_net ->" + "~n Nodes: ~p", [Nodes]), ?UNTIL(begin - lists:all(fun(N) -> Nodes =:= get_known(N) end, Nodes) and + lists:all(fun(N) -> + ?P("wait_for_ready_net -> " + "get known (by global) for ~p", [N]), + GNs = get_known(N), + ?P("wait_for_ready_net -> verify same for ~p:" + "~n Global Known: ~p" + "~n Nodes: ~p", [N, GNs, Nodes]), + GRes = Nodes =:= GNs, + ?P("wait_for_ready_net => ~p", [GRes]), + GRes + end, + Nodes) and lists:all(fun(N) -> - LNs = rpc:call(N, erlang, nodes, []), - Nodes =:= lists:sort([N | LNs]) - end, Nodes) + ?P("wait_for_ready_net -> " + "get erlang nodes for ~p", [N]), + case rpc:call(N, erlang, nodes, []) of + RNs0 when is_list(RNs0) -> + RNs = lists:sort([N | RNs0]), + ?P("wait_for_ready_net -> " + "verify same for ~p: " + "~n Remote nodes: ~p" + "~n (Local) Nodes: ~p", + [N, RNs, Nodes]), + ERes = Nodes =:= RNs, + ?P("wait_for_ready_net => ~p", + [ERes]), + ERes; + BadRes -> + ?P("failed get remote nodes: " + "~n ~p", [BadRes]), + false + end + end, + Nodes) end). get_known(Node) -> @@ -4180,7 +4367,7 @@ rpc_cast(Node, Module, Function, Args, File) -> pong -> rpc:cast(Node, Module, Function, Args); Else -> - append_to_file(File, {now(), {rpc_cast, Node, Module, Function, + append_to_file(File, {erlang:timestamp(), {rpc_cast, Node, Module, Function, Args, Else}}) %% Maybe we should crash, but it probably doesn't matter. end. @@ -4218,14 +4405,14 @@ start_tracer() -> tracer(L) -> receive %% {save, Term} -> - %% tracer([{now(),Term} | L]); + %% tracer([{erlang:timestamp(),Term} | L]); {get, From} -> From ! {trace, lists:reverse(L)}, tracer([]); stop -> exit(normal); Term -> - tracer([{now(),Term} | L]) + tracer([{erlang:timestamp(),Term} | L]) end. stop_tracer() -> @@ -4256,6 +4443,17 @@ trace_message(M) -> ok end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pi(Item) -> + pi(self(), Item). +pi(Pid, Item) -> + {Item, Val} = process_info(Pid, Item), + Val. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%----------------------------------------------------------------- %% The error_logger handler used for OTP-6931. %%----------------------------------------------------------------- diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 7aeb22d1be..8287068931 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -147,13 +147,33 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. -init_per_testcase(gethostnative_debug_level, Config) -> + +init_per_testcase(Case, Config0) -> + ?P("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config0, erlang:nodes(), pi(links), pi(monitors)]), + + kernel_test_global_sys_monitor:reset_events(), + + Config1 = init_per_testcase2(Case, Config0), + + ?P("init_per_testcase -> done when" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [Config1, erlang:nodes(), pi(links), pi(monitors)]), + Config1. + +init_per_testcase2(gethostnative_debug_level, Config) -> ?TT(?MINS(2)), Config; -init_per_testcase(gethostnative_soft_restart, Config) -> +init_per_testcase2(gethostnative_soft_restart, Config) -> ?TT(?MINS(2)), Config; -init_per_testcase(lookup_bad_search_option, Config) -> +init_per_testcase2(lookup_bad_search_option, Config) -> Db = inet_db, Key = res_lookup, %% The bad option cannot enter through inet_db:set_lookup/1, @@ -163,17 +183,37 @@ init_per_testcase(lookup_bad_search_option, Config) -> ets:insert(Db, {Key,[lookup_bad_search_option]}), ?P("init_per_testcase -> Misconfigured resolver lookup order"), [{Key,Prev}|Config]; -init_per_testcase(_Func, Config) -> +init_per_testcase2(_Func, Config) -> Config. -end_per_testcase(lookup_bad_search_option, Config) -> - Db = inet_db, - Key = res_lookup, +end_per_testcase(Case, Config) -> + ?P("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", + [Config, erlang:nodes(), pi(links), pi(monitors)]), + + ?P("system events during test: " + "~n ~p", [kernel_test_global_sys_monitor:events()]), + + end_per_testcase2(Case, Config), + + ?P("end_per_testcase -> done with" + "~n Nodes: ~p" + "~n Links: ~p" + "~n Monitors: ~p", [erlang:nodes(), pi(links), pi(monitors)]), + ok. + +end_per_testcase2(lookup_bad_search_option, Config) -> + ?P("end_per_testcase2 -> restore resolver lookup order"), + Db = inet_db, + Key = res_lookup, Prev = proplists:get_value(Key, Config), ets:delete(Db, Key), ets:insert(Db, Prev), - ?P("end_per_testcase -> Restored resolver lookup order"); -end_per_testcase(_Func, _Config) -> + ?P("end_per_testcase2 -> resolver lookup order restored"); +end_per_testcase2(_Func, _Config) -> ok. t_gethostbyaddr() -> @@ -1757,3 +1797,11 @@ add_del_host_v6(_Config) -> ok = inet_db:add_host(Ip, [Name, Alias]), {ok, HostEnt} = inet_hosts:gethostbyname(Name, inet6). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pi(Item) -> + {Item, Val} = process_info(self(), Item), + Val. + + diff --git a/lib/kernel/test/interactive_shell_SUITE.erl b/lib/kernel/test/interactive_shell_SUITE.erl index 0dc02e55f0..bd0fa755ac 100644 --- a/lib/kernel/test/interactive_shell_SUITE.erl +++ b/lib/kernel/test/interactive_shell_SUITE.erl @@ -19,10 +19,15 @@ %% -module(interactive_shell_SUITE). -include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, get_columns_and_rows/1, exit_initial/1, job_control_local/1, - job_control_remote/1,stop_during_init/1, custom_shell_history/1, + job_control_remote/1,stop_during_init/1, + shell_history/1, shell_history_resize/1, shell_history_eaccess/1, + shell_history_repair/1, shell_history_repair_corrupt/1, + shell_history_corrupt/1, + shell_history_custom/1, shell_history_custom_errors/1, job_control_remote_noshell/1,ctrl_keys/1, get_columns_and_rows_escript/1, remsh/1, remsh_longnames/1, remsh_no_epmd/1]). @@ -47,11 +52,20 @@ all() -> [get_columns_and_rows_escript,get_columns_and_rows, exit_initial, job_control_local, job_control_remote, job_control_remote_noshell, - ctrl_keys, stop_during_init, custom_shell_history, + ctrl_keys, stop_during_init, {group, shell_history}, remsh, remsh_longnames, remsh_no_epmd]. groups() -> - []. + [{shell_history, [], + [shell_history, + shell_history_resize, + shell_history_eaccess, + shell_history_repair, + shell_history_repair_corrupt, + shell_history_corrupt, + shell_history_custom, + shell_history_custom_errors + ]}]. init_per_suite(Config) -> Term = os:getenv("TERM", "dumb"), @@ -64,6 +78,11 @@ end_per_suite(Config) -> os:putenv("TERM",Term), ok. +init_per_group(shell_history, Config) -> + case proplists:get_value(default_shell, Config) of + old -> {skip, "Not supported in old shell"}; + new -> Config + end; init_per_group(_GroupName, Config) -> Config. @@ -73,7 +92,7 @@ end_per_group(_GroupName, Config) -> %%-define(DEBUG,1). -ifdef(DEBUG). --define(dbg(Data),erlang:display(Data)). +-define(dbg(Data),ct:pal("~p",[Data])). -else. -define(dbg(Data),noop). -endif. @@ -220,31 +239,297 @@ stop_during_init(Config) when is_list(Config) -> Tempdir -> XArg = " -kernel shell_history enabled -s init stop", start_runerl_command(RunErl, Tempdir, "\\\""++Erl++"\\\""++XArg), - {ok, Binary} = file:read_file(filename:join(Tempdir, "erlang.log.1")), - nomatch = binary:match(Binary, <<"*** ERROR: Shell process terminated! ***">>) + Logs = rtnode_read_logs(Tempdir), + rtnode_dump_logs(Logs), + nomatch = binary:match(maps:get("erlang.log.1",Logs), + <<"*** ERROR: Shell process terminated! ***">>) end end. -custom_shell_history(Config) when is_list(Config) -> - case proplists:get_value(default_shell, Config) of - old -> {skip, "Not supported in old shell"}; - new ->%% Up key: Ctrl + P = Cp=[$\^p] - rtnode([ +%% This testcase tests that shell_history works as it should. +%% We use Ctrl + P = Cp=[$\^p] in order to navigate up +%% We use Ctrl + N = Cp=[$\^n] in order to navigate down +%% We use Ctrl + B = Cp=[$\^b] in order to navigate left +%% in the console. We also need to sleep for a while in order +%% for the system to update the display before we enter more +%% commands. +shell_history(Config) when is_list(Config) -> + Path = shell_history_path(Config, "basic"), + rtnode([ + {putline, "echo1."}, + {getline, "echo1"}, + {putline, "echo2."}, + {getline, "echo2"}, + {putline, "echo3."}, + {getline, "echo3"}, + {putline, "echo4."}, + {getline, "echo4"}, + {putline, "echo5."}, + {getline, "echo5"} + ], [], [], " -kernel shell_history enabled " ++ + "-kernel shell_history_drop '[\\\"init:stop().\\\"]' " ++ + mk_sh_param(Path)), + rtnode([ + {putline, ""}, + %% the init:stop that stopped the node is dropped + {putdata, [$\^p]}, {getdata, "echo5."}, + {putdata, [$\n]}, + {getline, "echo5"}, + {putdata, [$\^p]}, {getdata,"echo5."}, + {putdata, [$\^p]}, {getdata,"echo4."}, + {putdata, [$\^p]}, {getdata,"echo3."}, + {putdata, [$\^p]}, {getdata,"echo2."}, + {putdata, [$\^n]}, {getdata,"echo3."}, + {putdata, [$\^n]}, {getdata,"echo4."}, + {putdata, [$\^b]}, {sleep,50}, %% the echo4. (cursor moved one left) + {putline, ["echo"]}, + {getline, "echo4echo"} + ], [], [], " -kernel shell_history enabled " ++ mk_sh_param(Path)). + +shell_history_resize(Config) -> + Path = shell_history_path(Config, "resize"), + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], " -kernel shell_history_file_bytes 123456 " ++ + "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + {ok, Logs} = + rtnode([ {putline, ""}, - {putline, [$\^p]}, - {putline_raw, ""}, - {getline, "0"}, - {putline, "echo."}, - {getline, "!echo"} %% exclamation sign is printed by custom history module - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -pz " ++ filename:dirname(code:which(?MODULE))) + {putdata, [$\^p]}, {getdata,"init:stop()."}, + {putdata, [$\^p]}, {getdata,"echo."}, + {putdata, [$\n]}, + {getline, "echo"} + ], [], [], " -kernel shell_history_file_bytes 654321 " ++ + "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + rtnode_check_logs( + "erlang.log.1", + "The configured log history file size is different from the size " + "of the log file on disk", Logs), + + ok. + +shell_history_eaccess(Config) -> + Path = shell_history_path(Config, "eaccess"), + file:make_dir(filename:dirname(Path)), + ok = file:make_dir(Path), + {ok, Info} = file:read_file_info(Path), + try + NoExecMode = Info#file_info.mode band (bnot 8#111), + file:write_file_info(Path,Info#file_info{ mode = NoExecMode }), + + %% Cannot create history log in folder + {ok, Logs1} = + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + rtnode_check_logs("erlang.log.1", "Error handling file", Logs1), + + %% shell_docs recursively creates the folder to store the + %% logs. This test checks that erlang still starts if we + %% cannot create the folders to the path. + {ok, Logs2} = + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ + mk_sh_param(filename:join(Path,"logs"))), + + rtnode_check_logs("erlang.log.1", "Error handling file", Logs2) + + after + file:write_file_info(Path, Info) end. +shell_history_repair(Config) -> + Path = shell_history_path(Config, "repair"), + + %% We stop a node without closing the log + try rtnode([ + {putline, "echo."}, + {getline, "echo"}, + {sleep, 2500}, %% disk_log internal cache timer is 2000 ms + {putline, "erlang:halt(0)."} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)) of + _ -> + ok + catch _:_ -> + ok + end, + + {ok, Logs} = + rtnode([ + {putline, ""}, + {putdata, [$\^p]}, {getdata,"echo."}, + {putdata, [$\n]}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + %% The regexp below checks that he string is NOT part of the log + rtnode_check_logs("erlang.log.1", + "The shell history log file was corrupted and was repaired", + false, + Logs), + ok. + +shell_history_repair_corrupt(Config) -> + Path = shell_history_path(Config, "repair_corrupt"), + + %% We stop a node without closing the log + try rtnode([ + {putline, "echo."}, + {getline, "echo"}, + {sleep, 2500}, %% disk_log internal cache timer is 2000 ms + {putline, "erlang:halt(0)."} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)) of + _ -> + ok + catch _:_ -> + ok + end, + + %% We corrupt the disklog + {ok, D} = file:open(filename:join(Path,"erlang-shell-log.1"), [read,append]), + ok = file:write(D, [10,10]), + ok = file:close(D), + + {ok, Logs} = + rtnode([ + {putline, ""}, + {putdata, [$\^p]}, {getdata,"echo."}, + {putdata, [$\n]}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + rtnode_check_logs("erlang.log.1", + "The shell history log file was corrupted and was repaired.", + Logs), + ok. + +shell_history_corrupt(Config) -> + Path = shell_history_path(Config, "corrupt"), + + %% We stop a node without closing the log + try rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)) of + _ -> + ok + catch _:_ -> + ok + end, + + %% We corrupt the disklog + {ok, D} = file:open(filename:join(Path,"erlang-shell-log.1"), [read, append]), + ok = file:write(D, [10, 10]), + ok = file:close(D), + + {ok, Logs} = + rtnode([ + {putline, ""}, + {putdata, [$\^p]}, {getdata,"init:stop()."}, + {putdata, [$\^p]}, {getdata,"echo."}, + {putdata, [$\n]}, + {getline, "echo"} + ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + + rtnode_check_logs("erlang.log.1", "Invalid chunk in the file", Logs), + ok. + + +shell_history_path(Config, TestCase) -> + filename:join([proplists:get_value(priv_dir, Config), + "shell_history", TestCase]). + +mk_sh_param(Path) -> + "-kernel shell_history_path '\\\"" ++ Path ++ "\\\"'". + +shell_history_custom(_Config) -> + %% Up key: Ctrl + P = Cp=[$\^p] + rtnode([ + {putline, ""}, + {putdata, [$\^p]}, {getdata,"0."}, + {putdata, [$\n]}, + {getline, "0"}, + {putline, "echo."}, + {getline, "!echo"} %% exclamation sign is printed by custom history module + ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ + " -pz " ++ filename:dirname(code:which(?MODULE))). + +shell_history_custom_errors(_Config) -> + + %% Check that we can start with a node with an undefined + %% provider module. + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], " -kernel shell_history very_broken " ++ + " -pz " ++ filename:dirname(code:which(?MODULE))), + + %% Check that we can start with a node with a provider module + %% that crashes in load/0 + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ + " -kernel provider_load crash" ++ + " -pz " ++ filename:dirname(code:which(?MODULE))), + + %% Check that we can start with a node with a provider module + %% that return incorrect in load/0 + rtnode([ + {putline, "echo."}, + {getline, "echo"} + ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ + " -kernel provider_load badreturn" ++ + " -pz " ++ filename:dirname(code:which(?MODULE))), + + %% Check that we can start with a node with a provider module + %% that crashes in load/0 + rtnode([ + {putline, "echo."}, + {getline, "Disabling shell history logging."}, + {getline, "echo"} + ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ + " -kernel provider_add crash" ++ + " -pz " ++ filename:dirname(code:which(?MODULE))), + + %% Check that we can start with a node with a provider module + %% that return incorrect in load/0 + rtnode([ + {putline, "echo."}, + {getline, "It returned {error,badreturn}."}, + {getline, "echo"} + ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ + " -kernel provider_add badreturn" ++ + " -pz " ++ filename:dirname(code:which(?MODULE))). + load() -> - ["0.\n\n"]. + case application:get_env(kernel,provider_load) of + {ok, crash} -> + error(crash); + {ok, badreturn} -> + %% Should return a list of string() + ok; + _ -> + ["0.\n\n"] + end. add(_Line) -> - io:format("!", []). + case application:get_env(kernel,provider_add) of + {ok, crash} -> + error(crash); + {ok, badreturn} -> + %% Should return ok + {error, badreturn}; + _ -> + io:format("!", []) + end. %% Tests that local shell can be started by means of job control. job_control_local(Config) when is_list(Config) -> @@ -436,8 +721,8 @@ remsh(Config) when is_list(Config) -> %% Test that remsh works with explicit -sname rtnode(Cmds ++ [{putline,"nodes()."}, - {getline,"['Remshtest@"++Host++"']"}], [], - [], " -sname Remshtest -remsh " ++ NodeStr), + {getline,"['Remshtest@"++Host++"']"}], + "Remshtest", [], "-remsh " ++ NodeStr), %% Test that remsh works without -sname rtnode(Cmds, [], [], " -remsh " ++ NodeStr) @@ -488,19 +773,27 @@ remsh_no_epmd(Config) when is_list(Config) -> EPMD_ARGS = "-start_epmd false -erl_epmd_port 12345 ", case rtstart([],"ERL_EPMD_PORT=12345 ", EPMD_ARGS ++ " -sname " ++ atom_to_list(?FUNCTION_NAME)) of - {ok, _SRPid, _STPid, SState} -> - {ok, _CRPid, CTPid, CState} = - rtstart([],"ERL_EPMD_PORT=12345 ", - EPMD_ARGS ++ " -remsh "++atom_to_list(?FUNCTION_NAME)), + {ok, _SRPid, STPid, SState} -> try ok = get_and_put( - CTPid, - [{kill_emulator_command,sigint}, - {putline,""}, + STPid, + [{putline,""}, {putline,"node()."}, - {getline_re,atom_to_list(?FUNCTION_NAME)}], 1) + {getline_re,atom_to_list(?FUNCTION_NAME)}], 1), + {ok, _CRPid, CTPid, CState} = + rtstart([],"ERL_EPMD_PORT=12345 ", + EPMD_ARGS ++ " -remsh "++atom_to_list(?FUNCTION_NAME)), + try + ok = get_and_put( + CTPid, + [{kill_emulator_command,sigint}, + {putline,""}, + {putline,"node()."}, + {getline_re,atom_to_list(?FUNCTION_NAME)}], 1) + after + rtstop(CState) + end after - rtstop(CState), %% Stop client before server rtstop(SState) end; Else -> @@ -517,8 +810,14 @@ rtnode(Commands,Nodename,ErlPrefix,Args) -> {ok, _SPid, CPid, RTState} -> erase(getline_skipped), Res = (catch get_and_put(CPid, Commands, 1)), - rtstop(RTState), - ok = Res; + Logs = rtstop(RTState), + case Res of + ok -> + {Res, Logs}; + _Else -> + rtnode_dump_logs(Logs), + ok = Res + end; Skip -> Skip end. @@ -563,8 +862,9 @@ rtstop({CPid, SPid, ToErl, Tempdir}) -> ok end, wait_for_runerl_server(SPid), + Logs = rtnode_read_logs(Tempdir), file:del_dir_r(Tempdir), - ok. + Logs. timeout(long) -> 2 * timeout(normal); @@ -650,6 +950,20 @@ get_and_put(CPid, [{getline_re, Match}|T],N) -> end end; +get_and_put(CPid, [{getdata, Match}|T],N) -> + ?dbg({getdata, Match}), + CPid ! {self(), {get_data, timeout(normal), Match}}, + receive + {get_data, timeout} -> + error_logger:error_msg("~p: getdata timeout waiting for \"~s\" " + "(command number ~p, skipped: ~p)~n", + [?MODULE, Match,N,get(getline_skipped)]), + {error, timeout}; + {get_data, _Data} -> + ?dbg({CPid,data,_Data}), + get_and_put(CPid, T, N+1) + end; + get_and_put(CPid, [{putline_raw, Line}|T],N) -> ?dbg({putline_raw, Line}), CPid ! {self(), {send_line, Line}}, @@ -676,6 +990,19 @@ get_and_put(CPid, [{putline, Line}|T],N) -> "\"~s\" (command number ~p)~n[~p]~n", [?MODULE, Timeout, Line, N,get()]), {error, timeout} + end; +get_and_put(CPid, [{putdata, Data}|T],N) -> + ?dbg({putdata, Data}), + CPid ! {self(), {send_data, Data}}, + Timeout = timeout(normal), + receive + {send_data, ok} -> + get_and_put(CPid, T,N+1) + after Timeout -> + error_logger:error_msg("~p: putline_raw timeout (~p) sending " + "\"~s\" (command number ~p)~n", + [?MODULE, Timeout, Data, N]), + {error, timeout} end. wait_for_runerl_server(SPid) -> @@ -805,13 +1132,11 @@ try_to_erl(_Command, 0) -> {error, cannot_to_erl}; try_to_erl(Command, N) -> ?dbg({?LINE,N}), - Port = open_port({spawn, Command},[eof,{line,1000}]), - Timeout = timeout(normal) div 2, + Port = open_port({spawn, Command},[eof]), + Timeout = timeout(short) div 2, receive - {Port, eof} -> - receive after Timeout -> - ok - end, + {Port, eof} -> + timer:sleep(Timeout), try_to_erl(Command, N-1) after Timeout -> ?dbg(Port), @@ -819,7 +1144,7 @@ try_to_erl(Command, N) -> end. toerl_server(Parent,ToErl,Tempdir) -> - Port = try_to_erl("\""++ToErl++"\" "++Tempdir++"/ 2>/dev/null",8), + Port = try_to_erl("\""++ToErl++"\" "++Tempdir++"/ 2>/dev/null", 8), case Port of P when is_port(P) -> Parent ! {self(),started}; @@ -827,7 +1152,7 @@ toerl_server(Parent,ToErl,Tempdir) -> Parent ! {self(),error,Other}, exit(Other) end, - case toerl_loop(Port,[]) of + case toerl_loop(#{ port => Port}) of normal -> ok; {error, Reason} -> @@ -836,63 +1161,59 @@ toerl_server(Parent,ToErl,Tempdir) -> exit(Reason) end. -toerl_loop(Port,Acc) -> - ?dbg({toerl_loop, Port, Acc}), +toerl_loop(#{ port := Port } = State0) -> + ?dbg({toerl_loop, Port, maps:get(acc,State0,[]), + maps:get(match,State0,nomatch)}), + + State = handle_match(State0), + receive - {Port,{data,{Tag0,Data}}} when is_port(Port) -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - case Acc of - [{noeol,Data0}|T0] -> - toerl_loop(Port,[{Tag0, Data0++Data}|T0]); - _ -> - toerl_loop(Port,[{Tag0,Data}|Acc]) - end; - {Pid,{get_line,Timeout}} -> - case Acc of - [] -> - case get_data_within(Port,Timeout,[]) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[]); - {noeol,Data1} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data1}]); - {eol,Data2} -> - Pid ! {get_line, Data2}, - toerl_loop(Port,[]) - end; - [{noeol,Data3}] -> - case get_data_within(Port,Timeout,Data3) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,Acc); - {noeol,Data4} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data4}]); - {eol,Data5} -> - Pid ! {get_line, Data5}, - toerl_loop(Port,[]) - end; - List -> - {NewAcc,[{eol,Data6}]} = lists:split(length(List)-1,List), - Pid ! {get_line,Data6}, - toerl_loop(Port,NewAcc) - end; + {Port,{data,Data}} when is_port(Port) -> + ?dbg({?LINE,Port,{data,Data}}), + toerl_loop(State#{ acc => lists:flatten([maps:get(acc,State,[]),Data])}); + {Pid, {get_data, Timeout, Match}} -> + toerl_loop( + State#{ get => + #{ match => Match, + timer => erlang:start_timer(Timeout, self(), timeout), + tag => get_data, + from => Pid } + }); + {Pid, {get_line, Timeout}} -> + toerl_loop( + State#{ get => + #{ match => "\r\n", + timer => erlang:start_timer(Timeout, self(), timeout), + tag => get_line, + from => Pid } + }); {Pid, {send_line, Data7}} -> Port ! {self(),{command, Data7++"\n"}}, Pid ! {send_line, ok}, - toerl_loop(Port,Acc); + toerl_loop(State); + {Pid, {send_data, Data}} -> + Port ! {self(),{command, Data}}, + Pid ! {send_data, ok}, + toerl_loop(State); {Pid, {kill_emulator_command, Cmd}} -> put(kill_emulator_command, Cmd), Pid ! {kill_emulator_command, ok}, - toerl_loop(Port,Acc); + toerl_loop(State); {_Pid, kill_emulator} -> case get(kill_emulator_command) of undefined -> Port ! {self(),{command, "init:stop().\n"}}; sigint -> - Port ! {self(),{command, [3]}}, - timer:sleep(200), + ?dbg({putdata,[$\^c]}), + Port ! {self(),{command, [$\^c]}}, + receive + {Port,{data,_Data}} -> + ?dbg({exit_data, _Data}), + ok + after 2000 -> + ok + end, + ?dbg({putdata,"a\n"}), Port ! {self(),{command, "a\n"}} end, Timeout1 = timeout(long), @@ -902,45 +1223,68 @@ toerl_loop(Port,Acc) -> after Timeout1 -> {error, kill_timeout} end; + {timeout,Timer,timeout} -> + #{ get := #{ tag := Tag, from := Pid, timer := Timer } } = State, + Pid ! {Tag, timeout}, + toerl_loop(maps:remove(get, State)); {Port, eof} -> {error, unexpected_eof}; Other -> {error, {unexpected, Other}} end. -millistamp() -> - erlang:monotonic_time(millisecond). - -get_data_within(Port, X, Acc) when X =< 0 -> - ?dbg({get_data_within, X, Acc, ?LINE}), - receive - {Port,{data,{Tag0,Data}}} -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - {Tag0, Acc++Data} - after 0 -> - case Acc of - [] -> - timeout; - Noeol -> - {noeol,Noeol} - end +handle_match(#{ acc := Acc, get := #{ tag := Tag, + match := Match, + from := From, + timer := Timer}} = State) -> + case string:split(Acc, Match) of + [Pre,Post] -> + ?dbg({match,Pre}), + From ! {Tag, Pre}, + erlang:cancel_timer(Timer), + receive + {timeout,Timer,timeout} -> + ok + after 0 -> + ok + end, + maps:put(acc, Post, maps:remove(get, State)); + [Acc] -> + State end; - - -get_data_within(Port, Timeout, Acc) -> - ?dbg({get_data_within, Timeout, Acc, ?LINE}), - T1 = millistamp(), - receive - {Port,{data,{noeol,Data}}} -> - ?dbg({?LINE,Port,{data,{noeol,Data}}}), - Elapsed = millistamp() - T1 + 1, - get_data_within(Port, Timeout - Elapsed, Acc ++ Data); - {Port,{data,{eol,Data1}}} -> - ?dbg({?LINE,Port,{data,{eol,Data1}}}), - {eol, Acc ++ Data1} - after Timeout -> - timeout - end. +handle_match(State) -> + State. + +rtnode_check_logs(Logname, Pattern, Logs) -> +rtnode_check_logs(Logname, Pattern, true, Logs). +rtnode_check_logs(Logname, Pattern, Match, Logs) -> + case re:run(maps:get(Logname, Logs), Pattern) of + {match, [_]} when Match -> + ok; + nomatch when not Match -> + ok; + _ -> + rtnode_dump_logs(Logs), + ct:fail("~p not found in log ~ts",[Pattern, Logname]) + end. + +rtnode_dump_logs(Logs) -> + maps:foreach( + fun(File, Data) -> + ct:pal("~ts: ~ts",[File, Data]) + end, Logs). + +rtnode_read_logs(Tempdir) -> + {ok, LogFiles} = file:list_dir(Tempdir), + lists:foldl( + fun(File, Acc) -> + case file:read_file(filename:join(Tempdir, File)) of + {ok, Data} -> + Acc#{ File => Data }; + _ -> + Acc + end + end, #{}, LogFiles). get_default_shell() -> try diff --git a/lib/kernel/test/kernel_test_global_sys_monitor.erl b/lib/kernel/test/kernel_test_global_sys_monitor.erl new file mode 100644 index 0000000000..43d4ab1334 --- /dev/null +++ b/lib/kernel/test/kernel_test_global_sys_monitor.erl @@ -0,0 +1,278 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(kernel_test_global_sys_monitor). + +-export([start/0, stop/0, + reset_events/0, + events/0, + log/1]). +-export([timestamp/0, format_timestamp/1]). +-export([init/1]). + +-include("kernel_test_lib.hrl"). + +-define(NAME, ?MODULE). +-define(TIMEOUT, timer:seconds(6)). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +start() -> + Parent = self(), + proc_lib:start(?MODULE, init, [Parent]). + +stop() -> + cast(stop). + +%% This does not reset the global counter but the "collector" +%% See events for more info. +reset_events() -> + call(reset_events, ?TIMEOUT). + +events() -> + call(events, ?TIMEOUT). + +log(Event) -> + cast({node(), Event}). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +timestamp() -> + erlang:timestamp(). + +format_timestamp({_N1, _N2, N3} = TS) -> + {Date, Time} = calendar:now_to_local_time(TS), + {YYYY,MM,DD} = Date, + {Hour,Min,Sec} = Time, + FormatDate = + io_lib:format("~.4w-~.2.0w-~.2.0w ~.2.0w:~.2.0w:~.2.0w.~.3.0w", + [YYYY, MM, DD, Hour, Min, Sec, N3 div 1000]), + lists:flatten(FormatDate). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +init(Parent) -> + process_flag(priority, high), + case global:register_name(?NAME, self()) of + yes -> + info_msg("Starting as ~p (on ~p)", [self(), node()]), + proc_lib:init_ack(Parent, {ok, self()}), + loop(#{parent => Parent, ev_cnt => 0, evs => []}); + no -> + warning_msg("Already started", []), + proc_lib:init_ack(Parent, {error, already_started}), + exit(normal) + end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +loop(State) -> + receive + {?MODULE, stop} -> + warning_msg("Stopping with ~w events counted", + [maps:get(ev_cnt, State)]), + exit(normal); + + {?MODULE, Ref, From, reset_events} -> + TotEvCnt = maps:get(ev_cnt, State), + EvCnt = length(maps:get(evs, State)), + info_msg("Reset events when" + "~n Total Number of Events: ~p" + "~n Current Number of Events: ~p", + [TotEvCnt, EvCnt]), + From ! {?MODULE, Ref, {ok, {TotEvCnt, EvCnt}}}, + loop(State#{evs => []}); + + {?MODULE, Ref, From, events} -> + Evs = maps:get(evs, State), + From ! {?MODULE, Ref, lists:reverse(Evs)}, + loop(State); + + {?MODULE, {Node, Event}} -> + State2 = process_event(State, Node, Event), + loop(State2); + + {nodedown = Event, Node} -> + State2 = process_event(State, Node, Event), + loop(State2); + + _ -> + loop(State) + end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +process_event(State, Node, {Pid, TS, Tag, Info}) -> + process_system_event(State, Node, Pid, TS, Tag, Info); + +process_event(State, Node, {TS, starting}) -> + FTS = format_timestamp(TS), + info_msg("System Monitor starting on node ~p at ~s", [Node, FTS]), + if + (Node =/= node()) -> + erlang:monitor_node(Node, true); + true -> + ok + end, + State; + +process_event(State, Node, {TS, stopping}) -> + FTS = format_timestamp(TS), + info_msg("System Monitor stopping on node ~p at ~s", [Node, FTS]), + if + (Node =/= node()) -> + erlang:monitor_node(Node, false); + true -> + ok + end, + State; + +process_event(State, Node, {TS, already_started}) -> + FTS = format_timestamp(TS), + info_msg("System Monitor already started on node ~p at ~s", [Node, FTS]), + State; + +process_event(State, Node, nodedown) -> + info_msg("Node ~p down", [Node]), + State; + +process_event(State, Node, Event) -> + warning_msg("Received unknown event from node ~p:" + "~n ~p", [Node, Event]), + State. + + +%% System Monitor events +%% We only *count* system events +process_system_event(#{ev_cnt := Cnt, evs := Evs} = State, + Node, Pid, TS, long_gc = Ev, Info) -> + print_system_event(f("Long GC (~w)", [length(Evs)]), Node, Pid, TS, Info), + State#{ev_cnt => Cnt + 1, evs => [{Node, Ev} | Evs]}; +process_system_event(#{ev_cnt := Cnt, evs := Evs} = State, + Node, Pid, TS, long_schedule = Ev, Info) -> + print_system_event(f("Long Schedule (~w)", [length(Evs)]), Node, Pid, TS, Info), + State#{ev_cnt => Cnt + 1, evs => [{Node, Ev} | Evs]}; +process_system_event(#{ev_cnt := Cnt, evs := Evs} = State, + Node, Pid, TS, large_heap = Ev, Info) -> + print_system_event(f("Large Heap (~w)", [length(Evs)]), Node, Pid, TS, Info), + State#{ev_cnt => Cnt + 1, evs => [{Node, Ev} | Evs]}; +process_system_event(#{ev_cnt := Cnt, evs := Evs} = State, + Node, Pid, TS, busy_port = Ev, Info) -> + print_system_event(f("Busy port (~w)", [length(Evs)]), Node, Pid, TS, Info), + State#{ev_cnt => Cnt + 1, evs => [{Node, Ev} | Evs]}; +process_system_event(#{ev_cnt := Cnt, evs := Evs} = State, + Node, Pid, TS, busy_dist_port = Ev, Info) -> + print_system_event(f("Busy dist port (~w)", [length(Evs)]), + Node, Pid, TS, Info), + State#{ev_cnt => Cnt + 1, evs => [{Node, Ev} | Evs]}; + +%% And everything else +process_system_event(State, Node, Pid, TS, Tag, Info) -> + Pre = f("Unknown Event '~p'", [Tag]), + print_system_event(Pre, Node, Pid, TS, Info), + State. + + +print_system_event(Pre, Node, Pid, TS, Info) -> + FTS = format_timestamp(TS), + warning_msg("~s from ~p (~p) at ~s:" + "~n ~p", [Pre, Node, Pid, FTS, Info]). + +f(F, A) -> + lists:flatten(io_lib:format(F, A)). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +cast(Msg) -> + try global:send(?NAME, {?MODULE, Msg}) of + Pid when is_pid(Pid) -> + ok + catch + C:E:_ -> + {error, {catched, C, E}} + end. + + +call(Req, Timeout) when (Timeout =:= infinity) -> + call(Req, Timeout, Timeout); +call(Req, Timeout) when is_integer(Timeout) andalso (Timeout > 2000) -> + call(Req, Timeout, Timeout - 1000); +call(Req, Timeout) when is_integer(Timeout) andalso (Timeout > 1000) -> + call(Req, Timeout, Timeout - 500); +call(Req, Timeout) when is_integer(Timeout) -> + call(Req, Timeout, Timeout div 2). + +%% This peace of wierdness is because on some machines this call has +%% hung (in a call during end_per_testcase, which had a 1 min timeout, +%% or if that was the total time for the test case). +%% But because it hung there, we don't really know what where it git stuck. +%% So, by making the call in a tmp process, that we supervise, we can +%% keep control. Also, we change the default timeout from infinity to an +%% actual time (16 seconds). +call(Req, Timeout1, Timeout2) -> + F = fun() -> + Ref = make_ref(), + try global:send(?NAME, {?MODULE, Ref, self(), Req}) of + NamePid when is_pid(NamePid) -> + receive + {?MODULE, Ref, Rep} -> + Rep + after Timeout2 -> + {error, timeout} + end + catch + C:E:_ -> + {error, {catched, C, E}} + end + end, + {Pid, Mon} = spawn_monitor(F), + receive + {'DOWN', Mon, process, Pid, Result} -> + Result + after Timeout1 -> + PInfo = process_info(Pid), + exit(Pid, kill), + {error, {timeout, PInfo}} + end. + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +info_msg(F, A) -> + error_logger:info_msg(format_msg(F, A), []). + +warning_msg(F, A) -> + error_logger:warning_msg(format_msg(F, A), []). + + +format_msg(F, A) -> + f("~n" ++ + "****** KERNEL TEST GLOBAL SYSTEM MONITOR ******~n~n" ++ + F ++ + "~n~n", + A). + diff --git a/lib/kernel/test/kernel_test_lib.erl b/lib/kernel/test/kernel_test_lib.erl index c4de916b1a..268b97ca3f 100644 --- a/lib/kernel/test/kernel_test_lib.erl +++ b/lib/kernel/test/kernel_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2020-2020. All Rights Reserved. +%% Copyright Ericsson AB 2020-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,9 +25,13 @@ -export([tc_try/3]). -export([listen/3, connect/4, connect/5, + is_socket_backend/1, inet_backend_opts/1, explicit_inet_backend/0, test_inet_backends/0]). +-export([start_slave_node/2, start_slave_node/3, + start_node/3, start_node/4, + stop_node/1]). -export([f/2, print/1, print/2]). -export([good_hosts/1, @@ -54,6 +58,7 @@ init_per_suite(AllowSkip, Config) when is_boolean(AllowSkip) -> true -> {skip, "Unstable host and/or os (or combo thererof)"}; false -> + kernel_test_global_sys_monitor:start(), [{kernel_factor, Factor} | Config] catch throw:{skip, _} = SKIP -> @@ -71,6 +76,7 @@ init_per_suite(AllowSkip, Config) when is_boolean(AllowSkip) -> end_per_suite(Config) when is_list(Config) -> + kernel_test_global_sys_monitor:stop(), Config. analyze_and_print_host_info() -> @@ -1663,6 +1669,36 @@ tc_which_name() -> end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +start_slave_node(Name, Args) -> + start_slave_node(Name, Args, []). + +start_slave_node(Name, Args, Opts) -> + start_node(Name, slave, Args, Opts). + + +start_node(Name, Type, Args) -> + start_node(Name, Type, Args, []). + +start_node(Name, Type, Args, Opts) -> + Pa = filename:dirname(code:which(?MODULE)), + A = Args ++ + " -pa " ++ Pa ++ + " -s " ++ atom_to_list(kernel_test_sys_monitor) ++ " start" ++ + " -s global sync", + case test_server:start_node(Name, Type, [{args, A}|Opts]) of + {ok, _Node} = OK -> + global:sync(), + OK; + ERROR -> + ERROR + end. + +stop_node(Node) -> + test_server:stop_node(Node). + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% timetrap_scale_factor() -> @@ -1711,6 +1747,14 @@ inet_backend_opts(Config) when is_list(Config) -> [] end. +is_socket_backend(Config) when is_list(Config) -> + case lists:keysearch(socket_create_opts, 1, Config) of + {value, {socket_create_opts, [{inet_backend, socket}]}} -> + true; + _ -> + false + end. + explicit_inet_backend() -> case application:get_all_env(kernel) of diff --git a/lib/kernel/test/kernel_test_lib.hrl b/lib/kernel/test/kernel_test_lib.hrl index ec0ca05573..d49c31f4d6 100644 --- a/lib/kernel/test/kernel_test_lib.hrl +++ b/lib/kernel/test/kernel_test_lib.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2020-2020. All Rights Reserved. +%% Copyright Ericsson AB 2020-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,7 +41,15 @@ -define(INET_BACKEND_OPTS(C), ?LIB:inet_backend_opts(C)). -define(EXPLICIT_INET_BACKEND(), ?LIB:explicit_inet_backend()). -define(TEST_INET_BACKENDS(), ?LIB:test_inet_backends()). +-define(IS_SOCKET_BACKEND(C), ?LIB:is_socket_backend(C)). +-define(START_SLAVE_NODE(__N__, __A__), + ?LIB:start_slave_node(__N__, __A__)). +-define(START_SLAVE_NODE(__N__, __A__, __O__), + ?LIB:start_slave_node(__N__, __A__, __O__)). + +-define(STOP_NODE(__N__), ?LIB:stop_node(__N__)). + -define(F(FORMAT, ARGS), ?LIB:f(FORMAT, ARGS)). -define(P(F), ?LIB:print(F)). -define(P(F,A), ?LIB:print(F, A)). diff --git a/lib/kernel/test/kernel_test_sys_monitor.erl b/lib/kernel/test/kernel_test_sys_monitor.erl new file mode 100644 index 0000000000..f6cc277358 --- /dev/null +++ b/lib/kernel/test/kernel_test_sys_monitor.erl @@ -0,0 +1,94 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(kernel_test_sys_monitor). + +-export([start/0, stop/0, + init/1]). + +-define(NAME, ?MODULE). +-define(GSM, kernel_test_global_sys_monitor). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +start() -> + Parent = self(), + proc_lib:start(?MODULE, init, [Parent]). + +stop() -> + case whereis(?NAME) of + Pid when is_pid(Pid) -> + Pid ! {?MODULE, self(), stop}, + receive + {?MODULE, Pid, stop} -> + ok + end; + _ -> + ok + end. + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +init(Parent) -> + process_flag(priority, high), + try register(?NAME, self()) of + true -> + global:sync(), + MonSettings = [ + busy_port, + busy_dist_port, + {long_gc, 1000}, + {long_schedule, 1000}, + {large_heap, 8*1024*1024} % 8 MB + ], + erlang:system_monitor(self(), MonSettings), + ?GSM:log({?GSM:timestamp(), starting}), + proc_lib:init_ack(Parent, {ok, self()}), + loop(#{parent => Parent}) + catch + _:_:_ -> + ?GSM:log({?GSM:timestamp(), already_started}), + proc_lib:init_ack(Parent, {error, already_started}), + exit(normal) + end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +loop(State) -> + receive + {monitor, Pid, Tag, Info} -> + ?GSM:log({Pid, ?GSM:timestamp(), Tag, Info}), + loop(State); + + {?MODULE, From, stop} -> + ?GSM:log({?GSM:timestamp(), stopping}), + From ! {?MODULE, self(), stop}, + exit(normal); + + _ -> + loop(State) + end. + + + diff --git a/lib/kernel/test/os_SUITE_data/my_echo.c b/lib/kernel/test/os_SUITE_data/my_echo.c index 712c828bb5..439f812af1 100644 --- a/lib/kernel/test/os_SUITE_data/my_echo.c +++ b/lib/kernel/test/os_SUITE_data/my_echo.c @@ -1,3 +1,4 @@ +#include <stdio.h> #ifdef __WIN32__ #include <windows.h> @@ -25,8 +26,6 @@ int wmain(int argc, wchar_t **argv) } #else -#include <stdio.h> - int main(int argc, char** argv) { diff --git a/lib/kernel/test/pg_SUITE.erl b/lib/kernel/test/pg_SUITE.erl index f03b8a6a39..3a69e33bcf 100644 --- a/lib/kernel/test/pg_SUITE.erl +++ b/lib/kernel/test/pg_SUITE.erl @@ -295,6 +295,16 @@ empty_group_by_remote_leave(Config) when is_list(Config) -> % empty group should be deleted. ?assertEqual(#{}, NewRemoteMap), + %% another variant of emptying a group remotely: join([Pi1, Pid2]) and leave ([Pid2, Pid1]) + RemotePid2 = erlang:spawn(TwoPeer, forever()), + ?assertEqual(ok, rpc:call(TwoPeer, pg, join, [?FUNCTION_NAME, ?FUNCTION_NAME, [RemotePid, RemotePid2]])), + sync({?FUNCTION_NAME, TwoPeer}), + ?assertEqual([RemotePid, RemotePid2], pg:get_members(?FUNCTION_NAME, ?FUNCTION_NAME)), + %% now leave + ?assertEqual(ok, rpc:call(TwoPeer, pg, leave, [?FUNCTION_NAME, ?FUNCTION_NAME, [RemotePid2, RemotePid]])), + sync({?FUNCTION_NAME, TwoPeer}), + ?assertEqual([], pg:get_members(?FUNCTION_NAME, ?FUNCTION_NAME)), + {state, _, _, #{RemoteNode := {_, NewRemoteMap}}} = sys:get_state(?FUNCTION_NAME), stop_node(TwoPeer, Socket), ok. diff --git a/lib/kernel/test/socket_SUITE.erl b/lib/kernel/test/socket_SUITE.erl index 139a886aa5..2ec9227a84 100644 --- a/lib/kernel/test/socket_SUITE.erl +++ b/lib/kernel/test/socket_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2018-2020. All Rights Reserved. +%% Copyright Ericsson AB 2018-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19696,7 +19696,10 @@ api_opt_ip_recvtos_udp4(doc) -> api_opt_ip_recvtos_udp4(_Config) when is_list(_Config) -> ?TT(?SECS(5)), tc_try(api_opt_ip_recvtos_udp4, - fun() -> has_support_ip_recvtos() end, + fun() -> + has_support_ip_recvtos(), + has_support_ip_tos() % Used in the test + end, fun() -> Set = fun(Sock, Value) -> socket:setopt(Sock, ip, recvtos, Value) @@ -19706,15 +19709,15 @@ api_opt_ip_recvtos_udp4(_Config) when is_list(_Config) -> end, Send = fun(Sock, Data, Dest, default) -> Msg = #{addr => Dest, - iov => [Data]}, + iov => [Data]}, socket:sendmsg(Sock, Msg); (Sock, Data, Dest, TOS) -> CMsg = #{level => ip, - type => tos, - data => TOS}, + type => tos, + data => TOS}, Msg = #{addr => Dest, - ctrl => [CMsg], - iov => [Data]}, + ctrl => [CMsg], + iov => [Data]}, socket:sendmsg(Sock, Msg) end, Recv = fun(Sock) -> @@ -19856,23 +19859,23 @@ api_opt_ip_recvtos_udp(InitState) -> end end}, - #{desc => "set tos = mincost on src sock", + #{desc => "set tos = reliability on src sock", cmd => fun(#{sock_src := Sock}) -> - ok = socket:setopt(Sock, ip, tos, mincost) + ok = socket:setopt(Sock, ip, tos, reliability) end}, - #{desc => "send req (to dst) (w tos = mincost)", + #{desc => "send req (to dst) (w tos = reliability)", cmd => fun(#{sock_src := Sock, sa_dst := Dst, send := Send}) -> Send(Sock, ?BASIC_REQ, Dst, default) end}, - %% #{desc => "send req (to dst) (w explicit tos = mincost)", + %% #{desc => "send req (to dst) (w explicit tos = reliability)", %% cmd => fun(#{sock_src := Sock, %% sa_dst := Dst, %% send := Send}) -> %% socket:setopt(Sock, otp, debug, true), - %% case Send(Sock, ?BASIC_REQ, Dst, mincost) of + %% case Send(Sock, ?BASIC_REQ, Dst, reliability) of %% ok -> %% socket:setopt(Sock, otp, debug, false), %% ok; @@ -19975,21 +19978,21 @@ api_opt_ip_recvtos_udp(InitState) -> end end}, - #{desc => "set tos = mincost on src sock", + #{desc => "set tos = reliability on src sock", cmd => fun(#{sock_src := Sock}) -> - ok = socket:setopt(Sock, ip, tos, mincost) + ok = socket:setopt(Sock, ip, tos, reliability) end}, #{desc => "send req (to dst) (w tos = mincost)", cmd => fun(#{sock_src := Sock, sa_dst := Dst, send := Send}) -> Send(Sock, ?BASIC_REQ, Dst, default) end}, - #{desc => "recv req (from src) - w tos = mincost", + #{desc => "recv req (from src) - w tos = reliability", cmd => fun(#{sock_dst := Sock, sa_src := Src, recv := Recv}) -> case Recv(Sock) of {ok, {Src, [#{level := ip, type := TOS, - value := mincost = TOSData}], + value := reliability = TOSData}], ?BASIC_REQ}} when ((TOS =:= tos) orelse (TOS =:= recvtos)) -> ?SEV_IPRINT("got expected TOS (~w) = ~w " diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4a226539a3..c04299ae88 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 7.2.1 +KERNEL_VSN = 7.3 diff --git a/lib/megaco/test/megaco_config_SUITE.erl b/lib/megaco/test/megaco_config_SUITE.erl index 557406f8c4..79035b3b93 100644 --- a/lib/megaco/test/megaco_config_SUITE.erl +++ b/lib/megaco/test/megaco_config_SUITE.erl @@ -657,49 +657,86 @@ start_counter_working_procs([Pid | Pids]) -> start_counter_working_procs(Pids). await_completion_counter_working_procs(Pids) -> - await_completion_counter_working_procs(Pids, [], []). + await_completion_counter_working_procs(ts(), Pids, [], []). -await_completion_counter_working_procs([], _OKs, [] = _ERRs) -> +await_completion_counter_working_procs(_TS, [], _OKs, [] = _ERRs) -> ok; -await_completion_counter_working_procs([], _OKs, ERRs) -> +await_completion_counter_working_procs(_TS, [], _OKs, ERRs) -> {error, ERRs}; -await_completion_counter_working_procs(Pids, OKs, ERRs) -> +await_completion_counter_working_procs(TS, Pids, OKs, ERRs) -> receive {'EXIT', Pid, normal} -> %% i("counter working process completion[~w, ~w, ~w] -> " %% "Expected exit from counter process: " + %% "~n Time since last event: ~s" %% "~n Pid: ~p", - %% [length(Pids), length(OKs), length(ERRs), Pid]), + %% [length(Pids), length(OKs), length(ERRs), tsd(TS), Pid]), Pids2 = lists:delete(Pid, Pids), - await_completion_counter_working_procs(Pids2, [Pid | OKs], ERRs); + await_completion_counter_working_procs(ts(), + Pids2, [Pid | OKs], ERRs); + + {'EXIT', Pid, {timetrap_timeout, _Timeout, Stack} = _Reason} -> + e("counter working process completion[~w, ~w, ~w] -> " + "Unexpected exit from counter process: test case timeout" + "~n Time since last event: ~s" + "~n Pid: ~p" + "~n Stack: ~p", + [length(Pids), length(OKs), length(ERRs), tsd(TS), Pid, Stack]), + %% The test case (timetrap) has timed out, which either means + %% we are running on very slow hw or some system functions + %% are slowing us down (this test case should never normally + %% time out). + case megaco_test_global_sys_monitor:events() of + [] -> + i("counter working process completion[~w, ~w, ~w] -> idle", + [length(Pids), length(OKs), length(ERRs)]), + ?SKIP("TC idle"); + SysEvs -> + e("counter working process completion[~w, ~w, ~w] -> " + "system event(s): " + "~n ~p", + [length(Pids), length(OKs), length(ERRs), SysEvs]), + ?SKIP("TC system events") + end; + {'EXIT', Pid, Reason} -> + TSD = tsd(TS), e("counter working process completion[~w, ~w, ~w] -> " "Unexpected exit from counter process: " - "~n Pid: ~p" - "~n Reason: ~p", - [length(Pids), length(OKs), length(ERRs), Pid, Reason]), + "~n Time since last event: ~s" + "~n Pid: ~p" + "~n Reason: ~p", + [length(Pids), length(OKs), length(ERRs), TSD, Pid, Reason]), Pids2 = lists:delete(Pid, Pids), - await_completion_counter_working_procs(Pids2, OKs, [Pid | ERRs]); + await_completion_counter_working_procs(ts(), + Pids2, OKs, [Pid | ERRs]); Any -> + TSD = tsd(TS), e("counter working process completion[~w, ~w, ~w] -> " "Unexpected message: " - "~n ~p", [length(Pids), length(OKs), length(ERRs), Any]), - await_completion_counter_working_procs(Pids) + "~n Time since last event: ~s" + "~n ~p", + [length(Pids), length(OKs), length(ERRs), TSD, Any]), + await_completion_counter_working_procs(TS, Pids, OKs, ERRs) after 10000 -> %% If nothing has happened for this long, something is wrong: %% Check system events + TSD = tsd(TS), case megaco_test_global_sys_monitor:events() of [] -> - i("counter working process completion[~w, ~w, ~w] -> " - "idle", [length(Pids), length(OKs), length(ERRs)]), - await_completion_counter_working_procs(Pids); + i("counter working process completion[~w, ~w, ~w] -> idle" + "~n Time since last event: ~s", + [length(Pids), length(OKs), length(ERRs), TSD]), + await_completion_counter_working_procs(TS, + Pids, OKs, ERRs); SysEvs -> e("counter working process completion[~w, ~w, ~w] -> " "system event(s): " + "~n Time since last event: ~s" "~n ~p", - [length(Pids), length(OKs), length(ERRs), SysEvs]), + [length(Pids), length(OKs), length(ERRs), TSD, SysEvs]), ?SKIP("TC idle with system events") end end. @@ -864,17 +901,28 @@ create_counter_working_procs2([#conn_data{conn_handle = CH} | CDs], create_counter_working_procs2(CDs, NumCntProcs)]. -verify_counter_results([]) -> +verify_counter_results(CDs) -> + verify_counter_results(CDs, [], []). + +verify_counter_results([], _OKs, [] = _Errors) -> + i("counter verification success"), ok; -verify_counter_results([#conn_data{conn_handle = CH} | CDs]) -> +verify_counter_results([], OKs, Errors) -> + e("counter verification failed: " + "~n Num OKs: ~p" + "~n Num Errors: ~p", [length(OKs), length(Errors)]), + error; +verify_counter_results([#conn_data{conn_handle = CH} = CD| CDs], OKs, Errors) -> TransId = megaco_config:conn_info(CH, trans_id), if (TransId =:= 1) -> - ok; + verify_counter_results(CDs, [CD|OKs], Errors); true -> - ?ERROR({trans_id_verification_failed, CH, TransId}) - end, - verify_counter_results(CDs). + e("invalid transaction is for connection: " + "~n CD: ~p" + "~n TransId: ~p", [CD, TransId]), + verify_counter_results(CDs, OKs, [{CD, TransId}|Errors]) + end. delete_connections([]) -> @@ -1268,6 +1316,22 @@ try_tc(TCName, Name, Verbosity, Pre, Case, Post) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +ts() -> + erlang:monotonic_time(milli_seconds). + +tsd(TS) -> + TSD = ts() - TS, + if (TSD < 1000) -> + ?F("~w ms", [TSD]); + (TSD < 60000) -> + ?F("~w secs", [TSD div 1000]); + true -> + ?F("~w mins", [TSD div 60000]) + end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + p(F) -> p(F, []). diff --git a/lib/megaco/test/megaco_load_SUITE.erl b/lib/megaco/test/megaco_load_SUITE.erl index 876e16ecfb..d033e137ec 100644 --- a/lib/megaco/test/megaco_load_SUITE.erl +++ b/lib/megaco/test/megaco_load_SUITE.erl @@ -60,8 +60,8 @@ -define(SINGLE_USER_LOAD_NUM_REQUESTS, 1000). -define(MULTI_USER_LOAD_NUM_REQUESTS, 1000). --define(MGC_START(Pid, Mid, ET, Conf, Verb), - megaco_test_mgc:start(Pid, Mid, ET, +-define(MGC_START(Node, Mid, ET, Conf, Verb), + megaco_test_mgc:start(Node, Mid, ET, [{megaco_trace, false}] ++ Conf, Verb)). -define(MGC_STOP(Pid), megaco_test_mgc:stop(Pid)). -define(MGC_USER_INFO(Pid,Tag), megaco_test_mgc:user_info(Pid,Tag)). diff --git a/lib/megaco/test/megaco_test_mg.erl b/lib/megaco/test/megaco_test_mg.erl index 5979466785..e84adc7de6 100644 --- a/lib/megaco/test/megaco_test_mg.erl +++ b/lib/megaco/test/megaco_test_mg.erl @@ -110,32 +110,28 @@ start(Node, Mid, Encoding, Transport, Conf, Verbosity) -> case (catch mg(Self, Verbosity, Config)) of {'EXIT', Reason} -> e("LOADER(~p,~p) terminating with exit" - "~n ~p", [self(), node(), Reason]), + "~n ~p", [Self, node(), Reason]), exit(Reason); Else -> i("LOADER(~p,~p) terminating with" - "~n ~p", [self(), node(), Else]), + "~n ~p", [Self, node(), Else]), Else end end, - true = erlang:monitor_node(Node, true), - Pid = spawn_link(Node, Fun), - %% Pid = spawn_link(Node, ?MODULE, mg, [self(), Verbosity, Config]), - MonRef = (catch erlang:monitor(process, Pid)), - NodePing = net_adm:ping(Node), - ProcInfo = (catch proc_info(Pid)), - i("start -> " - "~n self(): ~p" - "~n node(): ~p" - "~n net_adm:ping(~p): ~p" - "~n Loader: ~p" - "~n Monitor ref: ~p" - "~n Process info: ~p", - [self(), node(), - Node, NodePing, - Pid, - MonRef, ProcInfo]), - await_started(Node, MonRef, Pid). + true = erlang:monitor_node(Node, true), + {Pid, MRef} = spawn_monitor(Node, Fun), + NodePing = net_adm:ping(Node), + ProcInfo = (catch proc_info(Pid)), + i("start mg[~p] -> ~p" + "~n self(): ~p" + "~n node(): ~p" + "~n Node ping: ~p" + "~n Loader: ~p" + "~n Monitor ref: ~p" + "~n Process info: ~p", + [Node, Pid, + Self, node(), NodePing, Pid, MRef, ProcInfo]), + await_started(Node, MRef, Pid). proc_info(Pid) -> rpc:call(node(Pid), erlang, process_info, [Pid]). diff --git a/lib/megaco/test/megaco_test_mgc.erl b/lib/megaco/test/megaco_test_mgc.erl index 1204dbba07..a7d0731b0f 100644 --- a/lib/megaco/test/megaco_test_mgc.erl +++ b/lib/megaco/test/megaco_test_mgc.erl @@ -93,10 +93,27 @@ start(Node, Mid, ET, Conf, Verbosity) -> d("start mgc[~p]: ~p" "~n ET: ~p" "~n Conf: ~p", [Node, Mid, ET, Conf]), - RI = {receive_info, mk_recv_info(ET)}, - Config = [{local_mid, Mid}, RI] ++ Conf, - Pid = spawn_link(Node, ?MODULE, mgc, [self(), Verbosity, Config]), - await_started(Pid). + RI = {receive_info, mk_recv_info(ET)}, + Config = [{local_mid, Mid}, RI] ++ Conf, + Self = self(), + true = erlang:monitor_node(Node, true), + MGC = fun() -> mgc(Self, Verbosity, Config) end, + {Pid, MRef} = spawn_monitor(Node, MGC), + NodePing = net_adm:ping(Node), + ProcInfo = (catch proc_info(Pid)), + i("start mgc[~p] -> ~p" + "~n self(): ~p" + "~n node(): ~p" + "~n Node ping: ~p" + "~n Loader: ~p" + "~n Monitor ref: ~p" + "~n Process info: ~p", + [Node, Pid, + Self, node(), NodePing, Pid, MRef, ProcInfo]), + await_started(Node, Pid, MRef). + +proc_info(Pid) -> + rpc:call(node(Pid), erlang, process_info, [Pid]). mk_recv_info(ET) -> mk_recv_info(ET, []). @@ -155,20 +172,40 @@ select_transport(Transport) -> throw({error, {invalid_transport, Transport}}). -await_started(Pid) -> +await_started(Node, Pid, MRef) -> receive {started, Pid} -> - d("await_started ~p: ok", [Pid]), + i("await_started ~p: ok", [Pid]), + true = erlang:monitor_node(Node, false), + erlang:demonitor(MRef), {ok, Pid}; - {'EXIT', Pid, - {failed_starting_tcp_listen, {could_not_start_listener, {gen_tcp_listen, eaddrinuse}}}} -> + + {nodedown, Node} -> + i("await_started ~p - received node down", [Pid]), + exit({node_down, Node}); + + {'DOWN', MRef, process, Pid, + {failed_starting_tcp_listen, + {could_not_start_listener, {gen_tcp_listen, eaddrinuse}}}} -> e("await_started ~p: address already in use", [Pid]), + true = erlang:monitor_node(Node, false), ?SKIP(eaddrinuse); - {'EXIT', Pid, Reason} -> + + {'DOWN', MRef, process, Pid, Reason} -> e("await_started ~p: received exit signal: ~p", [Pid, Reason]), + true = erlang:monitor_node(Node, false), exit({failed_starting, Pid, Reason}) + after 10000 -> - e("await_started ~p: timeout", [Pid]), + NodePing = net_adm:ping(Node), + ProcInfo = (catch proc_info(Pid)), + FlushQ = megaco_test_lib:flush(), + e("await_started ~p - timeout: " + "~n net_adm:ping(~p): ~p" + "~n Process info: ~p" + "~n Messages in my queue: ~p", + [Pid, Node, NodePing, ProcInfo, FlushQ]), + true = erlang:monitor_node(Node, false), exit({error, timeout}) end. diff --git a/lib/megaco/test/megaco_trans_SUITE.erl b/lib/megaco/test/megaco_trans_SUITE.erl index 78ba1f0515..d37d5a20d6 100644 --- a/lib/megaco/test/megaco_trans_SUITE.erl +++ b/lib/megaco/test/megaco_trans_SUITE.erl @@ -9526,15 +9526,20 @@ cre_serviceChangeProf(Name, Ver) when is_list(Name) andalso is_integer(Ver) -> await_ack(_User, 0, Timeout, _Expected) -> d("await_ack -> done when Timeout = ~p", [Timeout]), ok; -await_ack(User, N, Timeout, Expected) when (N > 0) andalso is_integer(Timeout) -> - d("await_ack -> entry with N: ~p, Timeout: ~p", [N,Timeout]), +await_ack(User, N, Timeout, Expected) + when (N > 0) andalso is_integer(Timeout) -> + d("await_ack -> entry with N: ~p, Timeout: ~p", [N, Timeout]), T = tim(), receive {ack_received, User, Expected} -> d("await_ack -> received another expected ack"), await_ack(User, N-1, Timeout - (tim() - T), Expected); {ack_received, User, UnExpected} -> - e("await_ack -> received unexpected ack result: ~p", [UnExpected]), + e("await_ack -> received unexpected ack result: ~p" + "~n when" + "~n N: ~p" + "~n Remaining: ~p", + [UnExpected, N, Timeout - (tim() - T)]), exit({unexpected_ack_result, UnExpected, Expected}) after Timeout -> exit({await_ack_timeout, N}) @@ -9546,7 +9551,9 @@ await_ack(User, N, infinity, Expected) when N > 0 -> d("await_ack -> received another ack"), await_ack(User, N-1, infinity, Expected); {ack_received, User, UnExpected} -> - e("await_ack -> unexpected ack result: ~p", [UnExpected]), + e("await_ack -> unexpected ack result: ~p" + "~n when" + "~n N: ~p", [UnExpected, N]), exit({unexpected_ack_result, UnExpected, Expected}) end. diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index c8c7c0c0a5..ba61efa7e9 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -39,7 +39,42 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.18.1</title> + <section><title>Mnesia 4.19</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed the type spec for <c>disc_only_copies</c>.</p> + <p> + Own Id: OTP-17249 Aux Id: PR-4578 </p> + </item> + <item> + <p> + Do not crash in <c>mnesia:change_config/2</c> if mnesia + is stopping or starting.</p> + <p> + Own Id: OTP-17274 Aux Id: GH-4616 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Optimized table loading time for tables that are updated + during the loading.</p> + <p> + Own Id: OTP-17271 Aux Id: PR-4575 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.18.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 55cb73e1ca..6b34634e61 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -461,8 +461,6 @@ connect_nodes(Ns) -> connect_nodes(Ns, UserFun) -> case mnesia:system_info(is_running) of - no -> - {error, {node_not_running, node()}}; yes -> Pid = spawn_link(?MODULE,connect_nodes2,[self(),Ns, UserFun]), receive @@ -479,7 +477,9 @@ connect_nodes(Ns, UserFun) -> end; {'EXIT', Pid, Reason} -> {error, Reason} - end + end; + _ -> %% no, starting or stopping not ready to make a connection yet + {error, {node_not_running, node()}} end. connect_nodes2(Father, Ns, UserFun) -> diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index c0da0674f0..e98a250aa2 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -1385,7 +1385,7 @@ do_add_backend_type(Name, Module) -> ModuleRegistered = lists:keymember(Module, 2, Types), do_write_table_property(schema, {mnesia_backend_types, [{Name, Module}|Types]}), - ModuleRegistered. + not ModuleRegistered. delete_backend_type(Name) -> schema_transaction(fun() -> do_delete_backend_type(Name) end). diff --git a/lib/mnesia/test/ext_test.erl b/lib/mnesia/test/ext_test.erl index ad32245a11..cd39f4a6f1 100644 --- a/lib/mnesia/test/ext_test.erl +++ b/lib/mnesia/test/ext_test.erl @@ -70,10 +70,26 @@ semantics(_Alias, _) -> init_backend() -> ?DBG(init_backend), + ct:log("init_backend ~p", [?MODULE]), + %% cheat and stuff a marker in mnesia_gvar + K = backend_init_marker(), + case try mnesia_lib:val(K) catch _:_ -> error end of + error -> + ct:log("BACKEND marker will be inserted (~p)", [?MODULE]), + mnesia_lib:set(K, true); + Other -> + ct:log("BACKEND marker already present (~p)", [?MODULE]), + error({backend_already_initialized, {?MODULE, Other}}) + end, ok. +backend_init_marker() -> + {test, ?MODULE, backend_init}. + add_aliases(_As) -> ?DBG(_As), + ct:log("add_aliases(~p)", [_As]), + true = mnesia_lib:val(backend_init_marker()), ok. remove_aliases(_) -> diff --git a/lib/mnesia/test/mnesia_config_test.erl b/lib/mnesia/test/mnesia_config_test.erl index 45da909264..08fa9ea613 100644 --- a/lib/mnesia/test/mnesia_config_test.erl +++ b/lib/mnesia/test/mnesia_config_test.erl @@ -38,6 +38,7 @@ dump_log_update_in_place/1, event_module/1, + backend_plugin_registration/1, inconsistent_database/1, max_wait_for_decision/1, send_compressed/1, @@ -104,7 +105,7 @@ all() -> [access_module, auto_repair, backup_module, debug, dir, dump_log_load_regulation, {group, dump_log_thresholds}, dump_log_update_in_place, - event_module, + event_module, backend_plugin_registration, inconsistent_database, max_wait_for_decision, send_compressed, app_test, {group, schema_config}, unknown_config]. @@ -721,6 +722,21 @@ event_module(Config) when is_list(Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +backend_plugin_registration(doc) -> + ["Ensure that backend plugins can be registered at runtime,", + "that the init_backend is called exactly once when registering", + "a plugin, and not again when registering a new alias"]; +backend_plugin_registration(Config) when is_list(Config) -> + Nodes = ?acquire_schema(1, [{default_properties, []} | Config]), + ?match(ok, mnesia:start()), + ?match({atomic,ok}, mnesia:add_backend_type(ext_ets, ext_test)), + ?match({atomic,ok}, mnesia:add_backend_type(ext_dets, ext_test)), + ?verify_mnesia(Nodes, []), + ?cleanup(1, Config). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% start_one_disc_full_then_one_disc_less(doc)-> ["Start a disk node and then a disk less one. Distribute some", diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 1cdac3cde6..848f023aad 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -672,7 +672,7 @@ do_prepare([delete_schema | Actions], Selected, All, Config, File, Line) -> end, do_prepare(Actions, Selected, All, Config, File, Line); do_prepare([create_schema | Actions], Selected, All, Config, File, Line) -> - Ext = ?BACKEND, + Ext = proplists:get_value(default_properties, Config, ?BACKEND), case diskless(Config) of true -> rpc:multicall(Selected, application, set_env, [mnesia, schema, Ext]), diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 19a69fb714..a5832068a1 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.18.1 +MNESIA_VSN = 4.19 diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index 3533310e51..b219f79b87 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,23 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.13.2</title> + <section><title>ODBC 2.13.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Make sure odbc c-process exits when erlang process orders + it to shutdown.</p> + <p> + Own Id: OTP-17188 Aux Id: ERL-1448 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.13.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index c08f40d233..717da2b77c 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.13.2 +ODBC_VSN = 2.13.3 diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index 7cabfed09a..e831773871 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,46 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.10</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed case insensitive hostname check.</p> + <p> + Own Id: OTP-17242 Aux Id: GH-4500 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add sanity check of trusted anchor certificate expiration + to pkix_path_validation/3. Although the anchor is + considered a trusted input this sanity check does provide + extra security for the users of the public_key + application as this property needs to be checked at time + of usage and fits very well with the other checks + performed here.</p> + <p> + Own Id: OTP-16907</p> + </item> + <item> + <p> + Adjust generation of test certificates to conform to RFC + 5280 rules for formatting of the certificates validity</p> + <p> + Own Id: OTP-17111</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.9.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index 597aca35d5..8c8b5585a0 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -992,10 +992,19 @@ pkix_path_validation(TrustedCert, CertChain, Options) pkix_path_validation(#'OTPCertificate'{} = TrustedCert, CertChain, Options) when is_list(CertChain), is_list(Options) -> MaxPathDefault = length(CertChain), - ValidationState = pubkey_cert:init_validation_state(TrustedCert, - MaxPathDefault, - Options), - path_validation(CertChain, ValidationState). + {VerifyFun, Userstat0} = + proplists:get_value(verify_fun, Options, ?DEFAULT_VERIFYFUN), + try pubkey_cert:validate_time(TrustedCert, Userstat0, VerifyFun) of + Userstate1 -> + ValidationState = pubkey_cert:init_validation_state(TrustedCert, + MaxPathDefault, + [{verify_fun, {VerifyFun, Userstate1}} | + proplists:delete(verify_fun, Options)]), + path_validation(CertChain, ValidationState) + catch + throw:{bad_cert, cert_expired} = Reason -> + {error, Reason} + end. %-------------------------------------------------------------------- -spec pkix_crls_validate(OTPcertificate, DPandCRLs, Options) -> diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index 348e0daa62..cf3b5ea0c8 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -1,4 +1,3 @@ - %% %% %CopyrightBegin% %% @@ -85,6 +84,8 @@ pkix_emailaddress/1, pkix_path_validation/0, pkix_path_validation/1, + pkix_path_validation_root_expired/0, + pkix_path_validation_root_expired/1, pkix_verify_hostname_cn/1, pkix_verify_hostname_subjAltName/1, pkix_verify_hostname_options/1, @@ -126,12 +127,18 @@ suite() -> []. all() -> - [app, appup, + [app, + appup, {group, pem_decode_encode}, encrypt_decrypt, {group, sign_verify}, - pkix, pkix_countryname, pkix_emailaddress, pkix_path_validation, - pkix_iso_rsa_oid, pkix_iso_dsa_oid, + pkix, + pkix_countryname, + pkix_emailaddress, + pkix_path_validation, + pkix_path_validation_root_expired, + pkix_iso_rsa_oid, + pkix_iso_dsa_oid, pkix_dsa_sha2_oid, pkix_crl, pkix_hash_type, @@ -142,7 +149,8 @@ all() -> pkix_verify_hostname_options, pkix_test_data_all_default, pkix_test_data, - short_cert_issuer_hash, short_crl_issuer_hash + short_cert_issuer_hash, + short_crl_issuer_hash ]. groups() -> @@ -737,9 +745,24 @@ pkix_path_validation(Config) when is_list(Config) -> {error, custom_reason} = public_key:pkix_path_validation(selfsigned_peer, [Trusted], [{verify_fun, - VerifyFunAndState2}]), - ok. - + VerifyFunAndState2}]). +pkix_path_validation_root_expired() -> + [{doc, "Test root expiration so that it does not fall between chairs"}]. +pkix_path_validation_root_expired(Config) when is_list(Config) -> + {Year, Month, Day} = date(), + SRoot = public_key:pkix_test_root_cert("OTP test server ROOT", [{validity, {{Year-2, Month, Day}, + {Year-1, Month, Day}}}]), + #{server_config := Conf} = public_key:pkix_test_data(#{server_chain => #{root => SRoot, + intermediates => [], + peer => []}, + client_chain => #{root => [], + intermediates => [], + peer => []}}), + [ICA, Root] = proplists:get_value(cacerts, Conf), + true = public_key:pkix_is_self_signed(Root), + Peer = proplists:get_value(cert, Conf), + {error, {bad_cert, cert_expired}} = public_key:pkix_path_validation(Root, [ICA, Peer], []). + %%-------------------------------------------------------------------- %% To generate the PEM file contents: %% diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 0de40d1d3f..442b3cc18e 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.9.2 +PUBLIC_KEY_VSN = 1.10 diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index ba1562bf15..773e752ad4 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2018. All Rights Reserved. +%% Copyright Ericsson AB 2009-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -796,12 +796,8 @@ do_spec_rel_files(#rel{name = RelName} = Rel, Sys) -> PathFlag = true, {ok, Script} = do_gen_script(Rel, Sys, MergedApps, PathFlag, Variables), {ok, BootBin} = gen_boot(Script), - Date = date(), - Time = time(), - RelIoList = io_lib:format("%% rel generated at ~w ~w\n~tp.\n\n", - [Date, Time, GenRel]), - ScriptIoList = io_lib:format("%% script generated at ~w ~w\n~tp.\n\n", - [Date, Time, Script]), + RelIoList = io_lib:format("~tp.\n\n", [GenRel]), + ScriptIoList = io_lib:format("~tp.\n\n", [Script]), [ {write_file, RelFile, to_utf8_bin_with_enc_comment(RelIoList)}, {write_file, ScriptFile, to_utf8_bin_with_enc_comment(ScriptIoList)}, @@ -1199,8 +1195,7 @@ spec_app_file(#app{name = Name, Info#app_info.modules)], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", - [date(), time(), Contents]), + AppIoList = io_lib:format("~tp.\n\n", [Contents]), [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}]; all -> %% Include all included modules @@ -1208,8 +1203,7 @@ spec_app_file(#app{name = Name, ModNames = [M#mod.name || M <- Mods, M#mod.is_included], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", - [date(), time(), Contents]), + AppIoList = io_lib:format("~tp.\n\n", [Contents]), [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}] end. diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index efd5939fe0..6303397f57 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.16</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Clarify documentation of module 'scheduler'.</p> + <p> + Own Id: OTP-17208 Aux Id: GH-4502, PR-4532 </p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.15.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index b4e5e953e4..e62d59acf6 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.15.1 +RUNTIME_TOOLS_VSN = 1.16 diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index a15351cd23..46e6762b90 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -31,6 +31,31 @@ </header> <p>This document describes the changes made to the SASL application.</p> +<section><title>SASL 4.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix dependent application to be stopped after the primary + application when upgrading a release and the primary and + dependent application is removed.</p> + <p> + Example: In a release where app1 depends on app2 and we + should remove app1 and app2 using a release upgrade. When + the release upgrade is done app1 should be stopped and + purged before app2 as otherwise app1 could start crashing + when its dependency is removed.</p> + <p> + This bugfix changes the order of removal to be correct.</p> + <p> + Own Id: OTP-17113 Aux Id: ERL-1410 PR-2882 </p> + </item> + </list> + </section> + +</section> + <section><title>SASL 4.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/sasl/src/sasl.appup.src b/lib/sasl/src/sasl.appup.src index a8e1b85ff9..4fff8f79ab 100644 --- a/lib/sasl/src/sasl.appup.src +++ b/lib/sasl/src/sasl.appup.src @@ -38,7 +38,8 @@ {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.4\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^4\\.0$">>,[restart_new_emulator]}, - {<<"^4\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^4\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^4\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^3\\.2$">>,[restart_new_emulator]}, {<<"^3\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -49,4 +50,5 @@ {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.4\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^4\\.0$">>,[restart_new_emulator]}, - {<<"^4\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^4\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^4\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 7b86bf58d0..ca4fb4300a 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2020. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% -module(systools_make). @@ -1198,8 +1198,8 @@ generate_script(Output, Release, Appls, Flags) -> ScriptFile = Output ++ ".script", case file:open(ScriptFile, [write,{encoding,utf8}]) of {ok, Fd} -> - io:format(Fd, "%% ~s\n%% script generated at ~w ~w\n~tp.\n", - [epp:encoding_to_string(utf8), date(), time(), Script]), + io:format(Fd, "%% ~s\n~tp.\n", + [epp:encoding_to_string(utf8), Script]), case file:close(Fd) of ok -> BootFile = Output ++ ".boot", diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index f2b315c59e..a61b72d6f6 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 4.0.1 +SASL_VSN = 4.0.2 diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 792090f3bf..f8caf6228b 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -34,7 +34,24 @@ </header> - <section><title>SNMP 5.7.3</title> + <section><title>SNMP 5.8</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add function to get a list of configured agent + transports. Also improved agent info with regards to + transports.</p> + <p> + Own Id: OTP-17109 Aux Id: ERIERL-583 </p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.7.3</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/snmp/src/agent/snmpa.erl b/lib/snmp/src/agent/snmpa.erl index 729789d487..995e6d627d 100644 --- a/lib/snmp/src/agent/snmpa.erl +++ b/lib/snmp/src/agent/snmpa.erl @@ -39,7 +39,7 @@ int_to_enum/2, int_to_enum/3, enum_to_int/2, enum_to_int/3, - info/0, info/1, old_info_format/1, + info/0, info/1, load_mib/1, load_mib/2, load_mibs/1, load_mibs/2, load_mibs/3, unload_mib/1, unload_mib/2, @@ -125,9 +125,6 @@ mib_storage_options/0 ]). --deprecated([{old_info_format, 1, "use \"new\" format instead"}]). - - -include("snmpa_atl.hrl"). -include("snmpa_internal.hrl"). -include_lib("snmp/include/snmp_types.hrl"). % type of me needed. @@ -280,16 +277,6 @@ get_next(Agent, Vars, Context) -> snmpa_agent:get_next(Agent, Vars, Context). info() -> info(snmp_master_agent). info(Agent) -> snmpa_agent:info(Agent). -old_info_format(Info) when is_list(Info) -> - {value, Vsns} = lists:keysearch(vsns, 1, Info), - {value, {_, MibInfo}} = lists:keysearch(mib_server, 1, Info), - {value, SAa} = lists:keysearch(subagents, 1, MibInfo), - {value, LoadedMibs} = lists:keysearch(loaded_mibs, 1, MibInfo), - {value, TreeSz} = lists:keysearch(tree_size_bytes, 1, MibInfo), - {value, ProcMem} = lists:keysearch(process_memory, 1, MibInfo), - {value, DbMem} = lists:keysearch(db_memory, 1, MibInfo), - [Vsns, SAa, LoadedMibs, TreeSz, ProcMem, DbMem]. - %% - diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index e1e018878a..102de54127 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.7.3 +SNMP_VSN = 5.8 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/SSH_app.xml b/lib/ssh/doc/src/SSH_app.xml index 490920bd4e..3b24e40db9 100644 --- a/lib/ssh/doc/src/SSH_app.xml +++ b/lib/ssh/doc/src/SSH_app.xml @@ -67,13 +67,13 @@ <item><c>known_hosts</c></item> <item><c>authorized_keys</c></item> <item><c>authorized_keys2</c></item> - <item><c>id_dsa</c> <i>(disabled by default)</i></item> - <item><c>id_rsa</c> <i>(disabled by default from OTP-24)</i></item> + <item><c>id_dsa</c> <i>(supported but disabled by default)</i></item> + <item><c>id_rsa</c> <i>(SHA1 sign/verify are supported but disabled by default from OTP-24)</i></item> <item><c>id_ecdsa</c></item> <item><c>id_ed25519</c></item> <item><c>id_ed448</c></item> - <item><c>ssh_host_dsa_key</c> <i>(disabled by default)</i></item> - <item><c>ssh_host_rsa_key</c> <i>(disabled by default from OTP-24)</i></item> + <item><c>ssh_host_dsa_key</c> <i>(supported but disabled by default)</i></item> + <item><c>ssh_host_rsa_key</c> <i>(SHA1 sign/verify are supported but disabled by default from OTP-24)</i></item> <item><c>ssh_host_ecdsa_key</c></item> <item><c>ssh_host_ed25519_key</c></item> <item><c>ssh_host_ed448_key</c></item> @@ -95,6 +95,15 @@ <p>See also the default callback module documentation in <seeerl marker="ssh_file">ssh_file</seeerl>. </p> + <p>Disabled public key algorithms can be enabled with the + <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> + or + <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> + options. + See <seeguide marker="configure_algos#example-9">Example 9</seeguide> in + <seeguide marker="configure_algos">Configuring algorithms in SSH</seeguide> + for a description. + </p> </section> <section> @@ -174,13 +183,13 @@ <item>curve25519-sha256@libssh.org</item> <item>curve448-sha512</item> </list> - <p>The following unsecure <c>sha1</c> algorithms are now disabled by default:</p> + <p>The following unsecure <c>SHA1</c> algorithms are now disabled by default:</p> <list> <item>(diffie-hellman-group14-sha1)</item> <item>(diffie-hellman-group-exchange-sha1)</item> <item>(diffie-hellman-group1-sha1)</item> </list> - <p>They can be enabled with the + <p>They can be enabled with the <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> @@ -197,17 +206,22 @@ <item>ssh-ed448</item> <item>rsa-sha2-256</item> <item>rsa-sha2-512</item> - <item>ssh-rsa <i>(disabled by default from OTP-24)</i></item> </list> - <p>The following unsecure <c>sha1</c> algorithm is now disabled by default:</p> + <p>The following unsecure <c>SHA1</c> algorithms are supported but disabled by default:</p> <list> <item>(ssh-dss)</item> + <item>(ssh-rsa)</item> </list> - <p>It can be enabled with the + <p>See + Disabled public key algorithms can be enabled with the <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> - options. Use for example the Option value <c>{modify_algorithms, [{append, [{public_key,['ssh-dss']}]}]}</c>)</p> + options. + See <seeguide marker="configure_algos#example-9">Example 9</seeguide> in + <seeguide marker="configure_algos">Configuring algorithms in SSH</seeguide> + for a description. + </p> </item> <tag>MAC algorithms</tag> @@ -220,11 +234,11 @@ <item>hmac-sha2-512</item> <item>hmac-sha1</item> </list> - <p>The following unsecure <c>sha1</c> algorithm is disabled by default:</p> + <p>The following unsecure <c>SHA1</c> algorithm is disabled by default:</p> <list> <item>(hmac-sha1-96)</item> </list> - <p>It can be enabled with the + <p>It can be enabled with the <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> @@ -250,7 +264,7 @@ <p>See the text at the description of <seeapp marker="#rfc5647_note">the rfc 5647 further down</seeapp> for more information regarding AEAD_AES_*_GCM. </p> - <p>Following the internet de-facto standard, the cipher and mac algorithm AEAD_AES_128_GCM is selected when the + <p>Following the internet de-facto standard, the cipher and mac algorithm AEAD_AES_128_GCM is selected when the cipher aes128-gcm@openssh.com is negotiated. The cipher and mac algorithm AEAD_AES_256_GCM is selected when the cipher aes256-gcm@openssh.com is negotiated. </p> @@ -304,14 +318,21 @@ <item>6.6. Public Key Algorithms <list type="bulleted"> <item>ssh-dss</item> + <item>ssh-rsa</item> </list> </item> </list> - <p>They are disabled by default, but can be enabled with the + <p>They are disabled by default as they now are regarded insecure, but they can be enabled with the <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> options. + See <seeguide marker="configure_algos#example-8">Example 8</seeguide> (diffie-hellman-group1-sha1) + and + <seeguide marker="configure_algos#example-9">Example 9</seeguide> (ssh-dss) + in + <seeguide marker="configure_algos">Configuring algorithms in SSH</seeguide> + for descriptions. </p> </item> @@ -341,7 +362,7 @@ <list type="bulleted"> <item>4.1. diffie-hellman-group-exchange-sha1</item> </list> - <p>It is disabled by default, but can be enabled with the + <p>It is disabled by defaultas as it now is regarded insecure, but it can be enabled with the <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or <seetype marker="ssh:ssh#modify_algorithms_common_option">modify_algorithms</seetype> @@ -406,7 +427,7 @@ <item><c>diffie-hellman-group-exchange-sha1</c></item> <item><c>diffie-hellman-group14-sha1</c></item> </list> - <p>are not enabled by default, + <p>are not enabled by default as they now are regarded insecure, but are still supported and can be enabled with the options <seetype marker="ssh:ssh#preferred_algorithms_common_option">preferred_algorithms</seetype> or @@ -430,7 +451,7 @@ </item> <item> - <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-curves">Secure Shell (SSH) Key Exchange Method using Curve25519 and Curve448 (work in progress)</url> + <url href="https://tools.ietf.org/html/rfc8731">Secure Shell (SSH) Key Exchange Method Using Curve25519 and Curve448</url> <p/> </item> diff --git a/lib/ssh/doc/src/configure_algos.xml b/lib/ssh/doc/src/configure_algos.xml index 66426f3627..1b26cd2f25 100644 --- a/lib/ssh/doc/src/configure_algos.xml +++ b/lib/ssh/doc/src/configure_algos.xml @@ -429,7 +429,49 @@ but it is possible if an unforeseen need should arise.</p> </section> - + <section> + <title>Example 8</title> + <p>In this example, we need to use a diffie-hellman-group1-sha1 key exchange algorithm + although it is unsage and disabled by default. + </p> + <p>We use the + <seetype marker="ssh#modify_algorithms_common_option">modify_algorithms</seetype> + option, because we want to keep all other algorithm definitions. + </p> + <p>We add the option: + </p> + <code type="erl"> + {modify_algorithms, [{append, [{kex,['diffie-hellman-group1-sha1']}]}]} + </code> + <p>either to the Options list in a function call, in the <c>ssh.app</c> file or in a <c>.config</c> file for + the <c>erl</c> command. + See the chapter + <seeguide marker="ssh:configurations">Configuration in SSH</seeguide> + in the SSH User's Guide. + </p> + </section> + + <section> + <title>Example 9</title> + <p>In this example, we need to use a DSA key for sign and verify. + It might be either as a user's key, a host's key or both. + </p> + <p>To do that, we enable the 'ssh-dss' algorithm that is disabled by default by security reasons. We use the + <seetype marker="ssh#modify_algorithms_common_option">modify_algorithms</seetype> + option, because we want to keep all other algorithm definitions. + </p> + <p>We add the option: + </p> + <code type="erl"> + {modify_algorithms, [{append, [{public_key,['ssh-dss']}]}]} + </code> + <p>either to the Options list in a function call, in the <c>ssh.app</c> file or in a <c>.config</c> file for + the <c>erl</c> command. + See the chapter + <seeguide marker="ssh:configurations">Configuration in SSH</seeguide> + in the SSH User's Guide. + </p> + </section> </section> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 60c3a9b9a5..bd5bff57ab 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,41 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.11</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The long name field in SSH_FXP_NAME responses to display + file information in sftp version 3 now contains the + expanded format defined in the sftp draft. It is similar + to what is returned by "ls -l" on Unix systems.</p> + <p> + Own Id: OTP-17197 Aux Id: PR- 3049 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.10.8</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -486,6 +521,23 @@ </section> +<section><title>Ssh 4.9.1.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.9.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -823,6 +875,23 @@ </section> +<section><title>Ssh 4.7.6.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The idle_time timer was not cancelled when a channel was + opened within the timeout time on an empty connection + that have had channels previously.</p> + <p> + Own Id: OTP-17279</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.7.6.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/prebuild.skip b/lib/ssh/prebuild.skip new file mode 100644 index 0000000000..ff54b7fafb --- /dev/null +++ b/lib/ssh/prebuild.skip @@ -0,0 +1 @@ +src/deps/ssh.d diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 4f4b55f5c5..29d77fa6f5 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -126,6 +126,17 @@ INTERNAL_HRL_FILES = \ ssh_transport.hrl \ ssh_xfer.hrl +DEPDIR=$(ERL_TOP)/lib/ssh/src/deps +DEP_FILE=$(DEPDIR)/ssh.d +$(shell mkdir -p $(dir $(DEP_FILE)) >/dev/null) + +ifeq ($(TARGET), win32) + # Native path without C: ignore driveletter case + ERL_TOP_NATIVE = $(shell w32_path.sh -m $(ERL_TOP) | sed "s@[a-zA-Z]:@:@") +else + ERL_TOP_NATIVE = $(ERL_TOP) +endif + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- @@ -143,11 +154,22 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/kernel/src \ $(TARGET_FILES): $(BEHAVIOUR_TARGET_FILES_2) $(BEHAVIOUR_TARGET_FILES_2): $(BEHAVIOUR_TARGET_FILES_1) -debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) +$(DEP_FILE): $(ERL_FILES) + @echo SED $(TARGET) $(ERL_TOP_NATIVE) + $(gen_verbose)erlc -M $(ERL_FILES) \ + | sed "s@[a-zA-Z]\?$(ERL_TOP_NATIVE)@../../..@g" \ + | sed "s/\.$(EMULATOR)/\.$$\(EMULATOR\)/" \ + | sed 's@^ssh_@$$(EBIN)/ssh_@' \ + | sed 's@^sshc_@$$(EBIN)/sshc_@' \ + | sed 's@^sshd_@$$(EBIN)/sshd_@' \ + > $(DEP_FILE) + +debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(DEP_FILE) clean: rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(BEHAVIOUR_TARGET_FILES) rm -f errs core *~ + rm -rf $(DEPDIR) $(APP_TARGET): $(APP_SRC) ../vsn.mk $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ @@ -174,87 +196,7 @@ release_spec: opt release_docs_spec: - -deps: - erlc -M $(ERL_FILES) \ - | sed 's@$(ERL_TOP)/lib@../..@g' \ - | sed 's/\.$(EMULATOR)/\.$$\(EMULATOR\)/' \ - | sed 's@^ssh_@$$(EBIN)/ssh_@' - -ssh.$(EMULATOR): ssh.erl ssh.hrl ssh_connect.hrl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/file.hrl -$(EBIN)/ssh_sup.$(EMULATOR): ssh_sup.erl -sshc_sup.$(EMULATOR): sshc_sup.erl -sshd_sup.$(EMULATOR): sshd_sup.erl ssh.hrl -$(EBIN)/ssh_connection_sup.$(EMULATOR): ssh_connection_sup.erl -$(EBIN)/ssh_connection.$(EMULATOR): ssh_connection.erl ssh.hrl ssh_connect.hrl \ - ssh_transport.hrl -$(EBIN)/ssh_connection_handler.$(EMULATOR): ssh_connection_handler.erl ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_kexint.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_userauth_client.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_fsm_userauth_server.$(EMULATOR): ssh.hrl \ - ssh_transport.hrl ssh_auth.hrl ssh_connect.hrl ssh_fsm.hrl -$(EBIN)/ssh_shell.$(EMULATOR): ssh_shell.erl ssh_connect.hrl -$(EBIN)/ssh_system_sup.$(EMULATOR): ssh_system_sup.erl ssh.hrl -$(EBIN)/ssh_subsystem_sup.$(EMULATOR): ssh_subsystem_sup.erl -$(EBIN)/ssh_channel_sup.$(EMULATOR): ssh_channel_sup.erl ssh.hrl -$(EBIN)/ssh_acceptor_sup.$(EMULATOR): ssh_acceptor_sup.erl ssh.hrl -$(EBIN)/ssh_acceptor.$(EMULATOR): ssh_acceptor.erl ssh.hrl -$(EBIN)/ssh_agent.$(EMULATOR): ssh_agent.erl ssh.hrl ssh_agent.hrl -$(EBIN)/ssh_app.$(EMULATOR): ssh_app.erl -$(EBIN)/ssh_auth.$(EMULATOR): ssh_auth.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl ssh_auth.hrl ssh_transport.hrl -$(EBIN)/ssh_bits.$(EMULATOR): ssh_bits.erl ssh.hrl -$(EBIN)/ssh_cli.$(EMULATOR): ssh_cli.erl ssh.hrl ssh_connect.hrl -$(EBIN)/ssh_file.$(EMULATOR): ssh_file.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/file.hrl ssh.hrl -$(EBIN)/ssh_io.$(EMULATOR): ssh_io.erl ssh.hrl -$(EBIN)/ssh_info.$(EMULATOR): ssh_info.erl -$(EBIN)/ssh_message.$(EMULATOR): ssh_message.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl ssh_connect.hrl ssh_auth.hrl ssh_transport.hrl -$(EBIN)/ssh_no_io.$(EMULATOR): ssh_no_io.erl ssh_transport.hrl -$(EBIN)/ssh_sftp.$(EMULATOR): ssh_sftp.erl \ - ../../kernel/include/file.hrl ssh.hrl \ - ssh_xfer.hrl -$(EBIN)/ssh_sftpd.$(EMULATOR): ssh_sftpd.erl \ - ../../kernel/include/file.hrl ssh.hrl \ - ssh_xfer.hrl -$(EBIN)/ssh_sftpd_file.$(EMULATOR): ssh_sftpd_file.erl -$(EBIN)/ssh_transport.$(EMULATOR): ssh_transport.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ../../kernel/include/inet.hrl \ - ssh_transport.hrl ssh.hrl -$(EBIN)/ssh_xfer.$(EMULATOR): ssh_xfer.erl ssh.hrl ssh_xfer.hrl -$(EBIN)/ssh_sftpd_file_api.$(EMULATOR): ssh_sftpd_file_api.erl -$(EBIN)/ssh_client_channel.$(EMULATOR): ssh_client_channel.erl ssh_connect.hrl -$(EBIN)/ssh_channel.$(EMULATOR): ssh_channel.erl ssh_connect.hrl -$(EBIN)/ssh_daemon_channel.$(EMULATOR): ssh_daemon_channel.erl -$(EBIN)/ssh_server_channel.$(EMULATOR): ssh_server_channel.erl -$(EBIN)/ssh_client_key_api.$(EMULATOR): ssh_client_key_api.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl -$(EBIN)/ssh_server_key_api.$(EMULATOR): ssh_server_key_api.erl \ - ../../public_key/include/public_key.hrl \ - ../../public_key/include/OTP-PUB-KEY.hrl \ - ../../public_key/include/PKCS-FRAME.hrl \ - ssh.hrl - +# ---------------------------------------------------- +# Dependencies +# ---------------------------------------------------- +-include $(DEP_FILE) diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 5bc2859b4e..fd0ca11e6e 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -172,26 +172,28 @@ publickey_msg([SigAlg, #ssh{user = User, SigAlgStr = atom_to_list(SigAlg), SigData = build_sig_data(SessionId, User, Service, PubKeyBlob, SigAlgStr), - Sig = case Key of - {ssh2_pubkey, PubKeyBlob} -> - ssh_transport:call_KeyCb(sign, [PubKeyBlob, SigData], Opts); - - {PrivKey, PubKeyBlob} -> - Hash = ssh_transport:sha(SigAlg), - ssh_transport:sign(SigData, Hash, PrivKey) - end, - - SigBlob = list_to_binary([?string(SigAlgStr), - ?binary(Sig)]), - - {#ssh_msg_userauth_request{user = User, - service = Service, - method = "publickey", - data = [?TRUE, - ?string(SigAlgStr), - ?binary(PubKeyBlob), - ?binary(SigBlob)]}, - Ssh}; + SigRes = case Key of + {ssh2_pubkey, PubKeyBlob} -> + {ok, ssh_transport:call_KeyCb(sign, [PubKeyBlob, SigData], Opts)}; + {PrivKey, PubKeyBlob} -> + ssh_transport:sign(SigData, SigAlg, PrivKey, Ssh) + end, + case SigRes of + {ok,Sig} -> + SigBlob = list_to_binary([?string(SigAlgStr), + ?binary(Sig)]), + + {#ssh_msg_userauth_request{user = User, + service = Service, + method = "publickey", + data = [?TRUE, + ?string(SigAlgStr), + ?binary(PubKeyBlob), + ?binary(SigBlob)]}, + Ssh}; + {error,_} -> + {not_ok, Ssh} + end; _ -> {not_ok, Ssh} @@ -545,13 +547,16 @@ pre_verify_sig(User, KeyBlob, #ssh{opts=Opts}) -> verify_sig(SessionId, User, Service, AlgBin, KeyBlob, SigWLen, #ssh{opts=Opts} = Ssh) -> try Alg = binary_to_list(AlgBin), + true = lists:member(list_to_existing_atom(Alg), + proplists:get_value(public_key, + ?GET_OPT(preferred_algorithms,Opts))), Key = ssh_message:ssh2_pubkey_decode(KeyBlob), % or exception true = ssh_transport:call_KeyCb(is_auth_key, [Key, User], Opts), PlainText = build_sig_data(SessionId, User, Service, KeyBlob, Alg), <<?UINT32(AlgSigLen), AlgSig:AlgSigLen/binary>> = SigWLen, <<?UINT32(AlgLen), _Alg:AlgLen/binary, ?UINT32(SigLen), Sig:SigLen/binary>> = AlgSig, - ssh_transport:verify(PlainText, ssh_transport:sha(Alg), Sig, Key, Ssh) + ssh_transport:verify(PlainText, list_to_existing_atom(Alg), Sig, Key, Ssh) catch _:_ -> false diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 5d564b6db8..1bcfeb1146 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1254,8 +1254,13 @@ handle_event(info, {'DOWN', _Ref, process, ChannelPid, _Reason}, _, D) -> end, [], Cache), {keep_state, D, cond_set_idle_timer(D)}; -handle_event({timeout,idle_time}, _Data, _StateName, _D) -> - {stop, {shutdown, "Timeout"}}; +handle_event({timeout,idle_time}, _Data, _StateName, D) -> + case ssh_client_channel:cache_info(num_entries, cache(D)) of + 0 -> + {stop, {shutdown, "Timeout"}}; + _ -> + keep_state_and_data + end; %%% So that terminate will be run when supervisor is shutdown handle_event(info, {'EXIT', _Sup, Reason}, StateName, _D) -> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 813d5ad350..65d695ecb1 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -52,7 +52,9 @@ extract_public_key/1, ssh_packet/2, pack/2, valid_key_sha_alg/3, - sha/1, sign/3, verify/5, + sign/3, sign/4, + verify/5, + sha/1, get_host_key/2, call_KeyCb/3, public_algo/1]). @@ -166,6 +168,7 @@ default_algorithms1(mac) -> default_algorithms1(public_key) -> supported_algorithms(public_key, [ + 'ssh-rsa', %% Gone in OpenSSH 7.3.p1: 'ssh-dss' ]); @@ -488,17 +491,22 @@ handle_kexdh_init(#ssh_msg_kexdh_init{e = E}, MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Kex), {E,Public,K}), - H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), - {SshPacket, Ssh1} = - ssh_packet(#ssh_msg_kexdh_reply{public_host_key = {MyPubHostKey,SignAlg}, - f = Public, - h_sig = H_SIG - }, Ssh0), - {ok, SshPacket, Ssh1#ssh{keyex_key = {{Private, Public}, {G, P}}, - shared_secret = ssh_bits:mpint(K), - exchanged_hash = H, - session_id = sid(Ssh1, H)}}; - + case sign(H, SignAlg, MyPrivHostKey, Ssh0) of + {ok,H_SIG} -> + {SshPacket, Ssh1} = + ssh_packet(#ssh_msg_kexdh_reply{public_host_key = {MyPubHostKey,SignAlg}, + f = Public, + h_sig = H_SIG + }, Ssh0), + {ok, SshPacket, Ssh1#ssh{keyex_key = {{Private, Public}, {G, P}}, + shared_secret = ssh_bits:mpint(K), + exchanged_hash = H, + session_id = sid(Ssh1, H)}}; + {error,unsupported_sign_alg} -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Unsupported algorithm ~p", [SignAlg]) + ) + end; true -> ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, io_lib:format("Kexdh init failed, received 'e' out of bounds~n E=~p~n P=~p", @@ -635,15 +643,21 @@ handle_kex_dh_gex_init(#ssh_msg_kex_dh_gex_init{e = E}, MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Kex), {Min,NBits,Max,P,G,E,Public,K}), - H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), - {SshPacket, Ssh} = - ssh_packet(#ssh_msg_kex_dh_gex_reply{public_host_key = {MyPubHostKey,SignAlg}, - f = Public, - h_sig = H_SIG}, Ssh0), - {ok, SshPacket, Ssh#ssh{shared_secret = ssh_bits:mpint(K), - exchanged_hash = H, - session_id = sid(Ssh, H) - }}; + case sign(H, SignAlg, MyPrivHostKey, Ssh0) of + {ok,H_SIG} -> + {SshPacket, Ssh} = + ssh_packet(#ssh_msg_kex_dh_gex_reply{public_host_key = {MyPubHostKey,SignAlg}, + f = Public, + h_sig = H_SIG}, Ssh0), + {ok, SshPacket, Ssh#ssh{shared_secret = ssh_bits:mpint(K), + exchanged_hash = H, + session_id = sid(Ssh, H) + }}; + {error,unsupported_sign_alg} -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Unsupported algorithm ~p", [SignAlg]) + ) + end; true -> ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, "Kexdh init failed, received 'k' out of bounds" @@ -712,16 +726,22 @@ handle_kex_ecdh_init(#ssh_msg_kex_ecdh_init{q_c = PeerPublic}, MyPrivHostKey = get_host_key(SignAlg, Opts), MyPubHostKey = extract_public_key(MyPrivHostKey), H = kex_hash(Ssh0, MyPubHostKey, sha(Curve), {PeerPublic, MyPublic, K}), - H_SIG = sign(H, sha(SignAlg), MyPrivHostKey), - {SshPacket, Ssh1} = - ssh_packet(#ssh_msg_kex_ecdh_reply{public_host_key = {MyPubHostKey,SignAlg}, - q_s = MyPublic, - h_sig = H_SIG}, - Ssh0), - {ok, SshPacket, Ssh1#ssh{keyex_key = {{MyPublic,MyPrivate},Curve}, - shared_secret = ssh_bits:mpint(K), - exchanged_hash = H, - session_id = sid(Ssh1, H)}} + case sign(H, SignAlg, MyPrivHostKey, Ssh0) of + {ok,H_SIG} -> + {SshPacket, Ssh1} = + ssh_packet(#ssh_msg_kex_ecdh_reply{public_host_key = {MyPubHostKey,SignAlg}, + q_s = MyPublic, + h_sig = H_SIG}, + Ssh0), + {ok, SshPacket, Ssh1#ssh{keyex_key = {{MyPublic,MyPrivate},Curve}, + shared_secret = ssh_bits:mpint(K), + exchanged_hash = H, + session_id = sid(Ssh1, H)}}; + {error,unsupported_sign_alg} -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Unsupported algorithm ~p", [SignAlg]) + ) + end catch Class:Error -> ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, @@ -864,7 +884,7 @@ extract_public_key(#{engine:=_, key_id:=_, algorithm:=Alg} = M) -> verify_host_key(#ssh{algorithms=Alg}=SSH, PublicKey, Digest, {AlgStr,Signature}) -> case atom_to_list(Alg#alg.hkey) of AlgStr -> - case verify(Digest, sha(Alg#alg.hkey), Signature, PublicKey, SSH) of + case verify(Digest, Alg#alg.hkey, Signature, PublicKey, SSH) of false -> {error, bad_signature}; true -> @@ -1402,6 +1422,19 @@ payload(<<PacketLen:32, PaddingLen:8, PayloadAndPadding/binary>>) -> Payload. %%%---------------------------------------------------------------- +%% sign(SigData, SignAlg, Key, Opts) when is_list(SignAlg) -> +%% sign(SigData, list_to_existing_atom(SignAlg), Key, Opts); + +sign(SigData, SignAlg, Key, #ssh{opts=Opts}) when is_atom(SignAlg) -> + case lists:member(SignAlg, + proplists:get_value(public_key, + ?GET_OPT(preferred_algorithms,Opts,[]))) of + true -> + {ok, sign(SigData, sha(SignAlg), Key)}; + false -> + {error, unsupported_sign_alg} + end. + sign(SigData, HashAlg, #{algorithm:=dss} = Key) -> mk_dss_sig(crypto:sign(dss, HashAlg, SigData, Key)); sign(SigData, HashAlg, #{algorithm:=SigAlg} = Key) -> @@ -1421,7 +1454,11 @@ mk_dss_sig(DerSignature) -> <<R:160/big-unsigned-integer, S:160/big-unsigned-integer>>. %%%---------------------------------------------------------------- -verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key, _) -> +verify(PlainText, Alg, Sig, Key, Ssh) -> + do_verify(PlainText, sha(Alg), Sig, Key, Ssh). + + +do_verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key, _) -> case Sig of <<R:160/big-unsigned-integer, S:160/big-unsigned-integer>> -> Signature = public_key:der_encode('Dss-Sig-Value', #'Dss-Sig-Value'{r = R, s = S}), @@ -1429,7 +1466,7 @@ verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key, _) -> _ -> false end; -verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key, _) -> +do_verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key, _) -> case Sig of <<?UINT32(Rlen),R:Rlen/big-signed-integer-unit:8, ?UINT32(Slen),S:Slen/big-signed-integer-unit:8>> -> @@ -1440,14 +1477,14 @@ verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key, _) -> false end; -verify(PlainText, HashAlg, Sig, #'RSAPublicKey'{}=Key, #ssh{role = server, - c_version = "SSH-2.0-OpenSSH_7."++_}) +do_verify(PlainText, HashAlg, Sig, #'RSAPublicKey'{}=Key, #ssh{role = server, + c_version = "SSH-2.0-OpenSSH_7."++_}) when HashAlg == sha256; HashAlg == sha512 -> %% Public key signing bug in in OpenSSH >= 7.2 public_key:verify(PlainText, HashAlg, Sig, Key) orelse public_key:verify(PlainText, sha, Sig, Key); -verify(PlainText, HashAlg, Sig, Key, _) -> +do_verify(PlainText, HashAlg, Sig, Key, _) -> public_key:verify(PlainText, HashAlg, Sig, Key). diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 04627052e5..a3db492f0f 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -26,6 +26,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- MODULES= \ + ssh_cth \ ssh_algorithms_SUITE \ ssh_options_SUITE \ ssh_basic_SUITE \ diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 402d5b89f0..c1251338f9 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -430,43 +430,40 @@ exec_compressed(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -%%% Idle timeout test, client -idle_time_client(Config) -> - SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), - UserDir = proplists:get_value(priv_dir, Config), +%%% Idle timeout test +idle_time_client(Config) -> idle_time_common([], [{idle_time, 2000}], Config). - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {user_dir, UserDir}, - {failfun, fun ssh_test_lib:failfun/2}]), - ConnectionRef = - ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, - {user_dir, UserDir}, - {user_interaction, false}, - {idle_time, 2000}]), - {ok, Id} = ssh_connection:session_channel(ConnectionRef, 1000), - ssh_connection:close(ConnectionRef, Id), - receive - after 10000 -> - {error, closed} = ssh_connection:session_channel(ConnectionRef, 1000) - end, - ssh:stop_daemon(Pid). +idle_time_server(Config) -> idle_time_common([{idle_time, 2000}], [], Config). -%%-------------------------------------------------------------------- -%%% Idle timeout test, server -idle_time_server(Config) -> + +idle_time_common(DaemonExtraOpts, ClientExtraOpts, Config) -> SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), UserDir = proplists:get_value(priv_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, - {idle_time, 2000}, - {failfun, fun ssh_test_lib:failfun/2}]), + {failfun, fun ssh_test_lib:failfun/2} + | DaemonExtraOpts + ]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user_dir, UserDir}, - {user_interaction, false}]), - {ok, Id} = ssh_connection:session_channel(ConnectionRef, 1000), - ssh_connection:close(ConnectionRef, Id), + {user_interaction, false} + | ClientExtraOpts + ]), + {ok, Id1} = ssh_sftp:start_channel(ConnectionRef), + {ok, Id2} = ssh_sftp:start_channel(ConnectionRef), + ssh_sftp:stop_channel(Id2), + timer:sleep(2500), + {ok, Id3} = ssh_sftp:start_channel(ConnectionRef), + ssh_sftp:stop_channel(Id1), + ssh_sftp:stop_channel(Id3), + timer:sleep(1000), + {ok, Id4} = ssh_sftp:start_channel(ConnectionRef), + timer:sleep(2500), + {ok, Id5} = ssh_sftp:start_channel(ConnectionRef), + ssh_sftp:stop_channel(Id4), + ssh_sftp:stop_channel(Id5), receive after 10000 -> {error, closed} = ssh_connection:session_channel(ConnectionRef, 1000) diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 6df42660ed..bb46653657 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -310,6 +310,7 @@ all_algorithms_sftp_exec_reneg_otp_is_client(Config) -> %%-------------------------------------------------------------------- renegotiation_otp_is_server(Config) -> PublicKeyAlgs = [A || {public_key,A} <- proplists:get_value(common_remote_client_algs, Config, [])], + ct:log("PublicKeyAlgs = ~p", [PublicKeyAlgs]), UserDir = setup_remote_priv_and_local_auth_keys(hd(PublicKeyAlgs), Config), SftpRootDir = new_dir(Config), ct:log("Rootdir = ~p",[SftpRootDir]), @@ -321,6 +322,7 @@ renegotiation_otp_is_server(Config) -> {user_dir, UserDir}, {user_passwords, [{?USER,?PASSWD}]}, {failfun, fun ssh_test_lib:failfun/2}, + {modify_algorithms, [{append, [{public_key,PublicKeyAlgs}]}]}, {connectfun, fun(_,_,_) -> HostConnRef = self(), @@ -1237,13 +1239,13 @@ call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir, Ref) -> recv_log_msgs(C, Ch) -> receive {ssh_cm,C,{closed,Ch}} -> - %% ct:log("Channel closed ~p",[{closed,1}]), + ct:log("Channel closed ~p",[{closed,1}]), ok; {ssh_cm,C,{data,Ch,1,Msg}} -> ct:log("*** ERROR from docker:~n~s",[Msg]), recv_log_msgs(C, Ch); {ssh_cm,C,_Msg} -> - %% ct:log("Got ~p",[_Msg]), + ct:log("Got ~p",[_Msg]), recv_log_msgs(C, Ch) after 30000 -> diff --git a/lib/ssh/test/ssh_cth.erl b/lib/ssh/test/ssh_cth.erl new file mode 100644 index 0000000000..09d863cc51 --- /dev/null +++ b/lib/ssh/test/ssh_cth.erl @@ -0,0 +1,64 @@ +-module(ssh_cth). + +-export([id/1, + init/2, + pre_init_per_suite/3, + pre_end_per_suite/3, + pre_init_per_group/4, + pre_end_per_group/4, + post_end_per_group/5, + pre_init_per_testcase/3, + pre_end_per_testcase/4 + ]). + +-record(c, { + suite, % suite name + groups = [], % Group path in reversed order + n % test case number + }). + +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +init(_Id, _Opts) -> + {ok, #c{n=1}}. + +pre_init_per_suite(Suite, Config, CTHState) -> + {Config, CTHState#c{suite=Suite}}. + +pre_end_per_suite(Suite, Config, State) -> + ct:pal("BEGIN ~p:end_per_suite(...)", [Suite]), + {Config, State}. + + +pre_init_per_group(_Suite, Group, Config ,State) -> + {Config, State#c{groups = (State#c.groups ++ [Group])}}. + +pre_end_per_group(Suite, Group, Config, State) -> + ct:pal("BEGIN ~p:end_per_group(~p,...)", [Suite,Group]), + {Config, State}. + +post_end_per_group(_Suite, Group, _Config, Return, State) -> + {Return, State#c{groups = lists:reverse(lists:reverse(State#c.groups)--[Group])}}. + + +pre_init_per_testcase(SuiteName, Config, CTHState) -> + ct:pal("########## ~p ~p ~s ~p~n", [CTHState#c.suite, CTHState#c.n, groups(Config), SuiteName]), + {Config, CTHState#c{n = CTHState#c.n + 1}}. + +pre_end_per_testcase(Suite,TC,Config,State) -> + ct:pal("BEGIN ~p:end_per_testcase(~p,...)", [Suite,TC]), + {Config, State}. + + +groups(Config) -> + F = fun(X) -> io_lib:format("~w",[X]) end, + io_lib:format("~s", [lists:join("/", lists:map(F,get_groups(Config)))]). + +get_groups(Config) -> + P = proplists:get_value(tc_group_path, Config, []) ++ + [proplists:get_value(tc_group_properties, Config, [])], + [Name || L <- P, + is_list(L), + {name,Name} <- L]. + diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 5125710471..36d3e16a18 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -1569,6 +1569,18 @@ config_file(Config) -> ct:log("ServerAlgs =~n~p~n~nOurAlgs =~n~p~n~nCommonAlgs =~n~p",[ServerAlgs,OurAlgs,CommonAlgs]), Nkex = length(proplists:get_value(kex, CommonAlgs, [])), + %% Adjust for very old ssh daemons that only supports ssh-rsa and ssh-dss: + AdjustClient = + case proplists:get_value(public_key,ServerAlgs,[]) -- ['ssh-rsa','ssh-dss'] of + [] -> + %% Old, let the client support them also: + ct:log("Adjust the client's public_key set", []), + [{public_key, ['ssh-rsa','ssh-dss']}]; + [_|_] -> + %% Ok, let the client be un-modified: + [] + end, + case {ServerAlgs, ssh_test_lib:some_empty(CommonAlgs)} of {[],_} -> {skip, "No server algorithms found"}; @@ -1586,7 +1598,7 @@ config_file(Config) -> [{ssh, [{preferred_algorithms, [{cipher, [Ch1]}, {kex, [K1a]} - ]}, + ] ++ AdjustClient}, {client_options, [{modify_algorithms, [{rm, [{kex, [K1a]}]}, diff --git a/lib/ssh/test/ssh_pubkey_SUITE.erl b/lib/ssh/test/ssh_pubkey_SUITE.erl index 481666d25e..885c9914a0 100644 --- a/lib/ssh/test/ssh_pubkey_SUITE.erl +++ b/lib/ssh/test/ssh_pubkey_SUITE.erl @@ -34,31 +34,34 @@ ]). -export([ + check_dsa_disabled/1, + check_rsa_sha1_disabled/1, connect_dsa_to_dsa/1, connect_dsa_to_ecdsa/1, connect_dsa_to_ed25519/1, connect_dsa_to_ed448/1, - connect_dsa_to_rsa/1, + connect_dsa_to_rsa_sha2/1, connect_ecdsa_to_dsa/1, connect_ecdsa_to_ecdsa/1, connect_ecdsa_to_ed25519/1, connect_ecdsa_to_ed448/1, - connect_ecdsa_to_rsa/1, + connect_ecdsa_to_rsa_sha2/1, connect_ed25519_to_dsa/1, connect_ed25519_to_ecdsa/1, connect_ed25519_to_ed25519/1, connect_ed25519_to_ed448/1, - connect_ed25519_to_rsa/1, + connect_ed25519_to_rsa_sha2/1, connect_ed448_to_dsa/1, connect_ed448_to_ecdsa/1, connect_ed448_to_ed25519/1, connect_ed448_to_ed448/1, - connect_ed448_to_rsa/1, - connect_rsa_to_dsa/1, - connect_rsa_to_ecdsa/1, - connect_rsa_to_ed25519/1, - connect_rsa_to_ed448/1, - connect_rsa_to_rsa/1 + connect_ed448_to_rsa_sha2/1, + connect_rsa_sha1_to_dsa/1, + connect_rsa_sha2_to_dsa/1, + connect_rsa_sha2_to_ecdsa/1, + connect_rsa_sha2_to_ed25519/1, + connect_rsa_sha2_to_ed448/1, + connect_rsa_sha2_to_rsa_sha2/1 ]). -include_lib("common_test/include/ct.hrl"). @@ -79,19 +82,19 @@ all() -> ]. --define(tests_old, [connect_rsa_to_rsa, - connect_rsa_to_dsa, - connect_rsa_to_ecdsa, - connect_dsa_to_rsa, +-define(tests_old, [connect_rsa_sha2_to_rsa_sha2, + connect_rsa_sha1_to_dsa, + connect_rsa_sha2_to_dsa, + connect_rsa_sha2_to_ecdsa, + connect_dsa_to_rsa_sha2, connect_dsa_to_dsa, connect_dsa_to_ecdsa, - connect_ecdsa_to_rsa, + connect_ecdsa_to_rsa_sha2, connect_ecdsa_to_dsa, connect_ecdsa_to_ecdsa ]). --define(tests_new, [ - connect_dsa_to_ed25519, +-define(tests_new, [connect_dsa_to_ed25519, connect_dsa_to_ed448, connect_ecdsa_to_ed25519, connect_ecdsa_to_ed448, @@ -99,20 +102,20 @@ all() -> connect_ed25519_to_ecdsa, connect_ed25519_to_ed448, connect_ed25519_to_ed25519, - connect_ed25519_to_rsa, + connect_ed25519_to_rsa_sha2, connect_ed448_to_dsa, connect_ed448_to_ecdsa, connect_ed448_to_ed25519, connect_ed448_to_ed448, - connect_ed448_to_rsa, - connect_rsa_to_ed25519, - connect_rsa_to_ed448 + connect_ed448_to_rsa_sha2, + connect_rsa_sha2_to_ed25519, + connect_rsa_sha2_to_ed448 | ?tests_old % but taken from the new format directory ]). groups() -> [{new_format, [], ?tests_new}, - {old_format, [], ?tests_old++[{group,passphrase}]}, + {old_format, [], [check_dsa_disabled, check_rsa_sha1_disabled | ?tests_old++[{group,passphrase}] ]}, {passphrase, [], ?tests_old}, {option_space,[], [{group,new_format}]} ]. @@ -180,18 +183,20 @@ end_per_group(_, Config) -> Config. %%%---------------------------------------------------------------- -init_per_testcase(connect_rsa_to_rsa, Config0) -> - setup_user_system_dir(rsa, rsa, Config0); -init_per_testcase(connect_rsa_to_dsa, Config0) -> - setup_user_system_dir(rsa, dsa, Config0); -init_per_testcase(connect_rsa_to_ecdsa, Config0) -> - setup_user_system_dir(rsa, ecdsa, Config0); -init_per_testcase(connect_rsa_to_ed25519, Config0) -> - setup_user_system_dir(rsa, ed25519, Config0); -init_per_testcase(connect_rsa_to_ed448, Config0) -> - setup_user_system_dir(rsa, ed448, Config0); -init_per_testcase(connect_dsa_to_rsa, Config0) -> - setup_user_system_dir(dsa, rsa, Config0); +init_per_testcase(connect_rsa_sha2_to_rsa_sha2, Config0) -> + setup_user_system_dir(rsa_sha2, rsa_sha2, Config0); +init_per_testcase(connect_rsa_sha1_to_dsa, Config0) -> + setup_user_system_dir(rsa_sha1, dsa, Config0); +init_per_testcase(connect_rsa_sha2_to_dsa, Config0) -> + setup_user_system_dir(rsa_sha2, dsa, Config0); +init_per_testcase(connect_rsa_sha2_to_ecdsa, Config0) -> + setup_user_system_dir(rsa_sha2, ecdsa, Config0); +init_per_testcase(connect_rsa_sha2_to_ed25519, Config0) -> + setup_user_system_dir(rsa_sha2, ed25519, Config0); +init_per_testcase(connect_rsa_sha2_to_ed448, Config0) -> + setup_user_system_dir(rsa_sha2, ed448, Config0); +init_per_testcase(connect_dsa_to_rsa_sha2, Config0) -> + setup_user_system_dir(dsa, rsa_sha2, Config0); init_per_testcase(connect_dsa_to_dsa, Config0) -> setup_user_system_dir(dsa, dsa, Config0); init_per_testcase(connect_dsa_to_ecdsa, Config0) -> @@ -200,8 +205,8 @@ init_per_testcase(connect_dsa_to_ed25519, Config0) -> setup_user_system_dir(dsa, ed25519, Config0); init_per_testcase(connect_dsa_to_ed448, Config0) -> setup_user_system_dir(dsa, ed448, Config0); -init_per_testcase(connect_ecdsa_to_rsa, Config0) -> - setup_user_system_dir(ecdsa, rsa, Config0); +init_per_testcase(connect_ecdsa_to_rsa_sha2, Config0) -> + setup_user_system_dir(ecdsa, rsa_sha2, Config0); init_per_testcase(connect_ecdsa_to_dsa, Config0) -> setup_user_system_dir(ecdsa, dsa, Config0); init_per_testcase(connect_ecdsa_to_ecdsa, Config0) -> @@ -210,8 +215,8 @@ init_per_testcase(connect_ecdsa_to_ed25519, Config0) -> setup_user_system_dir(ecdsa, ed25519, Config0); init_per_testcase(connect_ecdsa_to_ed448, Config0) -> setup_user_system_dir(ecdsa, ed448, Config0); -init_per_testcase(connect_ed25519_to_rsa, Config0) -> - setup_user_system_dir(ed25519, rsa, Config0); +init_per_testcase(connect_ed25519_to_rsa_sha2, Config0) -> + setup_user_system_dir(ed25519, rsa_sha2, Config0); init_per_testcase(connect_ed25519_to_dsa, Config0) -> setup_user_system_dir(ed25519, dsa, Config0); init_per_testcase(connect_ed25519_to_ecdsa, Config0) -> @@ -220,8 +225,8 @@ init_per_testcase(connect_ed25519_to_ed25519, Config0) -> setup_user_system_dir(ed25519, ed25519, Config0); init_per_testcase(connect_ed25519_to_ed448, Config0) -> setup_user_system_dir(ed25519, ed448, Config0); -init_per_testcase(connect_ed448_to_rsa, Config0) -> - setup_user_system_dir(ed448, rsa, Config0); +init_per_testcase(connect_ed448_to_rsa_sha2, Config0) -> + setup_user_system_dir(ed448, rsa_sha2, Config0); init_per_testcase(connect_ed448_to_dsa, Config0) -> setup_user_system_dir(ed448, dsa, Config0); init_per_testcase(connect_ed448_to_ecdsa, Config0) -> @@ -230,31 +235,41 @@ init_per_testcase(connect_ed448_to_ed25519, Config0) -> setup_user_system_dir(ed448, ed25519, Config0); init_per_testcase(connect_ed448_to_ed448, Config0) -> setup_user_system_dir(ed448, ed448, Config0); + +init_per_testcase(check_dsa_disabled, Config0) -> + setup_default_user_system_dir(dsa, Config0); +init_per_testcase(check_rsa_sha1_disabled, Config0) -> + setup_default_user_system_dir(rsa_sha1, Config0); + init_per_testcase(_, Config) -> Config. + end_per_testcase(_, Config) -> Config. %%%---------------------------------------------------------------- %%% Test Cases ---------------------------------------------------- %%%---------------------------------------------------------------- -connect_rsa_to_rsa(Config) -> +connect_rsa_sha2_to_rsa_sha2(Config) -> + try_connect(Config). + +connect_rsa_sha1_to_dsa(Config) -> try_connect(Config). -connect_rsa_to_dsa(Config) -> +connect_rsa_sha2_to_dsa(Config) -> try_connect(Config). -connect_rsa_to_ecdsa(Config) -> +connect_rsa_sha2_to_ecdsa(Config) -> try_connect(Config). -connect_rsa_to_ed25519(Config) -> +connect_rsa_sha2_to_ed25519(Config) -> try_connect(Config). -connect_rsa_to_ed448(Config) -> +connect_rsa_sha2_to_ed448(Config) -> try_connect(Config). -connect_dsa_to_rsa(Config) -> +connect_dsa_to_rsa_sha2(Config) -> try_connect(Config). connect_dsa_to_dsa(Config) -> @@ -269,7 +284,7 @@ connect_dsa_to_ed25519(Config) -> connect_dsa_to_ed448(Config) -> try_connect(Config). -connect_ecdsa_to_rsa(Config) -> +connect_ecdsa_to_rsa_sha2(Config) -> try_connect(Config). connect_ecdsa_to_dsa(Config) -> @@ -284,7 +299,7 @@ connect_ecdsa_to_ed25519(Config) -> connect_ecdsa_to_ed448(Config) -> try_connect(Config). -connect_ed25519_to_rsa(Config) -> +connect_ed25519_to_rsa_sha2(Config) -> try_connect(Config). connect_ed25519_to_dsa(Config) -> @@ -299,7 +314,7 @@ connect_ed25519_to_ed25519(Config) -> connect_ed25519_to_ed448(Config) -> try_connect(Config). -connect_ed448_to_rsa(Config) -> +connect_ed448_to_rsa_sha2(Config) -> try_connect(Config). connect_ed448_to_dsa(Config) -> @@ -314,16 +329,26 @@ connect_ed448_to_ed25519(Config) -> connect_ed448_to_ed448(Config) -> try_connect(Config). +%%%---------------------------------------------------------------- +check_dsa_disabled(Config) -> + try_connect_disabled(Config). + +check_rsa_sha1_disabled(Config) -> + try_connect_disabled(Config). + %%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- try_connect({skip,Reson}) -> {skip,Reson}; try_connect(Config) -> +%ssh_dbg:start(fun ct:pal/2), dbg:tp(ssh_transport,sign,3,x), dbg:tp(ssh_transport,verify,5,x), SystemDir = proplists:get_value(system_dir, Config), UserDir = proplists:get_value(user_dir, Config), ClientOpts = proplists:get_value(client_opts, Config, []), DaemonOpts = proplists:get_value(daemon_opts, Config, []), + ssh_dbg:start(fun ct:log/2), ssh_dbg:on([alg]), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir} | DaemonOpts]), @@ -333,46 +358,29 @@ try_connect(Config) -> {user_interaction, false} | ClientOpts]), ssh:close(C), + ssh_dbg:stop(), ssh:stop_daemon(Pid). + +try_connect_disabled(Config) -> + try try_connect(Config) + of _ -> {fail, "non-default algorithm accepted"} + catch error:{badmatch,{error,"Service not available"}} -> ok + end. + %%%---------------------------------------------------------------- %%% Local --------------------------------------------------------- %%%---------------------------------------------------------------- setup_user_system_dir(ClientAlg, ServerAlg, Config) -> - case supported(public_keys, ClientAlg) andalso supported(public_keys, ServerAlg) of + case supported(public_key, ClientAlg) andalso supported(public_key, ServerAlg) of true -> - PrivDir = proplists:get_value(priv_dir, Config), - KeySrcDir = proplists:get_value(key_src_dir, Config), - Fmt = proplists:get_value(fmt, Config), - - System = lists:concat(["system_", ClientAlg, "_", ServerAlg, "_", Fmt]), - SystemDir = filename:join(PrivDir, System), - file:make_dir(SystemDir), - - User = lists:concat(["user_", ClientAlg, "_", ServerAlg, "_", Fmt]), - UserDir = filename:join(PrivDir, User), - file:make_dir(UserDir), - - HostSrcFile = filename:join(KeySrcDir, file(host,ServerAlg)), - HostDstFile = filename:join(SystemDir, file(host,ServerAlg)), - - UserSrcFile = filename:join(KeySrcDir, file(user,ClientAlg)), - UserDstFile = filename:join(UserDir, file(user,ClientAlg)), - - UserPubSrcFile = filename:join(KeySrcDir, file(user,ClientAlg)++".pub"), - AuthorizedKeys = filename:join(UserDir, "authorized_keys"), - try - {ok,_} = file:copy(UserSrcFile, UserDstFile), - {ok,_} = file:copy(UserPubSrcFile, AuthorizedKeys), - {ok,_} = file:copy(HostSrcFile, HostDstFile) + setup_dirs(ClientAlg, ServerAlg, Config) of - _ -> - ModAlgs = [{modify_algorithms, - [{append,[{public_key, - lists:usort([alg(ClientAlg), - alg(ServerAlg)])}]}]} - ], + {ok, {SystemDir,UserDir}} -> + ModAlgs = [{preferred_algorithms, + [{public_key, lists:usort([alg(ClientAlg), alg(ServerAlg)])}] + }], [{system_dir,SystemDir}, {user_dir,UserDir} | extend_optsL([daemon_opts,client_opts], ModAlgs, Config)] @@ -386,38 +394,120 @@ setup_user_system_dir(ClientAlg, ServerAlg, Config) -> {skip, unsupported_algorithm} end. + +setup_default_user_system_dir(ClientAlg, Config) -> + ServerAlg = ecdsa, + case default(public_key, ClientAlg) of + false -> + case supported(public_key, ClientAlg) of + true -> + case supported(public_key, ServerAlg) of + true -> + try + setup_dirs(ClientAlg, ServerAlg, Config) + of + {ok, {SystemDir,UserDir}} -> + ModAlgs = [{modify_algorithms, + [{append,[{public_key,[alg(ServerAlg)]}]}, + {rm, [{public_key,[alg(ClientAlg)|inv_algs(ClientAlg)]}]} + ]}], + [{system_dir,SystemDir}, + {user_dir,UserDir} + | extend_optsL([daemon_opts,client_opts], ModAlgs, Config)] + catch + error:{badmatch,{error,enoent}}:S -> + ct:log("~p:~p Stack:~n~p", [?MODULE,?LINE,S]), + {skip, no_key_file_found} + end; + false -> + {skip, unsupported_server_algorithm} + end; + false -> + {skip, unsupported_client_algorithm} + end; + true -> + {fail, disabled_algorithm_present} + end. + + +setup_dirs(ClientAlg, ServerAlg, Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + KeySrcDir = proplists:get_value(key_src_dir, Config), + Fmt = proplists:get_value(fmt, Config), + + System = lists:concat(["system_", ClientAlg, "_", ServerAlg, "_", Fmt]), + SystemDir = filename:join(PrivDir, System), + file:make_dir(SystemDir), + + User = lists:concat(["user_", ClientAlg, "_", ServerAlg, "_", Fmt]), + UserDir = filename:join(PrivDir, User), + file:make_dir(UserDir), + + HostSrcFile = filename:join(KeySrcDir, file(src,host,ServerAlg)), + HostDstFile = filename:join(SystemDir, file(dst,host,ServerAlg)), + + UserSrcFile = filename:join(KeySrcDir, file(src,user,ClientAlg)), + UserDstFile = filename:join(UserDir, file(dst,user,ClientAlg)), + + UserPubSrcFile = filename:join(KeySrcDir, file(src,user,ClientAlg)++".pub"), + AuthorizedKeys = filename:join(UserDir, "authorized_keys"), + + ct:log("UserSrcFile = ~p~nUserDstFile = ~p", [UserSrcFile, UserDstFile]), + {ok,_} = file:copy(UserSrcFile, UserDstFile), + ct:log("UserPubSrcFile = ~p~nAuthorizedKeys = ~p", [UserPubSrcFile, AuthorizedKeys]), + {ok,_} = file:copy(UserPubSrcFile, AuthorizedKeys), + ct:log("HostSrcFile = ~p~nHostDstFile = ~p", [HostSrcFile, HostDstFile]), + {ok,_} = file:copy(HostSrcFile, HostDstFile), + + ct:log("SystemDir = ~p~nUserDir = ~p", [SystemDir,UserDir]), + {ok, {SystemDir,UserDir}}. + %%%---------------------------------------------------------------- -file(host, dsa) -> "ssh_host_dsa_key"; -file(host, ecdsa) -> "ssh_host_ecdsa_key"; -file(host, ed25519) -> "ssh_host_ed25519_key"; -file(host, ed448) -> "ssh_host_ed448_key"; -file(host, rsa) -> "ssh_host_rsa_key"; -file(user, dsa) -> "id_dsa"; -file(user, ecdsa) -> "id_ecdsa"; -file(user, ed25519) -> "id_ed25519"; -file(user, ed448) -> "id_ed448"; -file(user, rsa) -> "id_rsa". +file( _, host, dsa) -> "ssh_host_dsa_key"; +file( _, host, ecdsa) -> "ssh_host_ecdsa_key"; +file( _, host, ed25519) -> "ssh_host_ed25519_key"; +file( _, host, ed448) -> "ssh_host_ed448_key"; +file( _, host, rsa_sha2)-> "ssh_host_rsa_key"; +file(src, host, rsa_sha1)-> "ssh_host_rsa_key"; +file(dst, host, rsa_sha1)-> "ssh_host_rsa_key"; +file( _, user, dsa) -> "id_dsa"; +file( _, user, ecdsa) -> "id_ecdsa"; +file( _, user, ed25519) -> "id_ed25519"; +file( _, user, ed448) -> "id_ed448"; +file( _, user, rsa_sha2)-> "id_rsa"; +file(src, user, rsa_sha1)-> "id_rsa"; +file(dst, user, rsa_sha1)-> "id_rsa". alg(dsa) -> 'ssh-dss'; alg(ecdsa) -> 'ecdsa-sha2-nistp256'; alg(ed25519) -> 'ssh-ed25519'; alg(ed448) -> 'ssh-ed448'; -alg(rsa) -> 'ssh-rsa'. - - -supported(public_keys, rsa) -> supported(public_key, 'ssh-rsa') orelse - supported(public_key, 'rsa-sha2-256') orelse - supported(public_key, 'rsa-sha2-521'); -supported(public_keys, dsa) -> supported(public_key, 'ssh-dss'); -supported(public_keys, ecdsa) -> supported(public_key, 'ecdsa-sha2-nistp256') orelse - supported(public_key, 'ecdsa-sha2-nistp384') orelse - supported(public_key, 'ecdsa-sha2-nistp521'); -supported(public_keys, ed448) -> supported(public_key, 'ssh-ed448'); -supported(public_keys, ed25519) -> supported(public_key, 'ssh-ed25519'); -supported(Type, Alg) -> - case proplists:get_value(Type,ssh_transport:supported_algorithms()) of - undefined -> - lists:member(Alg, crypto:supports(Type)); - L -> - lists:member(Alg, L) - end. +alg(rsa_sha2)-> 'rsa-sha2-256'; +alg(rsa_sha1)-> 'ssh-rsa'. + +inv_algs(rsa_sha1) -> algs(rsa_sha2); +inv_algs(_) -> []. + +algs(dsa) -> ['ssh-dss']; +algs(ecdsa) -> ['ecdsa-sha2-nistp256', 'ecdsa-sha2-nistp384', 'ecdsa-sha2-521']; +algs(ed25519) -> ['ssh-ed25519']; +algs(ed448) -> ['ssh-ed448']; +algs(rsa_sha2)-> ['rsa-sha2-256', 'rsa-sha2-384', 'rsa-sha2-512']; +algs(rsa_sha1)-> ['ssh-rsa']; +algs(A) -> [A]. + + + +default(Type, Alg) -> listed(algs(Alg), ssh_transport:default_algorithms(Type)). + +supported(Type, Alg) -> listed(algs(Alg), + try + ssh_transport:supported_algorithms(Type) + catch + error:function_clause -> crypto:supports(Type) + end). + +listed(As, L) -> lists:any(fun(A) -> lists:member(A,L) end, + As). + + diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 4e74eeddde..2a4487a813 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.10.8 +SSH_VSN = 4.11.1 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 368f215f16..7ee4b308e3 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,93 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 10.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix CRL handling that previously could fail to find the + issuer cert under some circumstances.</p> + <p> + Own Id: OTP-17261 Aux Id: GH-4589 </p> + </item> + <item> + <p> + TLS-1.3 client could, under some circumstances, select an + incorrect algorithm to sign the certificate verification + message causing a TLS Decrypt Alert being issued by the + server.</p> + <p> + Own Id: OTP-17281 Aux Id: GH-4620 </p> + </item> + <item> + <p> + Correct handling of default values for emulated socket + options and retain the order of the ssl options list to + ensure backwards compatible behavior if options should be + set more than once.</p> + <p> + Own Id: OTP-17282</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Enhance pre TLS-1.3 session handling so the client and + server side handling is completely separated and client + disregards oldest session when reaching max limit of the + session table.</p> + <p> + Own Id: OTP-16876</p> + </item> + <item> + <p> + This change implements the early data feature for TLS 1.3 + clients.</p> + <p> + TLS 1.3 allows clients to send data in the first flight + using a Pre-Shared Key to authenticate the server and to + encrypt the early data.</p> + <p> + Own Id: OTP-16985</p> + </item> + <item> + <p> + This change implements the early data feature for TLS 1.3 + servers.</p> + <p> + Own Id: OTP-17042</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 10.2.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Backport of OTP-17282</p> + <p> + Correct handling of default values for emulated socket + options and retain the order of the ssl options list to + ensure backwards compatible behavior if options should be + set more than once.</p> + <p> + Own Id: OTP-17289 Aux Id: GH-4585 </p> + </item> + </list> + </section> + +</section> + <section><title>SSL 10.2.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index ebf085ebc9..0f09d873f9 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -88,15 +88,37 @@ </p></item> <tag><c><![CDATA[session_cb = atom() <optional>]]></c></tag> - <item><p>Name of the session cache callback module that implements + <item><p> Deprecated Since OTP-23.3 replaced by <c>client_session_cb</c> + and <c>server_session_cb</c> + </p></item> + + <tag><c><![CDATA[client_session_cb = atom() <optional>]]></c></tag> + <item><p> Since OTP-23.3 Name client of the session cache callback module that implements + the <c>ssl_session_cache_api</c> behavior. Defaults to + <c>ssl_client_session_cache_db</c>.</p></item> + + + <tag><c><![CDATA[server_session_cb = atom() <optional>]]></c></tag> + <item><p>Since OTP-23.3 Name of the server session cache callback module that implements the <c>ssl_session_cache_api</c> behavior. Defaults to - <c>ssl_session_cache</c>.</p></item> + <c>ssl_server_session_cache_db</c>.</p></item> <tag><c><![CDATA[session_cb_init_args = proplist:proplist() <optional>]]></c></tag> + <item><p>Deprecated Since OTP-23.3 replaced by <c>client_session_cb_init_args</c> + and <c>server_session_cb_init_args</c></p></item> + + <tag><c><![CDATA[client_session_cb_init_args = proplist:proplist() <optional>]]></c></tag> + + <item><p>List of extra user-defined arguments to the <c>init</c> function + in the session cache callback module. Defaults to <c>[]</c>.</p></item> + + <tag><c><![CDATA[server_session_cb_init_args = proplist:proplist() <optional>]]></c></tag> + <item><p>List of extra user-defined arguments to the <c>init</c> function in the session cache callback module. Defaults to <c>[]</c>.</p></item> + <tag><c><![CDATA[session_cache_client_max = integer() <optional>]]></c><br/></tag> <item><p>Limits the growth of the clients session cache, that is how many sessions towards servers that are cached to be used by diff --git a/lib/ssl/doc/src/ssl_crl_cache_api.xml b/lib/ssl/doc/src/ssl_crl_cache_api.xml index 3f127676ce..69f1b3f4ae 100644 --- a/lib/ssl/doc/src/ssl_crl_cache_api.xml +++ b/lib/ssl/doc/src/ssl_crl_cache_api.xml @@ -77,8 +77,8 @@ <funcs> <func> - <name since="@maint@">fresh_crl(DistributionPoint, CRL) -> FreshCRL </name> - <name since="OTP 18.0">fresh_crl(DistributionPoint, CRL) -> FreshCRL | {LoggerInfo, FreshCRL}</name> + <name since="OTP 22.2">Module:fresh_crl(DistributionPoint, CRL) -> FreshCRL </name> + <name since="OTP 18.0">Module:fresh_crl(DistributionPoint, CRL) -> FreshCRL | {LoggerInfo, FreshCRL}</name> <fsummary> <c>fun fresh_crl/2 </c> will be used as input option <c>update_crl</c> to public_key:pkix_crls_validate/3 </fsummary> <type> @@ -100,10 +100,10 @@ </func> <func> - <name since="@maint@">lookup(DistributionPoint, Issuer, DbHandle) -> not_available | CRLs | + <name since="OTP 22.2">Module:lookup(DistributionPoint, Issuer, DbHandle) -> not_available | CRLs | {LoggerInfo, CRLs} </name> - <name since="OTP 19.0">lookup(DistributionPoint, Issuer, DbHandle) -> not_available | CRLs </name> - <name since="OTP 18.0">lookup(DistributionPoint, DbHandle) -> not_available | CRLs </name> + <name since="OTP 19.0">Module:lookup(DistributionPoint, Issuer, DbHandle) -> not_available | CRLs </name> + <name since="OTP 18.0">Module:lookup(DistributionPoint, DbHandle) -> not_available | CRLs </name> <fsummary> </fsummary> <type> <v> DistributionPoint = <seetype marker="#dist_point"> dist_point() </seetype> </v> @@ -137,8 +137,8 @@ </func> <func> - <name since="@maint@">select(Issuer, DbHandle) -> CRLs | {LoggerInfo, CRLs} </name> - <name since="OTP 18.0">select(Issuer, DbHandle) -> CRLs </name> + <name since="OTP 22.2">Module:select(Issuer, DbHandle) -> CRLs | {LoggerInfo, CRLs} </name> + <name since="OTP 18.0">Module:select(Issuer, DbHandle) -> CRLs </name> <fsummary>Select the CRLs in the cache that are issued by <c>Issuer</c></fsummary> <type> <v> Issuer = <seetype diff --git a/lib/ssl/doc/src/ssl_session_cache_api.xml b/lib/ssl/doc/src/ssl_session_cache_api.xml index 86fdc34fd8..3067d89b13 100644 --- a/lib/ssl/doc/src/ssl_session_cache_api.xml +++ b/lib/ssl/doc/src/ssl_session_cache_api.xml @@ -33,7 +33,7 @@ <description> <p> - Defines the API for the TLS session cache so + Defines the API for the TLS session cache (pre TLS-1.3) so that the data storage scheme can be replaced by defining a new callback module implementing this API. </p> @@ -77,7 +77,7 @@ <funcs> <func> - <name since="OTP R14B">delete(Cache, Key) -> _</name> + <name since="OTP R14B">Module:delete(Cache, Key) -> _</name> <fsummary>Deletes a cache entry.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> @@ -91,7 +91,7 @@ </func> <func> - <name since="OTP R14B">foldl(Fun, Acc0, Cache) -> Acc</name> + <name since="OTP R14B">Module:foldl(Fun, Acc0, Cache) -> Acc</name> <fsummary></fsummary> <type> <v>Fun = fun()</v> @@ -105,11 +105,16 @@ The function returns the final value of the accumulator. <c>Acc0</c> is returned if the cache is empty. </p> + + <note><p>Since OTP-23.3 this functions is only used on the client side + and does not need to implemented for a server cache. + </p></note> + </desc> </func> <func> - <name since="OTP 18.0">init(Args) -> Cache </name> + <name since="OTP 18.0">Module:init(Args) -> Cache </name> <fsummary>Returns cache reference.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> @@ -135,7 +140,7 @@ </func> <func> - <name since="OTP R14B">lookup(Cache, Key) -> Entry</name> + <name since="OTP R14B">Module:lookup(Cache, Key) -> Entry</name> <fsummary>Looks up a cache entry.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> @@ -150,7 +155,7 @@ </func> <func> - <name since="OTP R14B">select_session(Cache, PartialKey) -> [Session]</name> + <name since="OTP R14B">Module:select_session(Cache, PartialKey) -> [Session]</name> <fsummary>Selects sessions that can be reused.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> @@ -158,14 +163,17 @@ <v>Session = <seetype marker="#session">session()</seetype></v> </type> <desc> - <p>Selects sessions that can be reused. Is to be callable - from any process. - </p> + <p>Selects sessions that can be reused, that is sessions that + include <c>PartialKey</c> in its key. Is to be callable from + any process.</p> + <note><p>Since OTP-23.3 This functions is only used on the client side + and does not need to implemented for a server cache. + </p></note> </desc> </func> <func> - <name since="OTP 19.3">size(Cache) -> integer()</name> + <name since="OTP 19.3">Module:size(Cache) -> integer()</name> <fsummary>Returns the number of sessions in the cache.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> @@ -180,7 +188,7 @@ </func> <func> - <name since="OTP R14B">terminate(Cache) -> _</name> + <name since="OTP R14B">Module:terminate(Cache) -> _</name> <fsummary>Called by the process that handles the cache when it is about to terminate.</fsummary> <type> @@ -195,7 +203,7 @@ </func> <func> - <name since="OTP R14B">update(Cache, Key, Session) -> _</name> + <name since="OTP R14B">Module:update(Cache, Key, Session) -> _</name> <fsummary>Caches a new session or updates an already cached one.</fsummary> <type> <v>Cache = <seetype marker="#session_cache_ref"> session_cache_ref() </seetype></v> diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile index 1a55ee7b83..5edd6cb4b9 100644 --- a/lib/ssl/src/Makefile +++ b/lib/ssl/src/Makefile @@ -65,6 +65,7 @@ MODULES= \ ssl_certificate \ ssl_cipher \ ssl_cipher_format \ + ssl_client_session_cache_db \ ssl_config \ ssl_connection_sup \ ssl_crl \ @@ -87,7 +88,6 @@ MODULES= \ ssl_server_session_cache_sup \ ssl_upgrade_server_session_cache_sup \ ssl_session \ - ssl_session_cache \ ssl_srp_primes \ ssl_sup \ tls_bloom_filter \ diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index fb389dcb4d..78348826e4 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -583,19 +583,8 @@ initial_state(Role, Host, Port, Socket, {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #{beast_mitigation := BeastMitigation} = SSLOptions, ConnectionStates = dtls_record:init_connection_states(Role, BeastMitigation), - - SessionCacheCb = case application:get_env(ssl, session_cb) of - {ok, Cb} when is_atom(Cb) -> - Cb; - _ -> - ssl_session_cache - end, - InternalActiveN = case application:get_env(ssl, internal_active_n) of - {ok, N} when is_integer(N) -> - N; - _ -> - ?INTERNAL_ACTIVE_N - end, + #{session_cb := SessionCacheCb} = ssl_config:pre_1_3_session_opts(Role), + InternalActiveN = ssl_config:get_internal_active_n(), Monitor = erlang:monitor(process, User), InitStatEnv = #static_env{ role = Role, diff --git a/lib/ssl/src/dtls_server_session_cache_sup.erl b/lib/ssl/src/dtls_server_session_cache_sup.erl index 457eb90167..65fbb34918 100644 --- a/lib/ssl/src/dtls_server_session_cache_sup.erl +++ b/lib/ssl/src/dtls_server_session_cache_sup.erl @@ -42,7 +42,7 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). start_child(Listener) -> - supervisor:start_child(?MODULE, [Listener | ssl_config:pre_1_3_session_opts()]). + supervisor:start_child(?MODULE, [Listener | [ssl_config:pre_1_3_session_opts(server)]]). %%%========================================================================= %%% Supervisor callback diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 200d10a1b6..9f43bb5500 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -58,7 +58,7 @@ tls_dist_server_sup, %% SSL/TLS session and cert handling ssl_session, - ssl_session_cache, + ssl_client_session_cache_db, ssl_server_session_cache, ssl_server_session_cache_db, ssl_server_session_cache_sup, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index b32daf5d6e..d15345ac57 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -576,9 +576,7 @@ connect(Socket, SslOptions0, Timeout) when is_list(SslOptions0) andalso CbInfo = handle_option_cb_info(SslOptions0, tls), Transport = element(1, CbInfo), - EmulatedOptions = tls_socket:emulated_options(), - {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), - try handle_options(SslOptions0 ++ SocketValues, client) of + try handle_options(Transport, Socket, SslOptions0, client, undefined) of {ok, Config} -> tls_socket:upgrade(Socket, Config, Timeout) catch @@ -736,10 +734,8 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) handshake(Socket, SslOptions, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> CbInfo = handle_option_cb_info(SslOptions, tls), Transport = element(1, CbInfo), - EmulatedOptions = tls_socket:emulated_options(), - {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), ConnetionCb = connection_cb(SslOptions), - try handle_options(SslOptions ++ SocketValues, server) of + try handle_options(Transport, Socket, SslOptions, server, undefined) of {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} -> ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), {ok, Port} = tls_socket:port(Transport, Socket), @@ -1499,24 +1495,24 @@ do_listen(Port, #config{transport_info = {Transport, _, _, _,_}} = Config, tls_g do_listen(Port, Config, dtls_gen_connection) -> dtls_socket:listen(Port, Config). - - -spec handle_options([any()], client | server) -> {ok, #config{}}; ([any()], ssl_options()) -> ssl_options(). handle_options(Opts, Role) -> - handle_options(Opts, Role, undefined). + handle_options(undefined, undefined, Opts, Role, undefined). +handle_options(Opts, Role, InheritedSslOpts) -> + handle_options(undefined, undefined, Opts, Role, InheritedSslOpts). %% Handle ssl options at handshake, handshake_continue -handle_options(Opts0, Role, InheritedSslOpts) when is_map(InheritedSslOpts) -> +handle_options(_, _, Opts0, Role, InheritedSslOpts) when is_map(InheritedSslOpts) -> {SslOpts, _} = expand_options(Opts0, ?RULES), process_options(SslOpts, InheritedSslOpts, #{role => Role, rules => ?RULES}); %% Handle all options in listen, connect and handshake -handle_options(Opts0, Role, Host) -> - {SslOpts0, SockOpts} = expand_options(Opts0, ?RULES), - +handle_options(Transport, Socket, Opts0, Role, Host) -> + {SslOpts0, SockOpts0} = expand_options(Opts0, ?RULES), + %% Ensure all options are evaluated at startup SslOpts1 = add_missing_options(SslOpts0, ?RULES), SslOpts = #{protocol := Protocol} @@ -1527,7 +1523,7 @@ handle_options(Opts0, Role, Host) -> rules => ?RULES}), %% Handle special options - {Sock, Emulated} = emulated_options(Protocol, SockOpts), + {Sock, Emulated} = emulated_options(Transport, Socket, Protocol, SockOpts0), ConnetionCb = connection_cb(Protocol), CbInfo = handle_option_cb_info(Opts0, Protocol), @@ -1955,8 +1951,9 @@ expand_options(Opts0, Rules) -> cb_info, client_preferred_next_protocols, %% next_protocol_selector log_alert]), %% obsoleted by log_level - - SslOpts = {Opts -- SockOpts, [], length(Opts -- SockOpts)}, + + SslOpts0 = Opts -- SockOpts, + SslOpts = {SslOpts0, [], length(SslOpts0)}, {SslOpts, SockOpts}. @@ -2541,13 +2538,18 @@ ca_cert_default(verify_peer, {Fun,_}, _) when is_function(Fun) -> %% some trusted certs. ca_cert_default(verify_peer, undefined, _) -> "". -emulated_options(Protocol, Opts) -> +emulated_options(undefined, undefined, Protocol, Opts) -> case Protocol of tls -> tls_socket:emulated_options(Opts); dtls -> dtls_socket:emulated_options(Opts) - end. + end; +emulated_options(Transport, Socket, Protocol, Opts) -> + EmulatedOptions = tls_socket:emulated_options(), + {ok, Original} = tls_socket:getopts(Transport, Socket, EmulatedOptions), + {Inet, Emulated0} = emulated_options(undefined, undefined, Protocol, Opts), + {Inet, lists:ukeymerge(1, Emulated0, Original)}. handle_cipher_option(Value, Versions) when is_list(Value) -> try binary_cipher_suites(Versions, Value) of @@ -2720,7 +2722,7 @@ binary_filename(FileName) -> %% Assert that basic options are on the format {Key, Value} %% with a few exceptions and phase out log_alert handle_option_format([], Acc) -> - Acc; + lists:reverse(Acc); handle_option_format([{log_alert, Bool} | Rest], Acc) when is_boolean(Bool) -> case proplists:get_value(log_level, Acc ++ Rest, undefined) of undefined -> diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index 674870dd15..0b4211e1c3 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -75,7 +75,7 @@ trusted_cert_and_paths(Chain, CertDbHandle, CertDbRef, PartialChainHandler) -> %% If the chain contains extraneous certificates there could be %% more than one possible path such chains might be used to phase out %% an old certificate. - Paths = paths(Chain, CertDbHandle, CertDbRef), + Paths = paths(Chain, CertDbHandle), lists:map(fun(Path) -> case handle_partial_chain(Path, PartialChainHandler, CertDbHandle, CertDbRef) of {unknown_ca, _} = Result -> @@ -86,8 +86,7 @@ trusted_cert_and_paths(Chain, CertDbHandle, CertDbRef, PartialChainHandler) -> Result -> Result end - end, Paths). - + end, Paths). %%-------------------------------------------------------------------- -spec certificate_chain(undefined | binary() | #'OTPCertificate'{} , db_handle(), certdb_ref() | {extracted, list()}) -> {error, no_cert} | {ok, der_cert() | undefined, [der_cert()]}. @@ -439,38 +438,38 @@ pre_1_3_hash(sha1) -> pre_1_3_hash(Hash) -> Hash. -paths(Chain, CertDbHandle, CertDbRef) -> - paths(Chain, Chain, CertDbHandle, CertDbRef, []). +paths(Chain, CertDbHandle) -> + paths(Chain, Chain, CertDbHandle, []). -paths([Root], _, _, _, Path) -> - [[Root | Path]]; -paths([Cert1, Cert2 | Rest], Chain, CertDbHandle, CertDbRef, Path) -> +paths([Root], _, _, Path) -> + [[Root | Path]]; +paths([Cert1, Cert2 | Rest], Chain, CertDbHandle, Path) -> case public_key:pkix_is_issuer(Cert1, Cert2) of true -> %% Chain orded so far - paths([Cert2 | Rest], Chain, CertDbHandle, CertDbRef, [Cert1 | Path]); + paths([Cert2 | Rest], Chain, CertDbHandle, [Cert1 | Path]); false -> %% Chain is unorded and/or contains extraneous certificates - unorded_or_extraneous(Chain, CertDbHandle, CertDbRef) + unorded_or_extraneous(Chain, CertDbHandle) end. - -unorded_or_extraneous([Peer | FalseChain], CertDbHandle, CertDbRef) -> - ChainCandidates = extraneous_chains(FalseChain), - lists:map(fun(Candidate) -> - path_candidate(Peer, Candidate, CertDbHandle, CertDbRef) - end, + +unorded_or_extraneous([Peer | UnorderedChain], CertDbHandle) -> + ChainCandidates = extraneous_chains(UnorderedChain), + lists:map(fun(Candidate) -> + path_candidate(Peer, Candidate, CertDbHandle) + end, ChainCandidates). -path_candidate(Peer, ChainCandidateCAs, CertDbHandle, _CertDbRef) -> +path_candidate(Peer, ChainCandidateCAs, CertDbHandle) -> {ok, ExtractedCerts} = ssl_pkix_db:extract_trusted_certs({der, ChainCandidateCAs}), - %% certificate_chain/4 will make sure the chain is ordered - case certificate_chain(Peer, CertDbHandle, ExtractedCerts, []) of + %% certificate_chain/4 will make sure the chain is ordered + case certificate_chain(Peer, CertDbHandle, ExtractedCerts, []) of {ok, undefined, Chain} -> lists:reverse(Chain); {ok, Root, Chain} -> [Root | lists:reverse(Chain)] - end. - + end. + handle_partial_chain([IssuerCert| Rest] = Path, PartialChainHandler, CertDbHandle, CertDbRef) -> case public_key:pkix_is_self_signed(IssuerCert) of true -> %% IssuerCert = ROOT (That is ROOT was included in chain) @@ -533,7 +532,12 @@ handle_incomplete_chain([PeerCert| _] = Chain0, PartialChainHandler, Default, Ce %% See if we have the certificates to rebuild it. case certificate_chain(PeerCert, CertDbHandle, CertDbRef) of {ok, _, [PeerCert | _] = Chain} when Chain =/= Chain0 -> %% Chain candidate found - handle_partial_chain(lists:reverse(Chain), PartialChainHandler, CertDbHandle, CertDbRef); + case lists:prefix(Chain0, Chain) of + true -> + handle_partial_chain(lists:reverse(Chain), PartialChainHandler, CertDbHandle, CertDbRef); + false -> + Default + end; _ -> Default end. @@ -546,8 +550,8 @@ extraneous_chains(Certs) -> Subjects = [{subject(Cert), Cert} || Cert <- Certs], Duplicates = find_duplicates(Subjects), %% Number of certs with duplicates (same subject) has been limited - %% to two and the maximum number of combinations is limited to 4. - build_candidates(Duplicates, 2, 4). + %% to 4 and the maximum number of combinations is limited to 16. + build_candidates(Duplicates, 4, 16). build_candidates(Map, Duplicates, Combinations) -> Subjects = maps:keys(Map), @@ -568,7 +572,7 @@ build_candidates([H|T], Map, Duplicates, Combinations, Max, Acc0) -> Acc = [[Cert|L] || Cert <- Certs, L <- Acc0], build_candidates(T, Map, Duplicates - 1, Combinations * Counter, Max, Acc) end; - {[Cert|_], _} -> + {[Cert|_Throw], _Counter} -> case Acc0 of [] -> Acc = [[Cert]], diff --git a/lib/ssl/src/ssl_session_cache.erl b/lib/ssl/src/ssl_client_session_cache_db.erl index b2b33b9af3..d344294231 100644 --- a/lib/ssl/src/ssl_session_cache.erl +++ b/lib/ssl/src/ssl_client_session_cache_db.erl @@ -19,24 +19,30 @@ %% %% --module(ssl_session_cache). +-module(ssl_client_session_cache_db). -behaviour(ssl_session_cache_api). -include("ssl_handshake.hrl"). -include("ssl_internal.hrl"). --export([init/1, terminate/1, lookup/2, update/3, delete/2, foldl/3, - select_session/2, size/1, take_oldest/1]). +-export([init/1, + terminate/1, + lookup/2, + update/3, + delete/2, + foldl/3, + select_session/2, + size/1]). %%-------------------------------------------------------------------- -%% Description: Return table reference. Called by ssl_manager process. +%% Description: Return table reference. Called by ssl_manager process. %%-------------------------------------------------------------------- init(Options) -> ets:new(cache_name(proplists:get_value(role, Options)), [ordered_set, protected]). %%-------------------------------------------------------------------- -%% Description: Handles cache table at termination of ssl manager. +%% Description: Handles cache table at termination of ssl manager. %%-------------------------------------------------------------------- terminate(Cache) -> ets:delete(Cache). @@ -85,17 +91,17 @@ foldl(Fun, Acc0, Cache) -> _:_ -> Acc0 end. - + %%-------------------------------------------------------------------- %% Description: Selects a session that could be reused. Should be callable %% from any process. %%-------------------------------------------------------------------- -select_session(Cache, PartialKey) -> - try ets:select(Cache, +select_session(Cache, PartialKey) -> + try ets:select(Cache, [{{{PartialKey,'_'}, '$1'},[],['$1']}]) of Result -> Result - catch + catch _:_ -> [] end. @@ -107,14 +113,6 @@ size(Cache) -> ets:info(Cache, size). %%-------------------------------------------------------------------- -%% Description: Returns the oldest entry -%%-------------------------------------------------------------------- -take_oldest(Cache) -> - {Key, Oldest} = ets:first(Cache), - delete(Cache, Key), - {Oldest, Cache}. - -%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- cache_name(Name) -> diff --git a/lib/ssl/src/ssl_config.erl b/lib/ssl/src/ssl_config.erl index e03f117c82..520979f368 100644 --- a/lib/ssl/src/ssl_config.erl +++ b/lib/ssl/src/ssl_config.erl @@ -29,10 +29,12 @@ -define(DEFAULT_MAX_SESSION_CACHE, 1000). -export([init/2, - pre_1_3_session_opts/0, + pre_1_3_session_opts/1, get_max_early_data_size/0, get_ticket_lifetime/0, - get_ticket_store_size/0 + get_ticket_store_size/0, + get_internal_active_n/0, + get_internal_active_n/1 ]). %%==================================================================== @@ -54,20 +56,13 @@ init(#{erl_dist := ErlDist, DHParams = init_diffie_hellman(PemCache, DH, DHFile, Role), {ok, Config#{private_key => PrivateKey, dh_params => DHParams}}. -pre_1_3_session_opts() -> - CbOpts = case application:get_env(ssl, session_cb) of - {ok, Cb} when is_atom(Cb) -> - InitArgs = session_cb_init_args(), - #{session_cb => Cb, - session_cb_init_args => InitArgs}; - _ -> - #{session_cb => ssl_server_session_cache_db, - session_cb_init_args => []} - end, - LifeTime = session_lifetime(), - Max = max_session_cache_size(), - [CbOpts#{lifetime => LifeTime, max => Max}]. - +pre_1_3_session_opts(Role) -> + {Cb, InitArgs} = session_cb_opts(Role), + CbOpts = #{session_cb => Cb, + session_cb_init_args => InitArgs}, + LifeTime = session_lifetime(Role), + Max = max_session_cache_size(Role), + CbOpts#{lifetime => LifeTime, max => Max}. get_ticket_lifetime() -> case application:get_env(ssl, server_session_ticket_lifetime) of @@ -78,7 +73,6 @@ get_ticket_lifetime() -> 7200 %% Default 2 hours end. - get_ticket_store_size() -> case application:get_env(ssl, server_session_ticket_store_size) of {ok, Size} when is_integer(Size) -> @@ -95,6 +89,26 @@ get_max_early_data_size() -> ?DEFAULT_MAX_EARLY_DATA_SIZE end. +get_internal_active_n() -> + get_internal_active_n(false). + +get_internal_active_n(true) -> + %% Start with a random number between 1 and ?INTERNAL_ACTIVE_N + %% In most cases distribution connections are established all at + %% the same time, and flow control engages with ?INTERNAL_ACTIVE_N for + %% all connections. Which creates a wave of "passive" messages, leading + %% to significant bump of memory & scheduler utilisation. Starting with + %% a random number between 1 and ?INTERNAL_ACTIVE_N helps to spread the + %% spike. + erlang:system_time() rem ?INTERNAL_ACTIVE_N + 1; +get_internal_active_n(false) -> + case application:get_env(ssl, internal_active_n) of + {ok, N} when is_integer(N) -> + N; + _ -> + ?INTERNAL_ACTIVE_N + end. + %%==================================================================== %% Internal functions %%==================================================================== @@ -232,15 +246,32 @@ init_diffie_hellman(DbHandle,_, DHParamFile, server) -> end. -session_cb_init_args() -> - case application:get_env(ssl, session_cb_init_args) of - {ok, Args} when is_list(Args) -> - Args; - _ -> - [] +session_cb_init_args(client) -> + case application:get_env(ssl, client_session_cb_init_args) of + undefined -> + case application:get_env(ssl, session_cb_init_args) of + {ok, Args} when is_list(Args) -> + Args; + _ -> + [] + end; + {ok, Args} -> + Args + end; +session_cb_init_args(server) -> + case application:get_env(ssl, server_session_cb_init_args) of + undefined -> + case application:get_env(ssl, session_cb_init_args) of + {ok, Args} when is_list(Args) -> + Args; + _ -> + [] + end; + {ok, Args} -> + Args end. -session_lifetime() -> +session_lifetime(_Role) -> case application:get_env(ssl, session_lifetime) of {ok, Time} when is_integer(Time) -> Time; @@ -248,7 +279,14 @@ session_lifetime() -> ?'24H_in_sec' end. -max_session_cache_size() -> +max_session_cache_size(client) -> + case application:get_env(ssl, session_cache_client_max) of + {ok, Size} when is_integer(Size) -> + Size; + _ -> + ?DEFAULT_MAX_SESSION_CACHE + end; +max_session_cache_size(server) -> case application:get_env(ssl, session_cache_server_max) of {ok, Size} when is_integer(Size) -> Size; @@ -256,3 +294,17 @@ max_session_cache_size() -> ?DEFAULT_MAX_SESSION_CACHE end. +session_cb_opts(client = Role)-> + case application:get_env(ssl, session_cb, ssl_client_session_cache_db) of + ssl_client_session_cache_db = ClientCb -> + {ClientCb, []}; + ClientCb -> + {ClientCb, session_cb_init_args(Role)} + end; +session_cb_opts(server = Role) -> + case application:get_env(ssl, session_cb, ssl_server_session_cache_db) of + ssl_server_session_cache_db = ServerCb -> + {ServerCb, []}; + ServerCb -> + {ServerCb, session_cb_init_args(Role)} + end. diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index 790a115983..3f06eaa095 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -38,19 +38,20 @@ -define(ECDSA, 3). -record(session, { - session_id, - peer_certificate, - own_certificates, - compression_method, - cipher_suite, - master_secret, - srp_username, - is_resumable, - time_stamp, - ecc, %% TLS 1.3 Group - sign_alg, %% TLS 1.3 Signature Algorithm - dh_public_value %% TLS 1.3 DH Public Value from peer - }). + session_id, + internal_id, + peer_certificate, + own_certificates, + compression_method, + cipher_suite, + master_secret, + srp_username, + is_resumable, + time_stamp, + ecc, %% TLS 1.3 Group + sign_alg, %% TLS 1.3 Signature Algorithm + dh_public_value %% TLS 1.3 DH Public Value from peer + }). -define(NUM_OF_SESSION_ID_BYTES, 32). % TSL 1.1 & SSL 3 -define(NUM_OF_PREMASTERSECRET_BYTES, 48). diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index 4f6f12163f..9d0dbc2d9b 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -48,13 +48,14 @@ -record(state, { session_cache_client :: db_handle(), - session_cache_cb :: atom(), + session_cache_client_cb :: atom(), session_lifetime :: integer(), certificate_db :: db_handle(), session_validation_timer :: reference(), session_cache_client_max :: integer(), session_client_invalidator :: undefined | pid(), - options :: list() + options :: list(), + client_session_order :: gb_trees:tree() }). -define(GEN_UNIQUE_ID_MAX_TRIES, 10). @@ -219,25 +220,31 @@ init([ManagerName, PemCacheName, Opts]) -> put(ssl_manager, ManagerName), put(ssl_pem_cache, PemCacheName), process_flag(trap_exit, true), - CacheCb = proplists:get_value(session_cb, Opts, ssl_session_cache), + + #{session_cb := DefaultCacheCb, + session_cb_init_args := DefaultCacheCbInitArgs, + lifetime := DefaultSessLifeTime, + max := ClientSessMax + } = ssl_config:pre_1_3_session_opts(client), + CacheCb = proplists:get_value(session_cb, Opts, DefaultCacheCb), SessionLifeTime = - proplists:get_value(session_lifetime, Opts, ?'24H_in_sec'), - CertDb = ssl_pkix_db:create(PemCacheName), + proplists:get_value(session_lifetime, Opts, DefaultSessLifeTime), ClientSessionCache = CacheCb:init([{role, client} | - proplists:get_value(session_cb_init_args, Opts, [])]), + proplists:get_value(session_cb_init_args, Opts, DefaultCacheCbInitArgs)]), + CertDb = ssl_pkix_db:create(PemCacheName), Timer = erlang:send_after(SessionLifeTime * 1000 + 5000, self(), validate_sessions), {ok, #state{certificate_db = CertDb, session_cache_client = ClientSessionCache, - session_cache_cb = CacheCb, + session_cache_client_cb = CacheCb, session_lifetime = SessionLifeTime, session_validation_timer = Timer, - session_cache_client_max = - max_session_cache_size(session_cache_client_max), + session_cache_client_max = ClientSessMax, session_client_invalidator = undefined, - options = Opts + options = Opts, + client_session_order = gb_trees:empty() }}. %%-------------------------------------------------------------------- @@ -306,7 +313,7 @@ handle_cast({register_session, Host, Port, Session, true}, State0) -> handle_cast({invalidate_session, Host, Port, #session{session_id = ID} = Session}, #state{session_cache_client = Cache, - session_cache_cb = CacheCb} = State) -> + session_cache_client_cb = CacheCb} = State) -> invalidate_session(Cache, CacheCb, {{Host, Port}, ID}, Session, State); handle_cast({insert_crls, Path, CRLs}, #state{certificate_db = Db} = State) -> @@ -326,7 +333,7 @@ handle_cast({delete_crls, CRLsOrPath}, %% %% Description: Handling all non call/cast messages %%------------------------------------------------------------------- -handle_info(validate_sessions, #state{session_cache_cb = CacheCb, +handle_info(validate_sessions, #state{session_cache_client_cb = CacheCb, session_cache_client = ClientCache, session_lifetime = LifeTime, session_client_invalidator = Client @@ -362,7 +369,7 @@ handle_info(_Info, State) -> %%-------------------------------------------------------------------- terminate(_Reason, #state{certificate_db = Db, session_cache_client = ClientSessionCache, - session_cache_cb = CacheCb, + session_cache_client_cb = CacheCb, session_validation_timer = Timer}) -> erlang:cancel_timer(Timer), ssl_pkix_db:remove(Db), @@ -420,21 +427,15 @@ session_validation({{Port, _}, Session}, LifeTime) -> validate_session(Port, Session, LifeTime), LifeTime. -max_session_cache_size(CacheType) -> - case application:get_env(ssl, CacheType) of - {ok, Size} when is_integer(Size) -> - Size; - _ -> - ?DEFAULT_MAX_SESSION_CACHE - end. - -invalidate_session(Cache, CacheCb, Key, _Session, State) -> +invalidate_session(Cache, CacheCb, Key, _Session, + #state{client_session_order = Order} = State) -> case CacheCb:lookup(Cache, Key) of undefined -> %% Session is already invalidated {noreply, State}; - #session{} -> + #session{internal_id = InternalId} -> CacheCb:delete(Cache, Key), - {noreply, State} + {noreply, State#state{session_cache_client = Cache, + client_session_order = gb_trees:delete(InternalId, Order)}} end. clean_cert_db(Ref, CertDb, RefDb, FileMapDb, File) -> @@ -448,49 +449,61 @@ clean_cert_db(Ref, CertDb, RefDb, FileMapDb, File) -> end. client_register_unique_session(Host, Port, Session, #state{session_cache_client = Cache0, - session_cache_cb = CacheCb, + session_cache_client_cb = CacheCb, session_cache_client_max = Max, - options = Options} = State) -> + options = Options, + client_session_order = Order0} = State) -> TimeStamp = erlang:monotonic_time(), NewSession = Session#session{time_stamp = TimeStamp}, case CacheCb:select_session(Cache0, {Host, Port}) of no_session -> - Cache = do_register_session({{Host, Port}, - NewSession#session.session_id}, - NewSession, Max, Cache0, CacheCb, Options), - State#state{session_cache_client = Cache}; + {Cache, Order} = do_register_session({{Host, Port}, + NewSession#session.session_id}, + NewSession, Max, Cache0, CacheCb, Options, Order0), + State#state{session_cache_client = Cache, client_session_order = Order}; Sessions -> register_unique_session(Sessions, NewSession, {Host, Port}, State) end. client_register_session(Host, Port, Session, #state{session_cache_client = Cache0, - session_cache_cb = CacheCb, + session_cache_client_cb = CacheCb, session_cache_client_max = Max, - options = Options} = State) -> + options = Options, + client_session_order = Order0} = State) -> TimeStamp = erlang:monotonic_time(), NewSession = Session#session{time_stamp = TimeStamp}, - Cache = do_register_session({{Host, Port}, - NewSession#session.session_id}, - NewSession, Max, Cache0, CacheCb, Options), - State#state{session_cache_client = Cache}. - -do_register_session(Key, Session, Max, Cache0, CacheCb, Options) -> + SessionId = NewSession#session.session_id, + {Cache, Order} = do_register_session({{Host, Port}, SessionId}, + NewSession, Max, Cache0, CacheCb, Options, Order0), + State#state{session_cache_client = Cache, + client_session_order = Order}. + +do_register_session(Key, #session{time_stamp = TimeStamp} = Session0, + Max, Cache, CacheCb, Options, Order0) -> try - case CacheCb:size(Cache0) of + case CacheCb:size(Cache) of Max -> - {_, Cache} = CacheCb:take_oldest(Cache0), + InternalId = {TimeStamp, erlang:unique_integer([monotonic])}, + Session = Session0#session{internal_id = InternalId}, + {_, OldKey, Order1} = gb_trees:take_smallest(Order0), + Order = gb_trees:insert(InternalId, Key, Order1), + CacheCb:delete(Cache, OldKey), + CacheCb:update(Cache, Key, Session), + {Cache, Order}; + _ -> + InternalId = {TimeStamp, erlang:unique_integer([monotonic])}, + Session = Session0#session{internal_id = InternalId}, + Order = gb_trees:insert(InternalId, Key, Order0), CacheCb:update(Cache, Key, Session), - Cache; - _ -> - CacheCb:update(Cache0, Key, Session), - Cache0 + {Cache, Order} end catch _:_ -> + %% Backwards compatibility if size functions is not implemented by callback Args = proplists:get_value(session_cb_init_args, Options, []), - CacheCb:terminate(Cache0), - CacheCb:init(Args) + CacheCb:terminate(Cache), + {CacheCb:init(Args), gb_trees:empty()} end. @@ -499,16 +512,18 @@ do_register_session(Key, Session, Max, Cache0, CacheCb, Options) -> register_unique_session(Sessions, Session, PartialKey, #state{session_cache_client_max = Max, session_cache_client = Cache0, - session_cache_cb = CacheCb, - options = Options} = State) -> + session_cache_client_cb = CacheCb, + options = Options, + client_session_order = Order0} = State) -> case exists_equivalent(Session , Sessions) of true -> State; false -> - Cache = do_register_session({PartialKey, - Session#session.session_id}, - Session, Max, Cache0, CacheCb, Options), - State#state{session_cache_client = Cache} + {Cache, Order} = do_register_session({PartialKey, + Session#session.session_id}, + Session, Max, Cache0, CacheCb, Options, Order0), + State#state{session_cache_client = Cache, + client_session_order = Order} end. exists_equivalent(_, []) -> diff --git a/lib/ssl/src/ssl_server_session_cache.erl b/lib/ssl/src/ssl_server_session_cache.erl index 44862e5cad..b444cf8865 100644 --- a/lib/ssl/src/ssl_server_session_cache.erl +++ b/lib/ssl/src/ssl_server_session_cache.erl @@ -36,7 +36,6 @@ new_session_id/1, register_session/2, reuse_session/2 - %%invalidate_session/2 ]). %% gen_server callbacks @@ -53,7 +52,7 @@ lifetime, db, max, - session_index, + session_order, id_generator, listner }). @@ -121,7 +120,7 @@ init([Listner, #{lifetime := Lifetime, lifetime = Lifetime, db = DbRef, max = Max, - session_index = #{}, + session_order = gb_trees:empty(), id_generator = crypto:strong_rand_bytes(16), listner = Monitor }, @@ -135,64 +134,59 @@ handle_call(new_session_id, _From, #state{id_generator = IdGen} = State) -> handle_call({reuse_session, SessionId}, _From, #state{store_cb = Cb, db = Store0, lifetime = Lifetime, - session_index = Index0} = State0) -> - case maps:get(SessionId, Index0, undefined) of + session_order = Order0} = State0) -> + case lookup(Cb, Store0, SessionId) of undefined -> {reply, not_reusable, State0}; - Key -> - case lookup(Cb, Store0, Key) of - undefined -> - {reply, not_reusable, State0}; - Session -> - case ssl_session:valid_session(Session, Lifetime) of - true -> - {reply, Session, State0}; - false -> - {Store, Index} = invalidate_session(Cb, Store0, Index0, SessionId, Key), - {reply, not_reusable, State0#state{db = Store, session_index = Index}} - end + #session{internal_id = InId} = Session -> + case ssl_session:valid_session(Session, Lifetime) of + true -> + {reply, Session, State0}; + false -> + Order = invalidate_session(Cb, Store0, Order0, SessionId, InId), + {reply, not_reusable, State0#state{session_order = Order}} end end. -spec handle_cast(Request :: term(), State :: term()) -> {noreply, NewState :: term()}. -handle_cast({register_session, #session{session_id = SessionId} = Session}, +handle_cast({register_session, #session{session_id = SessionId, time_stamp = TimeStamp} = Session0}, #state{store_cb = Cb, db = Store0, max = Max, lifetime = Lifetime, - session_index = Index0} + session_order = Order0} = State0) -> - TimeStamp = erlang:monotonic_time(), - Id = {TimeStamp, erlang:unique_integer([monotonic])}, + InternalId = {TimeStamp, erlang:unique_integer([monotonic])}, + Session = Session0#session{internal_id = InternalId}, State = case size(Cb, Store0) of Max -> - {#session{session_id = OldSessionId}, Store1} - = oldest_session(Cb, Store0), %% Throw away oldest session table may not grow larger than max - Store = update(Cb, Store1, Id, Session), - Index = maps:without([OldSessionId], Index0), - State0#state{db = Store, - session_index = Index#{SessionId => Id}}; + {_, OldSessId, Order1} = gb_trees:take_smallest(Order0), + Store1 = delete(Cb, Store0, OldSessId), + %% Insert new session + Order = gb_trees:insert(InternalId, SessionId, Order1), + Store = update(Cb, Store1, SessionId, Session), + Store#state{db = Store, session_order = Order}; Size when Size > 0 -> - {#session{session_id = OldSessionId} = OldestSession, Store1} - = oldest_session(Cb, Store0), + {_, OldSessId, Order1} = gb_trees:take_smallest(Order0), + OldestSession = lookup(Cb, Store0, OldSessId), case ssl_session:valid_session(OldestSession, Lifetime) of true -> - Store = update(Cb, Store0, Id, Session#session{time_stamp = TimeStamp}), + Store = update(Cb, Store0, SessionId, Session#session{time_stamp = TimeStamp}), State0#state{db = Store, - session_index = Index0#{SessionId => Id}}; + session_order = gb_trees:insert(InternalId, SessionId, Order0)}; false -> %% Throw away oldest session as it is not valid anymore - Store = update(Cb, Store1, Id, Session#session{time_stamp = TimeStamp}), - Index = maps:without([OldSessionId], Index0), + Store1 = delete(Cb, Store0, OldSessId), + Store = update(Cb, Store1, SessionId, Session#session{time_stamp = TimeStamp}), State0#state{db = Store, - session_index = Index#{SessionId => Id}} + session_order = gb_trees:insert(InternalId, SessionId, Order1)} end; 0 -> - Store = update(Cb, Store0, Id, Session#session{time_stamp = TimeStamp}), + Store = update(Cb, Store0, SessionId, Session#session{time_stamp = TimeStamp}), State0#state{db = Store, - session_index = Index0#{SessionId => Id}} + session_order = gb_trees:insert(InternalId, SessionId, Order0)} end, {noreply, State}. @@ -226,25 +220,9 @@ session_id(Key) -> Bin2 = crypto:crypto_one_time(aes_128_ecb, Key, <<Unique2:128>>, true), <<Bin1/binary, Bin2/binary>>. -invalidate_session(Cb, Store, Index, SessionId, Key) -> - {delete(Cb, Store, Key), - maps:without([SessionId], Index)}. - -oldest_session(Cb, Store0) -> - try Cb:take_oldest(Store0) of - {_, Element, Store} -> - {Element, Store} - catch - _:_ -> %% Backwards compatible - {Key, OldApiElement} = Cb:foldl(fun(E, []) -> - E; - (_, Acc)-> - Acc - end, [], - Store0), - OldAPIStore = delete(Cb, Store0, Key), - {OldApiElement, OldAPIStore} - end. +invalidate_session(Cb, Store, Order, SessionId, InternalId) -> + Cb:delete(Store, SessionId), + gb_trees:delete(InternalId, Order). init(Cb, Options) -> Cb:init(Options). diff --git a/lib/ssl/src/ssl_server_session_cache_db.erl b/lib/ssl/src/ssl_server_session_cache_db.erl index cab018c5c2..b55d2e306f 100644 --- a/lib/ssl/src/ssl_server_session_cache_db.erl +++ b/lib/ssl/src/ssl_server_session_cache_db.erl @@ -24,15 +24,15 @@ -module(ssl_server_session_cache_db). -%%-behaviour(ssl_session_cache_api). +-behaviour(ssl_session_cache_api). %% API -export([init/1, + terminate/1, lookup/2, update/3, delete/2, - size/1, - take_oldest/1]). + size/1]). %%%=================================================================== %%% API @@ -76,8 +76,5 @@ delete(Cache, Key) -> size(Cache) -> gb_trees:size(Cache). -%%-------------------------------------------------------------- -%% Description: Returns the oldest cache entry -%%-------------------------------------------------------------------- -take_oldest(Cache) -> - gb_trees:take_smallest(Cache). +terminate(_) -> + ok. diff --git a/lib/ssl/src/ssl_server_session_cache_sup.erl b/lib/ssl/src/ssl_server_session_cache_sup.erl index 2f0c3dc823..88f068a319 100644 --- a/lib/ssl/src/ssl_server_session_cache_sup.erl +++ b/lib/ssl/src/ssl_server_session_cache_sup.erl @@ -42,7 +42,7 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). start_child(Listner) -> - supervisor:start_child(?MODULE, [Listner | ssl_config:pre_1_3_session_opts()]). + supervisor:start_child(?MODULE, [Listner | [ssl_config:pre_1_3_session_opts(server)]]). %%%========================================================================= diff --git a/lib/ssl/src/ssl_session_cache_api.erl b/lib/ssl/src/ssl_session_cache_api.erl index d438a9dafd..59fbd9b3c3 100644 --- a/lib/ssl/src/ssl_session_cache_api.erl +++ b/lib/ssl/src/ssl_session_cache_api.erl @@ -40,3 +40,5 @@ -callback foldl(fun(), term(), session_cache_ref()) -> term(). -callback select_session(session_cache_ref(), {ssl:host(), inet:port_number()} | inet:port_number()) -> [#session{}]. -callback size(session_cache_ref()) -> integer(). + +-optional_callbacks([select_session/2, foldl/3]). diff --git a/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl b/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl index 936ffcc0ac..69169cca0d 100644 --- a/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl +++ b/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl @@ -55,7 +55,7 @@ start_child(Type) -> %% only one will be able to grab the local name and we will use %% that process for handling pre TLS-1.3 sessions for %% servers with to us unknown listeners. - case supervisor:start_child(SupName, [ssl_unknown_listener | ssl_config:pre_1_3_session_opts()]) of + case supervisor:start_child(SupName, [ssl_unknown_listener, ssl_config:pre_1_3_session_opts(server)]) of {error, {already_started, Child}} -> {ok, Child}; {ok, _} = Return -> diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index cf70236e6c..90024b64b1 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -452,12 +452,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac erl_dist := IsErlDist, client_renegotiation := ClientRenegotiation} = SSLOptions, ConnectionStates = tls_record:init_connection_states(Role, BeastMitigation), - SessionCacheCb = case application:get_env(ssl, session_cb) of - {ok, Cb} when is_atom(Cb) -> - Cb; - _ -> - ssl_session_cache - end, + #{session_cb := SessionCacheCb} = ssl_config:pre_1_3_session_opts(Role), UserMonitor = erlang:monitor(process, User), InitStatEnv = #static_env{ role = Role, @@ -490,28 +485,11 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac start_or_recv_from = undefined, flight_buffer = [], protocol_specific = #{sender => Sender, - active_n => internal_active_n(IsErlDist), + active_n => ssl_config:get_internal_active_n(IsErlDist), active_n_toggle => true } }. -internal_active_n(true) -> - %% Start with a random number between 1 and ?INTERNAL_ACTIVE_N - %% In most cases distribution connections are established all at - %% the same time, and flow control engages with ?INTERNAL_ACTIVE_N for - %% all connections. Which creates a wave of "passive" messages, leading - %% to significant bump of memory & scheduler utilisation. Starting with - %% a random number between 1 and ?INTERNAL_ACTIVE_N helps to spread the - %% spike. - erlang:system_time() rem ?INTERNAL_ACTIVE_N + 1; -internal_active_n(false) -> - case application:get_env(ssl, internal_active_n) of - {ok, N} when is_integer(N) -> - N; - _ -> - ?INTERNAL_ACTIVE_N - end. - handle_client_hello(#client_hello{client_version = ClientVersion} = Hello, State0) -> case tls_dtls_connection:handle_sni_extension(State0, Hello) of #state{connection_states = ConnectionStates0, diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 640b999884..cd9beecb3f 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -896,10 +896,15 @@ do_wait_cert(#certificate_1_3{} = Certificate, State0) -> end. -do_wait_cv(#certificate_verify_1_3{} = CertificateVerify, State0) -> +do_wait_cv(#certificate_verify_1_3{} = CertificateVerify, #state{static_env = #static_env{role = Role}} = State0) -> {Ref,Maybe} = maybe(), try - State1 = Maybe(verify_signature_algorithm(State0, CertificateVerify)), + State1 = case Role of + server -> + Maybe(verify_signature_algorithm(State0, CertificateVerify)); + client -> + State0 + end, Maybe(verify_certificate_verify(State1, CertificateVerify)) catch {Ref, {#alert{} = Alert, State}} -> @@ -1403,27 +1408,34 @@ process_certificate_request(#certificate_request_1_3{}, {ok, {State#state{client_certificate_requested = true}, wait_cert}}; process_certificate_request(#certificate_request_1_3{ - extensions = Extensions}, - #state{session = #session{own_certificates = [Cert|_]} = Session} = + extensions = Extensions}, + #state{ssl_options = #{signature_algs := ClientSignAlgs}, + session = #session{own_certificates = [Cert|_]} = Session} = State) -> ServerSignAlgs = get_signature_scheme_list( maps:get(signature_algs, Extensions, undefined)), ServerSignAlgsCert = get_signature_scheme_list( maps:get(signature_algs_cert, Extensions, undefined)), - {_PublicKeyAlgo, SignAlgo, SignHash, _} = get_certificate_params(Cert), - - %% Check if server supports signature algorithm of client certificate - case check_cert_sign_algo(SignAlgo, SignHash, ServerSignAlgs, ServerSignAlgsCert) of - ok -> - {ok, {State#state{client_certificate_requested = true}, wait_cert}}; - {error, _} -> - %% Certificate not supported: send empty certificate in state 'wait_finished' + {PublicKeyAlgo, SignAlgo, SignHash, MaybeRSAKeySize} = get_certificate_params(Cert), + {Ref, Maybe} = maybe(), + try + SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, MaybeRSAKeySize, ServerSignAlgs, ClientSignAlgs)), + %% Check if server supports signature algorithm of client certificate + case check_cert_sign_algo(SignAlgo, SignHash, ServerSignAlgs, ServerSignAlgsCert) of + ok -> {ok, {State#state{client_certificate_requested = true, - session = Session#session{own_certificates = undefined}}, wait_cert}} + session = Session#session{sign_alg = SelectedSignAlg}}, wait_cert}}; + {error, _} -> + %% Certificate not supported: send empty certificate in state 'wait_finished' + {ok, {State#state{client_certificate_requested = true, + session = Session#session{own_certificates = undefined}}, wait_cert}} + end + catch + {Ref, #alert{} = Alert} -> + Alert end. - process_certificate(#certificate_1_3{ certificate_request_context = <<>>, certificate_list = []}, @@ -2079,13 +2091,12 @@ maybe_update_selected_sign_alg(State, _, _) -> State. -verify_certificate_verify(#state{ - static_env = #static_env{role = Role}, - connection_states = ConnectionStates, - handshake_env = - #handshake_env{ - public_key_info = PublicKeyInfo, - tls_handshake_history = HHistory}} = State0, +verify_certificate_verify(#state{static_env = #static_env{role = Role}, + connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + public_key_info = PublicKeyInfo, + tls_handshake_history = HHistory}} = State0, #certificate_verify_1_3{ algorithm = SignatureScheme, signature = Signature}) -> @@ -2311,12 +2322,12 @@ check_cert_sign_algo(SignAlgo, SignHash, _, ClientSignAlgsCert) -> %% DSA keys are not supported by TLS 1.3 -select_sign_algo(dsa, _RSAKeySize, _ClientSignAlgs, _ServerSignAlgs) -> +select_sign_algo(dsa, _RSAKeySize, _PeerSignAlgs, _OwnSignAlgs) -> {error, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_public_key)}; select_sign_algo(_, _RSAKeySize, [], _) -> {error, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm)}; -select_sign_algo(PublicKeyAlgo, RSAKeySize, [C|ClientSignAlgs], ServerSignAlgs) -> - {_, S, _} = ssl_cipher:scheme_to_components(C), +select_sign_algo(PublicKeyAlgo, RSAKeySize, [PeerSignAlg|PeerSignAlgs], OwnSignAlgs) -> + {_, S, _} = ssl_cipher:scheme_to_components(PeerSignAlg), %% RSASSA-PKCS1-v1_5 and Legacy algorithms are not defined for use in signed %% TLS handshake messages: filter sha-1 and rsa_pkcs1. %% @@ -2328,25 +2339,25 @@ select_sign_algo(PublicKeyAlgo, RSAKeySize, [C|ClientSignAlgs], ServerSignAlgs) orelse (PublicKeyAlgo =:= rsa_pss_pss andalso S =:= rsa_pss_pss) orelse (PublicKeyAlgo =:= ecdsa andalso S =:= ecdsa)) andalso - lists:member(C, ServerSignAlgs) of + lists:member(PeerSignAlg, OwnSignAlgs) of true -> validate_key_compatibility(PublicKeyAlgo, RSAKeySize, - [C|ClientSignAlgs], ServerSignAlgs); + [PeerSignAlg|PeerSignAlgs], OwnSignAlgs); false -> - select_sign_algo(PublicKeyAlgo, RSAKeySize, ClientSignAlgs, ServerSignAlgs) + select_sign_algo(PublicKeyAlgo, RSAKeySize, PeerSignAlgs, OwnSignAlgs) end. -validate_key_compatibility(PublicKeyAlgo, RSAKeySize, [C|ClientSignAlgs], ServerSignAlgs) +validate_key_compatibility(PublicKeyAlgo, RSAKeySize, [PeerSignAlg|PeerSignAlgs], OwnSignAlgs) when PublicKeyAlgo =:= rsa orelse PublicKeyAlgo =:= rsa_pss_pss -> - case is_rsa_key_compatible(RSAKeySize, C) of + case is_rsa_key_compatible(RSAKeySize, PeerSignAlg) of true -> - {ok, C}; + {ok, PeerSignAlg}; false -> - select_sign_algo(PublicKeyAlgo, RSAKeySize, ClientSignAlgs, ServerSignAlgs) + select_sign_algo(PublicKeyAlgo, RSAKeySize, PeerSignAlgs, OwnSignAlgs) end; -validate_key_compatibility(_, _, [C|_], _) -> - {ok, C}. +validate_key_compatibility(_, _, [PeerSignAlg|_], _) -> + {ok, PeerSignAlg}. is_rsa_key_compatible(KeySize, SigAlg) -> {Hash, _, _} = ssl_cipher:scheme_to_components(SigAlg), diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index ea59bc9ffb..4e81a761f6 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -76,6 +76,7 @@ MODULES = \ ssl_pem_cache_SUITE \ ssl_session_SUITE \ ssl_session_cache_SUITE \ + ssl_session_cache_api_SUITE\ \ ssl_session_ticket_SUITE \ openssl_session_ticket_SUITE \ openssl_session_SUITE \ diff --git a/lib/ssl/test/openssl_cipher_suite_SUITE.erl b/lib/ssl/test/openssl_cipher_suite_SUITE.erl index fb1f28aa4a..ad0ecec8b7 100644 --- a/lib/ssl/test/openssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/openssl_cipher_suite_SUITE.erl @@ -401,8 +401,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; TestCase == ecdhe_rsa_rc4_128; TestCase == ecdhe_ecdsa_rc4_128; TestCase == dh_anon_rc4_128 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(rc4, SupCiphers) of + case supported_cipher(rc4, "RC4") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -413,8 +412,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; TestCase == psk_aes_128_ccm_8; TestCase == dhe_psk_aes_128_ccm_8; TestCase == ecdhe_psk_aes_128_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_128_ccm, SupCiphers) of + case supported_cipher(aes_128_ccm, "AES128-CCM8") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -425,8 +423,8 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; TestCase == psk_aes_256_ccm_8; TestCase == dhe_psk_aes_256_ccm_8; TestCase == ecdhe_psk_aes_256_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_256_ccm, SupCiphers) of + + case supported_cipher(aes_256_ccm, "AES128-CCM8") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -434,10 +432,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; {skip, "Missing AES_256_CCM crypto support"} end; init_per_testcase(aes_256_gcm_sha384, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_256_gcm, SupCiphers)) andalso - (lists:member(sha384, SupHashs)) of + case supported_cipher(aes_256_gcm, "AES256_GCM", sha384) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -445,10 +440,7 @@ init_per_testcase(aes_256_gcm_sha384, Config) -> {skip, "Missing AES_256_GCM crypto support"} end; init_per_testcase(aes_128_gcm_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case lists:member(aes_128_gcm, SupCiphers) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_gcm, "AES128_GCM", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -457,10 +449,7 @@ init_per_testcase(aes_128_gcm_sha256, Config) -> end; init_per_testcase(chacha20_poly1305_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(chacha20_poly1305, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(chacha20_poly1305_sha256, "CHACHA", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -468,10 +457,7 @@ init_per_testcase(chacha20_poly1305_sha256, Config) -> {skip, "Missing CHACHA20_POLY1305 crypto support"} end; init_per_testcase(aes_128_ccm_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_128_ccm, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_ccm, "AES128_CCM", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -480,10 +466,7 @@ init_per_testcase(aes_128_ccm_sha256, Config) -> end; init_per_testcase(aes_128_ccm_8_sha256, Config) -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - SupHashs = proplists:get_value(hashs, crypto:supports()), - case (lists:member(aes_128_ccm, SupCiphers)) andalso - (lists:member(sha256, SupHashs)) of + case supported_cipher(aes_128_ccm, "AES128_CCM8", sha256) of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -493,8 +476,7 @@ init_per_testcase(aes_128_ccm_8_sha256, Config) -> init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_ccm; TestCase == ecdhe_ecdsa_with_aes_128_ccm_8-> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_128_ccm, SupCiphers) of + case supported_cipher(aes_128_ccm, "AES128_CCM") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -504,8 +486,8 @@ init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_cc init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_256_ccm; TestCase == ecdhe_ecdsa_with_aes_256_ccm_8 -> - SupCiphers = proplists:get_value(ciphers, crypto:supports()), - case lists:member(aes_256_ccm, SupCiphers) of + + case supported_cipher(aes_256_ccm, "AES256_CCM") of true -> ct:timetrap(?DEFAULT_TIMEOUT), Config; @@ -983,3 +965,13 @@ test_ciphers(Kex, Cipher, Version) -> openssl_suitestr_to_map(OpenSSLSuiteStrs) -> [ssl_cipher_format:suite_openssl_str_to_map(SuiteStr) || SuiteStr <- OpenSSLSuiteStrs]. + + +supported_cipher(Cipher, CipherStr) -> + SupCrypto = proplists:get_value(ciphers, crypto:supports()), + SupOpenssl = [OCipher || OCipher <- ssl_test_lib:openssl_ciphers(), string:find(OCipher, CipherStr) =/= nomatch], + lists:member(Cipher, SupCrypto) andalso SupOpenssl =/= []. + +supported_cipher(Cipher, CipherStr, Hash) -> + Hashes = proplists:get_value(hashs, crypto:supports()), + supported_cipher(Cipher, CipherStr) andalso lists:member(Hash, Hashes). diff --git a/lib/ssl/test/openssl_session_SUITE.erl b/lib/ssl/test/openssl_session_SUITE.erl index 08369733dc..2dcec86579 100644 --- a/lib/ssl/test/openssl_session_SUITE.erl +++ b/lib/ssl/test/openssl_session_SUITE.erl @@ -168,7 +168,6 @@ reuse_session_erlang_server(Config) when is_list(Config) -> {_, ServerNode, _} = ssl_test_lib:run_where(Config), - Ciphers = common_ciphers(Version), Data = "From openssl to erlang", @@ -176,14 +175,13 @@ reuse_session_erlang_server(Config) when is_list(Config) -> {from, self()}, {mfa, {ssl_test_lib, active_recv, [length(Data)]}}, {reconnect_times, 5}, - {options, [{ciphers, Ciphers}, - {versions, [Version]}| ServerOpts]}]), + {options, [{versions, [Version]}| ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), {_Client, OpenSSLPort} = ssl_test_lib:start_client(openssl, [{port, Port}, {reconnect, true}, - {options, [{ciphers, Ciphers} | ClientOpts]}, + {options, ClientOpts}, return_port], Config), true = port_command(OpenSSLPort, Data), @@ -200,9 +198,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> Version = ssl_test_lib:protocol_version(Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - Ciphers = common_ciphers(Version), Server = ssl_test_lib:start_server(openssl, [], - [{server_opts, [{ciphers, Ciphers} | ServerOpts]} | Config]), + [{server_opts, ServerOpts} | Config]), Port = ssl_test_lib:inet_port(Server), Client0 = @@ -212,7 +209,6 @@ reuse_session_erlang_client(Config) when is_list(Config) -> {from, self()}, {options, [{reuse_sessions, save}, {verify, verify_peer}, - {ciphers, Ciphers}, {versions, [Version]} | ClientOpts]}]), SID = receive @@ -226,9 +222,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [ {ciphers, Ciphers}, - {versions, [Version]}, - {reuse_session, SID} | ClientOpts]}]), + {from, self()}, {options, [{versions, [Version]}, + {reuse_session, SID} | ClientOpts]}]), receive {Client1, SID} -> ok @@ -245,8 +240,7 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [{ciphers, Ciphers}, - {versions, [Version]} | ClientOpts]}]), + {from, self()}, {options, [{versions, [Version]} | ClientOpts]}]), receive {Client2, ID} -> case ID of @@ -258,14 +252,3 @@ reuse_session_erlang_client(Config) when is_list(Config) -> end, ssl_test_lib:close(Client2). - -%%-------------------------------------------------------------------- -%% Internal functions ------------------------------------------------ -%%-------------------------------------------------------------------- - -common_ciphers(Version) -> - OpenSSLCiphers = ssl_test_lib:openssl_ciphers(), - ErlOpenSSLCiphers = [ssl:str_to_suite(C) || C <- OpenSSLCiphers], - ErlCiphers = ssl:cipher_suites(all, Version), - [Suite || Suite <- ErlOpenSSLCiphers, lists:member(Suite, ErlCiphers)]. - diff --git a/lib/ssl/test/property_test/ssl_eqc_chain.erl b/lib/ssl/test/property_test/ssl_eqc_chain.erl index e78dc3fc0e..e108591776 100644 --- a/lib/ssl/test/property_test/ssl_eqc_chain.erl +++ b/lib/ssl/test/property_test/ssl_eqc_chain.erl @@ -124,6 +124,23 @@ prop_tls_extraneous_and_unordered_path() -> end ). +prop_client_cert_auth() -> + ?FORALL({ClientOptions, ServerOptions}, ?LET(Version, tls_version(), client_cert_auth_opts(Version)), + try + [TLSVersion] = proplists:get_value(versions, ClientOptions), + ssl_test_lib:basic_test(ClientOptions, ServerOptions, [{server_type, erlang}, + {client_type, erlang}, + {version, TLSVersion} + ]) + of + _ -> + true + catch + _:_ -> + false + end + ). + %%-------------------------------------------------------------------- %% Chain Generators ----------------------------------------------- %%-------------------------------------------------------------------- @@ -232,6 +249,9 @@ unordered_extraneous_options(Version) -> der_extraneous_and_unorder_options(Version) -> ?LET(Alg, key_alg(Version), der_extraneous_and_unorder_chain(Version, Alg)). +client_cert_auth_opts(Version) -> + ?LET({SAlg, CAlg}, {key_alg(Version), key_alg(Version)}, der_cert_chains(Version, CAlg,SAlg)). + extraneous_der_cert_chain_opts(Version, Alg) -> #{cert := OrgSRoot} = SRoot = public_key:pkix_test_root_cert("OTP test server ROOT", root_key(Alg)), #{cert := OrgCRoot} = CRoot = public_key:pkix_test_root_cert("OTP test client ROOT", root_key(Alg)), @@ -275,7 +295,6 @@ extraneous_pem_cert_chain_opts(Version, Alg, PrivDir) -> extraneous_pem_conf(ServerChain, ClientRoot, OrgCRoot, ServerConf0, PrivDir)]}. extra_extraneous_der_cert_chain_opts(Version, Alg) -> - #{cert := OrgSRoot} = SRoot = public_key:pkix_test_root_cert("OTP test server ROOT", root_key(Alg)), #{cert := OrgCRoot} = CRoot = public_key:pkix_test_root_cert("OTP test client ROOT", root_key(Alg)), @@ -304,7 +323,6 @@ extra_extraneous_der_cert_chain_opts(Version, Alg) -> der_extraneous_and_unorder_chain(Version, Alg) -> - #{cert := OrgSRoot} = SRoot = public_key:pkix_test_root_cert("OTP test server ROOT", root_key(Alg)), #{cert := OrgCRoot} = CRoot = public_key:pkix_test_root_cert("OTP test client ROOT", root_key(Alg)), @@ -331,6 +349,20 @@ der_extraneous_and_unorder_chain(Version, Alg) -> server_options(Version) ++ [protocol(Version), {versions, [Version]} | extraneous_der_conf(ServerChain, ClientRoot1, [OrgCRoot, ClientRoot0], ServerConf0)]}. +der_cert_chains(Version, CAlg, SAlg) -> + SRoot = public_key:pkix_test_root_cert("OTP test server ROOT", root_key(SAlg)), + CRoot = public_key:pkix_test_root_cert("OTP test client ROOT", root_key(CAlg)), + + #{server_config := ServerConf, + client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => #{root => SRoot, + intermediates => intermediates(SAlg, 1), + peer => peer_key(SAlg)}, + client_chain => #{root => CRoot, + intermediates => intermediates(CAlg, 1), + peer => peer_key(CAlg)}}), + {client_options(Version) ++ [protocol(Version), {versions, [Version]} | ClientConf], + server_options(Version) ++ [protocol(Version), {versions, [Version]} | ServerConf]}. + chain_and_root(Config) -> OwnCert = proplists:get_value(cert, Config), {ok, ExtractedCAs} = ssl_pkix_db:extract_trusted_certs({der, proplists:get_value(cacerts, Config)}), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index b737c41ba2..a82b682526 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -73,7 +73,9 @@ fake_root_no_intermediate_legacy/0, fake_root_no_intermediate_legacy/1, fake_intermediate_cert/0, - fake_intermediate_cert/1 + fake_intermediate_cert/1, + incompleat_chain_length/0, + incompleat_chain_length/1 ]). %% Apply export @@ -124,7 +126,8 @@ basic_tests() -> fake_root_no_intermediate, fake_root_legacy, fake_root_no_intermediate_legacy, - fake_intermediate_cert + fake_intermediate_cert, + incompleat_chain_length ]. options_tests() -> @@ -730,6 +733,74 @@ fake_intermediate_cert(Config) when is_list(Config) -> ssl_test_lib:check_client_alert(Client1, bad_certificate). +incompleat_chain_length() -> + [{doc,"Test that attempts to reconstruct incomplete chains does not make shorter incomplete chains"}]. +incompleat_chain_length(Config) when is_list(Config)-> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + OtherROOT = public_key:pkix_test_root_cert("OTHER SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(3)}, + {extensions, Ext}]), + + + #{client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := ServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => OtherROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}], + [{key, ssl_test_lib:hardcode_rsa_key(3)}] + ], + peer => [{key, ssl_test_lib:hardcode_rsa_key(1)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + + VerifyFun = {fun(_,{bad_cert, unknown_ca}, UserState) -> + %% accept this error to provoke the + %% building of an shorter incomplete chain + %% than the one recived + {valid, UserState}; + (_,{extension, _} = Extension, #{ext := N} = UserState) -> + ct:pal("~p", [Extension]), + {unknown, UserState#{ext => N +1}}; + (_, valid, #{intermediates := N} = UserState) -> + {valid, UserState#{intermediates => N +1}}; + (_, valid_peer, #{intermediates := 2, + ext := 1} = UserState) -> + {valid, UserState}; + (_, valid_peer, UserState) -> + ct:pal("~p", [UserState]), + {error, {bad_cert, too_short_path}} + end, #{intermediates => 0, + ext => 0}}, + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerConf} + ]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{verify, verify_peer}, {verify_fun, VerifyFun} | ClientConf]}]), + ssl_test_lib:check_result(Client, ok, Server, ok). + %%-------------------------------------------------------------------- %% callback functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_cert_SUITE.erl b/lib/ssl/test/ssl_cert_SUITE.erl index efa01fda76..a142115b55 100644 --- a/lib/ssl/test/ssl_cert_SUITE.erl +++ b/lib/ssl/test/ssl_cert_SUITE.erl @@ -98,6 +98,8 @@ unsupported_sign_algo_cert_client_auth/1, longer_chain/0, longer_chain/1, + duplicate_chain/0, + duplicate_chain/1, key_auth_ext_sign_only/0, key_auth_ext_sign_only/1, hello_retry_request/0, @@ -870,6 +872,62 @@ longer_chain(Config) when is_list(Config) -> proplists:delete(cacerts, ClientOpts0)], Config), ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). +duplicate_chain() -> + [{doc, "Manual test of chain with duplicate entries"}]. +duplicate_chain(Config) + when is_list(Config) -> + Key1 = ssl_test_lib:hardcode_rsa_key(1), + Key2 = ssl_test_lib:hardcode_rsa_key(2), + Key3 = ssl_test_lib:hardcode_rsa_key(3), + Key4 = ssl_test_lib:hardcode_rsa_key(4), + Key5 = ssl_test_lib:hardcode_rsa_key(5), + + #{server_config := ServerOpts0, client_config := ClientOpts0} = + public_key:pkix_test_data(#{server_chain => #{root => [{key, Key1}], + peer => [{key, Key5}]}, + client_chain => #{root => [{key, Key3}], + intermediates => [[{key, Key2}], [{key, Key3}]], + peer => [{key, Key1}]}}), + + #{client_config := ClientOptsNew} = + public_key:pkix_test_data(#{server_chain => #{root => [{key, Key1}], + peer => [{key, Key5}]}, + client_chain => #{root => [{key, Key4}], + intermediates => [[{key, Key2}], [{key, Key1}]], + peer => [{key, Key1}]}}), + + ServerCas0 = proplists:get_value(cacerts, ServerOpts0), + ClientCas0 = proplists:get_value(cacerts, ClientOpts0), + + {[Peer,CI1,CI2,CROld], CROld} = chain_and_root(ClientOpts0), + {[_Peer,CI1New,CI2New,CRNew], CRNew} = chain_and_root(ClientOptsNew), + + ServerCas = [CRNew|ServerCas0 -- [CROld]], + ServerOpts = ssl_test_lib:ssl_options([{verify, verify_peer} | + lists:keyreplace(cacerts, 1, ServerOpts0, {cacerts, ServerCas})], + Config), + ClientOpts = ssl_test_lib:ssl_options([{verify, verify_peer} | + lists:keyreplace(cacerts, 1, + lists:keyreplace(cert, 1, ClientOpts0, + {cert, [Peer,CI1New,CI2New,CI1,CI2,CRNew,CROld]}), + {cacerts, ClientCas0})], + Config), + ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config), + ClientOpts2 = ssl_test_lib:ssl_options([{verify, verify_peer} | + lists:keyreplace(cacerts, 1, + lists:keyreplace(cert, 1, ClientOpts0, + {cert, [Peer,CI1,CI1New,CI2,CI2New,CROld,CRNew]}), + {cacerts, ClientCas0})], + Config), + ssl_test_lib:basic_test(ClientOpts2, ServerOpts, Config), + ok. + +chain_and_root(Config) -> + OwnCert = proplists:get_value(cert, Config), + {ok, ExtractedCAs} = ssl_pkix_db:extract_trusted_certs({der, proplists:get_value(cacerts, Config)}), + {ok, Root, Chain} = ssl_certificate:certificate_chain(OwnCert, ets:new(foo, []), ExtractedCAs, []), + {Chain, Root}. + %%-------------------------------------------------------------------- %% TLS 1.3 Test cases ----------------------------------------------- %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 2123ac5def..67be91196f 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -828,14 +828,17 @@ localhost_ip(InetVer) -> localhost_ipstr(InetVer) -> {ok, Addr} = inet:getaddr(net_adm:localhost(), InetVer), - case InetVer of - inet -> - lists:flatten(io_lib:format("'{~p,~p,~p,~p}'", - erlang:tuple_to_list(Addr))); - inet6 -> - lists:flatten(io_lib:format("'{~p,~p,~p,~p,~p,~p,~p,~p}'", - erlang:tuple_to_list(Addr))) - end. + Str = case InetVer of + inet -> + io_lib:format("{~p,~p,~p,~p}", erlang:tuple_to_list(Addr)); + inet6 -> + io_lib:format("{~p,~p,~p,~p,~p,~p,~p,~p}", erlang:tuple_to_list(Addr)) + end, + Qouted = case os:type() of + {win32, _} -> Str; + _ -> [$',Str,$'] + end, + lists:flatten(Qouted). inet_ver() -> inet. diff --git a/lib/ssl/test/ssl_eqc_SUITE.erl b/lib/ssl/test/ssl_eqc_SUITE.erl index 3c9a1d0ab0..4bfff1585e 100644 --- a/lib/ssl/test/ssl_eqc_SUITE.erl +++ b/lib/ssl/test/ssl_eqc_SUITE.erl @@ -39,7 +39,8 @@ tls_unorded_chains/1, tls_extraneous_chain/1, tls_extraneous_chains/1, - tls_extraneous_and_unorder_chains/1 + tls_extraneous_and_unorder_chains/1, + tls_client_cert_auth/1 ]). %%-------------------------------------------------------------------- @@ -56,7 +57,8 @@ all() -> tls_unorded_chains, tls_extraneous_chain, tls_extraneous_chains, - tls_extraneous_and_unorder_chains + tls_extraneous_and_unorder_chains, + tls_client_cert_auth ]. %%-------------------------------------------------------------------- @@ -123,3 +125,9 @@ tls_extraneous_and_unorder_chains(Config) when is_list(Config) -> ssl:start(), true = ct_property_test:quickcheck(ssl_eqc_chain:prop_tls_extraneous_and_unordered_path(), Config). + +tls_client_cert_auth(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(ssl_eqc_chain:prop_client_cert_auth() + ssl:start(), + true = ct_property_test:quickcheck(ssl_eqc_chain:prop_client_cert_auth(), + Config). diff --git a/lib/ssl/test/ssl_session_SUITE.erl b/lib/ssl/test/ssl_session_SUITE.erl index b11e49ad89..552be7ddd6 100644 --- a/lib/ssl/test/ssl_session_SUITE.erl +++ b/lib/ssl/test/ssl_session_SUITE.erl @@ -216,7 +216,7 @@ make_sure_expired(Host, Port, Id) -> State = ssl_test_lib:state(Prop), ClientCache = element(2, State), - case ssl_session_cache:lookup(ClientCache, {{Host, Port}, Id}) of + case ssl_client_session_cache_db:lookup(ClientCache, {{Host, Port}, Id}) of undefined -> ok; #session{is_resumable = false} -> diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index 7c182ee063..8f67908ad7 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -63,6 +63,7 @@ -define(SLEEP, 1000). -define(TIMEOUT, {seconds, 20}). -define(MAX_TABLE_SIZE, 5). +-define(CLIENT_CB, ssl_client_session_cache_db). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -225,7 +226,7 @@ client_unique_session(Config) when is_list(Config) -> State = ssl_test_lib:state(Prop), ClientCache = element(2, State), - 1 = ssl_session_cache:size(ClientCache), + 1 = ?CLIENT_CB:size(ClientCache), ssl_test_lib:close(Server, 500), ssl_test_lib:close(LastClient). @@ -263,7 +264,7 @@ session_cleanup(Config) when is_list(Config) -> SessionTimer = element(6, State), Id = proplists:get_value(session_id, SessionInfo), - CSession = ssl_session_cache:lookup(ClientCache, {{Hostname, Port}, Id}), + CSession = ?CLIENT_CB:lookup(ClientCache, {{Hostname, Port}, Id}), true = CSession =/= undefined, @@ -272,7 +273,7 @@ session_cleanup(Config) when is_list(Config) -> ct:sleep(?SLEEP), %% Make sure clean has had time to run - undefined = ssl_session_cache:lookup(ClientCache, {{Hostname, Port}, Id}), + undefined = ?CLIENT_CB:lookup(ClientCache, {{Hostname, Port}, Id}), process_flag(trap_exit, false), ssl_test_lib:close(Server), @@ -335,7 +336,7 @@ save_specific_session(Config) when is_list(Config) -> [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), ClientCache = element(2, State), - 2 = ssl_session_cache:size(ClientCache), + 2 = ?CLIENT_CB:size(ClientCache), Server ! listen, @@ -379,7 +380,7 @@ max_table_size(Config) when is_list(Config) -> [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), ClientCache = element(2, State), - M = ssl_session_cache:size(ClientCache), + M = ?CLIENT_CB:size(ClientCache), ct:pal("~p",[M]), ssl_test_lib:close(Server, 500), ssl_test_lib:close(LastClient), diff --git a/lib/ssl/test/ssl_session_cache_api_SUITE.erl b/lib/ssl/test/ssl_session_cache_api_SUITE.erl new file mode 100644 index 0000000000..03f88f755f --- /dev/null +++ b/lib/ssl/test/ssl_session_cache_api_SUITE.erl @@ -0,0 +1,105 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2020. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_session_cache_api_SUITE). + +-behaviour(ct_suite). + +-include_lib("common_test/include/ct.hrl"). +-include("tls_handshake.hrl"). + +%% Callback functions +-export([all/0]). + +%% Testcases +-export([server_cb/0, + server_cb/1, + client_cb/0, + client_cb/1 + ]). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [server_cb, + client_cb]. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- +server_cb() -> + [{doc, "Test ssl_session_cache_api server callback implementation" + "Note that default implementation is treated special to avoid" + "an extra process to handle the functional db structure used" + "The callback is provided for the main purpose of having a" + "session table that may survive node restart an assumes + a db reference that is not updated" + }]. + +server_cb(Config) when is_list(Config) -> + Cb = ssl_server_session_cache_db, + Db0 = Cb:init([]), + Id0 = crypto:strong_rand_bytes(32), + DummySession0 = #session{session_id = Id0}, + 0 = Cb:size(Db0), + + Db1 = Cb:update(Db0, Id0, DummySession0), + 1 = Cb:size(Db1), + + Id1 = crypto:strong_rand_bytes(32), + DummySession1 = #session{session_id = Id1}, + Db2 = Cb:update(Db1, Id1, DummySession1), + 2 = Cb:size(Db2), + + DummySession0 = Cb:lookup(Db2, Id0), + DummySession1 = Cb:lookup(Db2, Id1), + + Db3 = Cb:delete(Db2, Id1), + 1 = Cb:size(Db3), + undefined = Cb:lookup(Db3, crypto:strong_rand_bytes(32)), + Cb:terminate(Db3). + +client_cb() -> + [{doc, "Test ssl_session_cache_api client callback implementation"}]. + +client_cb(Config) when is_list(Config) -> + Cb = ssl_client_session_cache_db, + Db = Cb:init([]), + Id0 = crypto:strong_rand_bytes(32), + DummySession0 = #session{session_id = Id0}, + 0 = Cb:size(Db), + + Cb:update(Db, Id0, DummySession0), + 1 = Cb:size(Db), + + Id1 = crypto:strong_rand_bytes(32), + DummySession1 = #session{session_id = Id1}, + Cb:update(Db, Id1, DummySession1), + 2 = Cb:size(Db), + + DummySession0 = Cb:lookup(Db, Id0), + DummySession1 = Cb:lookup(Db, Id1), + undefined = Cb:lookup(Db, crypto:strong_rand_bytes(32)), + + Cb:delete(Db, Id1), + 1 = Cb:size(Db), + Cb:terminate(Db). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 3182cdb0d8..c17883bd6f 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -176,7 +176,9 @@ ecdsa_suites/1, der_to_pem/2, pem_to_der/1, - appropriate_sha/1 + appropriate_sha/1, + format_certs/1, + format_cert/1 ]). -export([maybe_force_ipv4/1, @@ -206,6 +208,7 @@ -record(sslsocket, { fd = nil, pid = nil}). -define(SLEEP, 1000). -define(DEFAULT_CURVE, secp256r1). +-define(PRINT_DEPTH, 100). %%==================================================================== %% API @@ -418,7 +421,7 @@ run_server(Opts) -> Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), - ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), + ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, format_options(Options)]), %% {ok, ListenSocket} = Transport:listen(Port, Options), case Transport:listen(Port, Options) of {ok, ListenSocket} -> @@ -542,9 +545,9 @@ connect(ListenSocket, _Opts) -> connect(_, _, 0, AcceptSocket, _, _, _) -> AcceptSocket; connect(ListenSocket, Node, _N, _, Timeout, SslOpts, cancel) -> - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("ssl:transport_accept(~P)~n", [ListenSocket, ?PRINT_DEPTH]), {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), - ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), + ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, format_options(SslOpts),Timeout]), case ssl:handshake(AcceptSocket, SslOpts, Timeout) of {ok, Socket0, Ext} -> @@ -556,7 +559,7 @@ connect(ListenSocket, Node, _N, _, Timeout, SslOpts, cancel) -> Result end; connect(ListenSocket, Node, N, _, Timeout, SslOpts, [_|_] =ContOpts0) -> - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("ssl:transport_accept(~P)~n", [ListenSocket, ?PRINT_DEPTH]), {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), @@ -588,7 +591,7 @@ connect(ListenSocket, Node, N, _, Timeout, SslOpts, [_|_] =ContOpts0) -> Result end; connect(ListenSocket, Node, N, _, Timeout, [], ContOpts) -> - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("ssl:transport_accept(~P)~n", [ListenSocket, ?PRINT_DEPTH]), {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("~p:~p~nssl:handshake(~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, Timeout]), @@ -600,7 +603,7 @@ connect(ListenSocket, Node, N, _, Timeout, [], ContOpts) -> Result end; connect(ListenSocket, _Node, _, _, Timeout, Opts, _) -> - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("ssl:transport_accept(~P)~n", [ListenSocket, ?PRINT_DEPTH]), {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("ssl:handshake(~p,~p, ~p)~n", [AcceptSocket, Opts, Timeout]), ssl:handshake(AcceptSocket, Opts, Timeout), @@ -915,7 +918,7 @@ run_client(Opts) -> Options = proplists:get_value(options, Opts), ContOpts = proplists:get_value(continue_options, Opts, []), ct:log("~p:~p~n~p:connect(~p, ~p)@~p~n", [?MODULE,?LINE, Transport, Host, Port, Node]), - ct:log("SSLOpts: ~p", [Options]), + ct:log("SSLOpts: ~p", [format_options(Options)]), case ContOpts of [] -> client_loop(Node, Host, Port, Pid, Transport, Options, Opts); @@ -1243,6 +1246,44 @@ wait_for_result(Pid, Msg) -> %% Unexpected end. +format_options([{cacerts, Certs}|R]) -> + [{cacerts, format_certs(Certs)} | format_options(R)]; +format_options([{cert, Certs}|R]) -> + [{cert, format_certs(Certs)} | format_options(R)]; +format_options([{key, Key}|R]) -> + [{key, lists:flatten(io_lib:format("~W",[Key, ?PRINT_DEPTH]))} | format_options(R)]; +format_options([Opt|R]) -> + [Opt | format_options(R)]; +format_options([]) -> + []. + +format_certs(Certs) when is_list(Certs) -> + [lists:flatten(format_cert(C)) || C <- Certs]; +format_certs(Cert) when is_binary(Cert) -> + lists:flatten(format_cert(Cert)). + +format_cert(BinCert) when is_binary(BinCert) -> + OtpCert = #'OTPCertificate'{tbsCertificate = Cert} = public_key:pkix_decode_cert(BinCert, otp), + #'OTPTBSCertificate'{subject = Subject, serialNumber = Nr, issuer = Issuer} = Cert, + case public_key:pkix_is_self_signed(OtpCert) of + true -> + io_lib:format("~.3w: ~s -> selfsigned", [Nr, format_subject(Subject)]); + false -> + case public_key:pkix_issuer_id(OtpCert, other) of + {ok, {IsNr, Issuer0}} -> + io_lib:format("~.3w:~s -> ~.3w:~s", [Nr, format_subject(Subject), IsNr, format_subject(Issuer0)]); + {error, _} -> + io_lib:format("~.3w:~s -> :~s", [Nr, format_subject(Subject), format_subject(Issuer)]) + end + end. + +format_subject({rdnSequence, Seq}) -> + format_subject(Seq); +format_subject([[{'AttributeTypeAndValue', ?'id-at-commonName', {_, String}}]|_]) -> + String; +format_subject([_|R]) -> + format_subject(R). + cert_options(Config) -> ClientCaCertFile = filename:join([proplists:get_value(priv_dir, Config), "client", "cacerts.pem"]), @@ -1909,8 +1950,8 @@ accepters(Acc, N) -> basic_test(COpts, SOpts, Config) -> - SType = proplists:get_value(server_type, Config), - CType = proplists:get_value(client_type, Config), + SType = proplists:get_value(server_type, Config, erlang), + CType = proplists:get_value(client_type, Config, erlang), {Server, Port} = start_server(SType, COpts, SOpts, Config), Client = start_client(CType, Port, COpts, Config), gen_check_result(Server, SType, Client, CType), @@ -3068,19 +3109,23 @@ close_loop(Port, Time, SentClose) -> end. portable_open_port("openssl" = Exe, Args0) -> - case os:getenv("WSLENV") of - false -> - AbsPath = os:find_executable(Exe), - ct:pal("open_port({spawn_executable, ~p}, [{args, ~p}, stderr_to_stdout]).", - [AbsPath, Args0]), - open_port({spawn_executable, AbsPath}, - [{args, Args0}, stderr_to_stdout]); + IsWindows = case os:type() of + {win32, _} -> true; + _ -> false + end, + case IsWindows andalso os:getenv("WSLENV") of + false -> + AbsPath = os:find_executable(Exe), + ct:pal("open_port({spawn_executable, ~p}, [{args, ~p}, stderr_to_stdout]).", + [AbsPath, Args0]), + open_port({spawn_executable, AbsPath}, + [{args, Args0}, stderr_to_stdout]); _ -> %% I can't get the new windows version of openssl.exe to be stable %% certain server tests are failing for no reason. %% This is using "linux" openssl via wslenv - Translate = fun("c:/" ++ _ = Path) -> + Translate = fun([_Drive|":/" ++ _ ]= Path) -> string:trim(os:cmd("wsl wslpath -u " ++ Path)); (Arg) -> Arg @@ -3095,8 +3140,8 @@ portable_open_port("openssl" = Exe, Args0) -> portable_open_port(Exe, Args) -> AbsPath = os:find_executable(Exe), ct:pal("open_port({spawn_executable, ~p}, [{args, ~p}, stderr_to_stdout]).", [AbsPath, Args]), - open_port({spawn_executable, AbsPath}, - [{args, Args}, stderr_to_stdout]). + open_port({spawn_executable, AbsPath}, + [{args, Args}, stderr_to_stdout]). portable_cmd(Exe, Args) -> Port = portable_open_port(Exe, Args), @@ -3204,14 +3249,6 @@ protocol_version(Config, atom) -> tls_record:protocol_version(protocol_version(Config, tuple)) end. -protocol_versions(Config) -> - Version = protocol_version(Config), - case Version of - 'tlsv1.3' -> %% TLS-1.3 servers shall also support 1.2 - ['tlsv1.3', 'tlsv1.2']; - _ -> - [Version] - end. protocol_options(Config, Options) -> Protocol = proplists:get_value(protocol, Config, tls), {Protocol, Opts} = lists:keyfind(Protocol, 1, Options), @@ -3261,21 +3298,6 @@ clean_start(keep_version) -> clean_env(keep_version), ssl:start(). -is_psk_anon_suite({psk, _,_}) -> - true; -is_psk_anon_suite({dhe_psk,_,_}) -> - true; -is_psk_anon_suite({ecdhe_psk,_,_}) -> - true; -is_psk_anon_suite({psk, _,_,_}) -> - true; -is_psk_anon_suite({dhe_psk, _,_,_}) -> - true; -is_psk_anon_suite({ecdhe_psk, _,_,_}) -> - true; -is_psk_anon_suite(_) -> - false. - tls_version('dtlsv1' = Atom) -> dtls_v1:corresponding_tls_version(dtls_record:protocol_version(Atom)); diff --git a/lib/ssl/test/tls_api_SUITE.erl b/lib/ssl/test/tls_api_SUITE.erl index 4906cf2ba3..594611e923 100644 --- a/lib/ssl/test/tls_api_SUITE.erl +++ b/lib/ssl/test/tls_api_SUITE.erl @@ -39,6 +39,8 @@ %% Test cases -export([tls_upgrade/0, tls_upgrade/1, + tls_upgrade_new_opts/0, + tls_upgrade_new_opts/1, tls_upgrade_with_timeout/0, tls_upgrade_with_timeout/1, tls_downgrade/0, @@ -83,6 +85,7 @@ %% Apply export -export([upgrade_result/1, + upgrade_result_new_opts/1, tls_downgrade_result/2, tls_shutdown_result/2, tls_shutdown_write_result/2, @@ -117,6 +120,7 @@ groups() -> api_tests() -> [ tls_upgrade, + tls_upgrade_new_opts, tls_upgrade_with_timeout, tls_downgrade, tls_shutdown, @@ -199,6 +203,44 @@ tls_upgrade(Config) when is_list(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +tls_upgrade_new_opts() -> + [{doc,"Test that you can upgrade an tcp connection to an ssl connection and give new socket opts"}]. + +tls_upgrade_new_opts(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + TcpOpts = [binary, {reuseaddr, true}], + + Server = ssl_test_lib:start_upgrade_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, + upgrade_result_new_opts, []}}, + {tcp_options, + [{active, false} | TcpOpts]}, + {ssl_options, [{verify, verify_peer}, + {mode, list} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_upgrade_client([{node, ClientNode}, + {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, upgrade_result_new_opts, []}}, + {tcp_options, [binary]}, + {ssl_options, [{verify, verify_peer}, + {mode, list}, + {server_name_indication, Hostname} | ClientOpts]}]), + + ct:log("Testcase ~p, Client ~p Server ~p ~n", + [self(), Client, Server]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + %%-------------------------------------------------------------------- tls_upgrade_with_timeout() -> [{doc,"Test handshake/3"}]. @@ -817,6 +859,14 @@ upgrade_result(Socket) -> <<"Hello world">> = ssl_test_lib:active_recv(Socket, length("Hello world")), ok. +upgrade_result_new_opts(Socket) -> + ssl:setopts(Socket, [{active, true}]), + ok = ssl:send(Socket, "Hello world"), + %% Make sure list option set in ssl:connect/handskae overrides + %% previous gen_tcp socket option that was set to binary. + "Hello world" = ssl_test_lib:active_recv(Socket, length("Hello world")), + ok. + tls_downgrade_result(Socket, Pid) -> ok = ssl_test_lib:send_recv_result(Socket), Pid ! {self(), ready}, diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 451777e4b7..ecb5111403 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 10.2.4 +SSL_VSN = 10.3 diff --git a/lib/stdlib/doc/src/binary.xml b/lib/stdlib/doc/src/binary.xml index 412a0f5def..d6118b937a 100644 --- a/lib/stdlib/doc/src/binary.xml +++ b/lib/stdlib/doc/src/binary.xml @@ -243,6 +243,34 @@ </func> <func> + <name name="encode_hex" arity="1" since="OTP @OTP-17236@"/> + <fsummary>Encodes a binary into a hex encoded binary.</fsummary> + <desc> + <p>Encodes a binary into a hex encoded binary.</p> + + <p><em>Example:</em></p> + + <code> +1> binary:encode_hex(<<"f">>). +<<"66">></code> + </desc> + </func> + + <func> + <name name="decode_hex" arity="1" since="OTP @OTP-17236@"/> + <fsummary>Decodes a hex encoded binary into a binary.</fsummary> + <desc> + <p>Decodes a hex encoded binary into a binary.</p> + + <p><em>Example</em></p> + + <code> +1> binary:decode_hex(<<"66">>). +<<"f">></code> + </desc> + </func> + + <func> <name name="first" arity="1" since="OTP R14B"/> <fsummary>Return the first byte of a binary.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/digraph.xml b/lib/stdlib/doc/src/digraph.xml index 721429d819..86d32fe24a 100644 --- a/lib/stdlib/doc/src/digraph.xml +++ b/lib/stdlib/doc/src/digraph.xml @@ -331,7 +331,7 @@ <desc> <p>Returns a list of all edges <seeerl marker="#emanate">emanating</seeerl> from or - <seeerl marker="#incident">incident</seeerl> on<c><anno>V</anno></c> + <seeerl marker="#incident">incident</seeerl> on <c><anno>V</anno></c> of digraph <c><anno>G</anno></c>, in some unspecified order.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/digraph_utils.xml b/lib/stdlib/doc/src/digraph_utils.xml index 94fcc26ee5..e9e9671b71 100644 --- a/lib/stdlib/doc/src/digraph_utils.xml +++ b/lib/stdlib/doc/src/digraph_utils.xml @@ -168,7 +168,7 @@ <fsummary>Return the components of a digraph.</fsummary> <desc> <p>Returns a list - of <seeerl marker="#components">connected components.</seeerl>. + of <seeerl marker="#components">connected components</seeerl>. Each component is represented by its vertices. The order of the vertices and the order of the components are arbitrary. Each vertex of digraph diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index ad41d33d1d..9c60fb860f 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,47 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.14.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Handle maps in <c>erl_parse:tokens()</c>.</p> + <p> + Own Id: OTP-16978</p> + </item> + <item> + <p> + The erlang shell function <c>rr</c> has been fixed to be + able to read records from files within a code archive.</p> + <p> + Own Id: OTP-17182 Aux Id: PR-3002 </p> + </item> + <item> + <p>If <c>beam_lib</c> is asked to return abstract code + for a BEAM file produced by Elixir and Elixir is not + installed on the computer, <c>beam_lib</c> will no longer + crash, but will return an error tuple. The + <c>cover:compile_beam()</c> and + <c>cover:compile_beam_directory()</c> functions have been + updated to also return an error tuple in that + situation.</p> + <p> + Own Id: OTP-17194 Aux Id: GH-4353 </p> + </item> + <item> + <p> + Correct example module <c>erl_id_trans</c> regarding the + <c>{char, C}</c> type.</p> + <p> + Own Id: OTP-17273</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.14</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/examples/erl_id_trans.erl b/lib/stdlib/examples/erl_id_trans.erl index 0aa23fb8b3..264148eaf4 100644 --- a/lib/stdlib/examples/erl_id_trans.erl +++ b/lib/stdlib/examples/erl_id_trans.erl @@ -612,6 +612,8 @@ type({atom,Anno,A}) -> {atom,Anno,A}; type({integer,Anno,I}) -> {integer,Anno,I}; +type({char,Anno,C}) -> + {char,Anno,C}; type({op,Anno,Op,T}) -> T1 = type(T), {op,Anno,Op,T1}; diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index eb5f225a98..73b619d192 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -20,7 +20,8 @@ -module(binary). %% %% Implemented in this module: --export([replace/3,replace/4]). +-export([replace/3,replace/4, + encode_hex/1, decode_hex/1]). -export_type([cp/0]). @@ -346,8 +347,6 @@ splitat(H,N,[]) -> splitat(H,N,[I|T]) -> [binary:part(H,{N,I-N})|splitat(H,I,T)]. - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Simple helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -363,6 +362,65 @@ get_opts_replace([{insert_replaced,N} | T],{Part,Global,_Insert}) -> get_opts_replace(_,_) -> throw(badopt). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Hex encoding functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-spec encode_hex(Bin) -> Bin2 when + Bin :: binary(), + Bin2 :: binary(). +encode_hex(Bin) when is_binary(Bin) -> + encode_hex(Bin, <<>>); +encode_hex(Bin) -> + badarg_with_info([Bin]). + +-spec encode_hex(Bin, Acc) -> Bin2 when + Bin :: binary(), + Acc :: binary(), + Bin2 :: Acc. +encode_hex(<<>>, Acc) -> + Acc; +encode_hex(<<A0:4, B0:4, Rest/binary>>, Acc) -> + A = encode_hex_digit(A0), + B = encode_hex_digit(B0), + encode_hex(Rest, <<Acc/binary, A, B>>). + +-spec encode_hex_digit(0..15) -> byte(). +encode_hex_digit(Char) when Char =< 9 -> + Char + $0; +encode_hex_digit(Char) when Char =< 15 -> + Char + $A - 10. + +-spec decode_hex(Bin) -> Bin2 when + Bin :: binary(), + Bin2 :: binary(). +decode_hex(Bin) when is_binary(Bin) -> + decode_hex(Bin, <<>>); +decode_hex(Bin) -> + badarg_with_info([Bin]). + +-spec decode_hex(Bin, Acc) -> Bin2 when + Bin :: binary(), + Acc :: binary(), + Bin2 :: Acc. +decode_hex(<<>>, Acc) -> + Acc; +decode_hex(<<A0:8, B0:8, Rest/binary>>, Acc) -> + A = decode_hex_char(A0), + B = decode_hex_char(B0), + decode_hex(Rest, <<Acc/binary, A:4, B:4>>); +decode_hex(Bin, _Acc) -> + badarg_with_info([Bin]). + +-spec decode_hex_char($A..$F | $a..$f | $0..$9) -> 0..15. +decode_hex_char(Char) when Char >= $a, Char =< $f -> + Char - $a + 10; +decode_hex_char(Char) when Char >= $A, Char =< $F -> + Char - $A + 10; +decode_hex_char(Char) when Char >= $0, Char =< $9 -> + Char - $0; +decode_hex_char(Char) -> + badarg_with_cause([<<Char>>], invalid_hex). + badarg_with_cause(Args, Cause) -> erlang:error(badarg, Args, [{error_info, #{module => erl_stdlib_errors, cause => Cause}}]). diff --git a/lib/stdlib/src/erl_expand_records.erl b/lib/stdlib/src/erl_expand_records.erl index 08b968f18b..8d0bcbbdc3 100644 --- a/lib/stdlib/src/erl_expand_records.erl +++ b/lib/stdlib/src/erl_expand_records.erl @@ -313,7 +313,8 @@ expr({record,Anno0,Name,Is}, St) -> expr({tuple,Anno,[{atom,Anno0,Name} | record_inits(record_fields(Name, Anno0, St), Is)]}, St); -expr({record_field,Anno,R,Name,F}, St) -> +expr({record_field,_A,R,Name,F}, St) -> + Anno = erl_parse:first_anno(R), get_record_field(Anno, R, F, Name, St); expr({record,Anno,R,Name,Us}, St0) -> {Ue,St1} = record_update(R, Name, record_fields(Name, Anno, St0), Us, St0), diff --git a/lib/stdlib/src/erl_stdlib_errors.erl b/lib/stdlib/src/erl_stdlib_errors.erl index 5e7f987272..7f6b1ce4ce 100644 --- a/lib/stdlib/src/erl_stdlib_errors.erl +++ b/lib/stdlib/src/erl_stdlib_errors.erl @@ -69,6 +69,25 @@ format_binary_error(encode_unsigned, [Subject], _) -> [must_be_non_neg_integer(Subject)]; format_binary_error(encode_unsigned, [Subject, Endianness], _) -> [must_be_non_neg_integer(Subject), must_be_endianness(Endianness)]; +format_binary_error(encode_hex, [Subject], _) -> + [must_be_binary(Subject)]; +format_binary_error(decode_hex, [Subject], _) -> + if + is_binary(Subject), byte_size(Subject) rem 2 == 1 -> + ["must contain an even number of bytes"]; + true -> + [must_be_binary(Subject)] + end; +format_binary_error(decode_hex_char, [Subject], _) -> + if + is_integer(Subject), + (Subject >= $a andalso Subject =< $f) orelse + (Subject >= $A andalso Subject =< $F) orelse + (Subject >= $0 andalso Subject =< $9) -> + [[]]; + true -> + [range] + end; format_binary_error(first, [Subject], _) -> [case Subject of <<>> -> empty_binary; diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 3ab281dbad..1e34728d0d 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -89,8 +89,6 @@ obsolete(net, sleep, 1) -> {deprecated, "use 'receive after T -> ok end' instead"}; obsolete(queue, lait, 1) -> {deprecated, "use queue:liat/1 instead"}; -obsolete(snmpa, old_info_format, 1) -> - {deprecated, "use \"new\" format instead", "OTP 24"}; obsolete(snmpm, async_get, 3) -> {deprecated, "use snmpm:async_get2/3 instead.", "OTP 25"}; obsolete(snmpm, async_get, 4) -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 2f44c251e0..bba0d1ceee 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -41,6 +41,8 @@ {<<"^3\\.13\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.13\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.13\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.14$">>,[restart_new_emulator]}, + {<<"^3\\.14\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.5$">>,[restart_new_emulator]}, {<<"^3\\.5\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.5\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -70,6 +72,8 @@ {<<"^3\\.13\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.13\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.13\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.14$">>,[restart_new_emulator]}, + {<<"^3\\.14\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.5$">>,[restart_new_emulator]}, {<<"^3\\.5\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.5\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index 2af4583d8b..70e100c9e6 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -23,7 +23,7 @@ interesting/1,scope_return/1,random_ref_comp/1,random_ref_sr_comp/1, random_ref_fla_comp/1,parts/1, bin_to_list/1, list_to_bin/1, copy/1, referenced/1,guard/1,encode_decode/1,badargs/1,longest_common_trap/1, - check_no_invalid_read_bug/1,error_info/1]). + check_no_invalid_read_bug/1,error_info/1, hex_encoding/1]). -export([random_number/1, make_unaligned/1]). @@ -38,12 +38,11 @@ all() -> random_ref_comp, parts, bin_to_list, list_to_bin, copy, referenced, guard, encode_decode, badargs, longest_common_trap, check_no_invalid_read_bug, - error_info]. + error_info, hex_encoding]. -define(MASK_ERROR(EXPR),mask_error((catch (EXPR)))). - %% Test various badarg exceptions in the module. badargs(Config) when is_list(Config) -> badarg = ?MASK_ERROR(binary:compile_pattern([<<1,2,3:3>>])), @@ -238,6 +237,12 @@ badargs(Config) when is_list(Config) -> badarg = ?MASK_ERROR( binary:at([1,2,4],2)), + + badarg = ?MASK_ERROR(binary:encode_hex("abc")), + badarg = ?MASK_ERROR(binary:encode_hex(123)), + badarg = ?MASK_ERROR(binary:encode_hex([])), + badarg = ?MASK_ERROR(binary:encode_hex(#{})), + badarg = ?MASK_ERROR(binary:encode_hex(foo)), ok. %% Whitebox test to force special trap conditions in @@ -1455,10 +1460,42 @@ error_info(_Config) -> {split, [<<1:15>>, <<"a">>, []]}, {split, [<<1,2,3>>, {bm,make_ref()}, []]}, - {split, [<<1,2,3>>, <<"2">>, [bad_option]]} + {split, [<<1,2,3>>, <<"2">>, [bad_option]]}, + + {encode_hex, [{no,a,binary}]}, + {decode_hex, [{no,a,binary}]}, + {decode_hex, [<<"000">>],[allow_rename]}, + {decode_hex, [<<"GG">>],[allow_rename]} ], error_info_lib:test_error_info(binary, L). +hex_encoding(Config) when is_list(Config) -> + %% Vector test imported from the RFC 4648 section 10. + <<>> = binary:encode_hex(<<>>), + <<"66">> = binary:encode_hex(<<"f">>), + <<"666F">> = binary:encode_hex(<<"fo">>), + <<"666F6F">> = binary:encode_hex(<<"foo">>), + <<"666F6F62">> = binary:encode_hex(<<"foob">>), + <<"666F6F6261">> = binary:encode_hex(<<"fooba">>), + <<"666F6F626172">> = binary:encode_hex(<<"foobar">>), + + <<>> = binary:decode_hex(<<>>), + <<"f">> = binary:decode_hex(<<"66">>), + <<"fo">> = binary:decode_hex(<<"666F">>), + <<"foo">> = binary:decode_hex(<<"666F6F">>), + <<"foob">> = binary:decode_hex(<<"666F6F62">>), + <<"fooba">> = binary:decode_hex(<<"666F6F6261">>), + <<"foobar">> = binary:decode_hex(<<"666F6F626172">>), + + <<"fo">> = binary:decode_hex(<<"666f">>), + <<"foo">> = binary:decode_hex(<<"666f6f">>), + <<"foob">> = binary:decode_hex(<<"666f6f62">>), + <<"fooba">> = binary:decode_hex(<<"666f6f6261">>), + <<"foobar">> = binary:decode_hex(<<"666f6f626172">>), + + <<"foobar">> = binary:decode_hex(<<"666f6F626172">>), + ok. + %%% %%% Utilities. %%% diff --git a/lib/stdlib/test/error_info_lib.erl b/lib/stdlib/test/error_info_lib.erl index d1afaf4589..aa4ef98dbc 100644 --- a/lib/stdlib/test/error_info_lib.erl +++ b/lib/stdlib/test/error_info_lib.erl @@ -77,12 +77,14 @@ eval_bif_error(F, Args, Opts, T, Module, Errors0) -> end catch error:Reason:Stk -> + AllowRename = lists:member(allow_rename, Opts), SF = fun(Mod, _, _) -> Mod =:= test_server end, Str = erl_error:format_exception(error, Reason, Stk, #{stack_trim_fun => SF}), BinStr = iolist_to_binary(Str), ArgStr = lists:join(", ", [io_lib:format("~p", [A]) || A <- Args]), io:format("\n~p:~p(~s)\n~ts", [Module,F,ArgStr,BinStr]), + case Stk of [{Module,ActualF,ActualArgs,Info}|_] -> RE = <<"[*][*][*] argument \\d+:">>, @@ -99,17 +101,20 @@ eval_bif_error(F, Args, Opts, T, Module, Errors0) -> [{no_explanation,{F,Args},Info}|Errors0] end end, - Errors = case {ActualF,ActualArgs} of {F,Args} -> Errors1; + _ when AllowRename -> + Errors1; _ -> [{renamed,{F,length(Args)},{ActualF,ActualArgs}}|Errors1] end, do_error_info(T, Module, Errors); - _ -> + _ when not AllowRename -> Errors = [{renamed,{F,length(Args)},hd(Stk)}|Errors0], - do_error_info(T, Module, Errors) + do_error_info(T, Module, Errors); + _ -> + do_error_info(T, Module, Errors0) end end. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index e2f1dd38c1..067828dc1a 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -31,6 +31,7 @@ -export([match_delete3/1]). -export([firstnext/1,firstnext_concurrent/1]). -export([slot/1]). +-export([hash_clash/1]). -export([match1/1, match2/1, match_object/1, match_object2/1]). -export([dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). -export([info_binary_stress/1]). @@ -132,7 +133,7 @@ suite() -> all() -> [{group, new}, {group, insert}, {group, lookup}, - {group, delete}, firstnext, firstnext_concurrent, slot, + {group, delete}, firstnext, firstnext_concurrent, slot, hash_clash, {group, match}, t_match_spec_run, {group, lookup_element}, {group, misc}, {group, files}, {group, heavy}, {group, insert_list}, ordered, ordered_match, @@ -4574,6 +4575,16 @@ slot_loop(Tab,SlotNo,EltsSoFar) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +hash_clash(Config) when is_list(Config) -> + %% ensure that erlang:phash2 and ets:slot use different hash seed + Tab = ets:new(tab, [set]), + Buckets = erlang:element(1, ets:info(Tab, stats)), + Phash = erlang:phash2(<<"123">>, Buckets), + true = ets:insert(Tab, {<<"123">>, "extra"}), + [] = ets:slot(Tab, Phash). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + match1(Config) when is_list(Config) -> repeat_for_opts_all_set_table_types(fun match1_do/1). diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index 5f8e568387..e234b0cd58 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.14 +STDLIB_VSN = 3.14.1 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index b7d304aabe..a1f280e594 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -32,6 +32,35 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 2.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p><c>epp_dodger</c> was unable to handle a parameterized + macro in a function head.</p> + <p> + Own Id: OTP-17064 Aux Id: GH-4445, PR-2964 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>All functions in <c>erl_tidy</c> in syntax_tools have + now been deprecated and are scheduled for removal in OTP + 24. Users who still need it can find it at <url + href="https://github.com/richcarl/erl_tidy">https://github.com/richcarl/erl_tidy</url>.</p> + <p> + Own Id: OTP-17167 Aux Id: OTP-17046 </p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 2.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 7f0b8f40dd..7a93e81b94 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 2.4 +SYNTAX_TOOLS_VSN = 2.5 diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 5b98d338b0..358bcf5043 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,40 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 3.4.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p><c>cover</c> would crash when compiling a module + having an exported function named <c>clauses</c>.</p> + <p> + Own Id: OTP-17162 Aux Id: GH-4549, PR-2997, PR-4555, + elixir-lang/elixir#10666 </p> + </item> + <item> + <p>If <c>beam_lib</c> is asked to return abstract code + for a BEAM file produced by Elixir and Elixir is not + installed on the computer, <c>beam_lib</c> will no longer + crash, but will return an error tuple. The + <c>cover:compile_beam()</c> and + <c>cover:compile_beam_directory()</c> functions have been + updated to also return an error tuple in that + situation.</p> + <p> + Own Id: OTP-17194 Aux Id: GH-4353 </p> + </item> + <item> + <p> + Make emacs mode work on emacs-27.</p> + <p> + Own Id: OTP-17225 Aux Id: PR-4542, GH-4451 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.4.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -107,6 +141,22 @@ </section> +<section><title>Tools 3.3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p><c>cover</c> would crash when compiling a module + having an exported function named <c>clauses</c>.</p> + <p> + Own Id: OTP-17162 Aux Id: GH-4549, PR-2997, PR-4555, + elixir-lang/elixir#10666 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 0a875414c5..69a5fdaeb5 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -55,10 +55,8 @@ ;; Reporting Bugs: ;; -------------- ;; -;; Please send bug reports to the following email address: -;; erlang-bugs@erlang.org -;; or if you have a patch suggestion to: -;; erlang-patches@erlang.org +;; Please report bugs at: +;; https://github.com/erlang/otp/issues ;; Please state as exactly as possible: ;; - Version number of Erlang Mode (see the menu), Emacs, Erlang, ;; and of any other relevant software. @@ -67,6 +65,8 @@ ;; - A description of the unexpected result. ;; - Relevant pieces of Erlang code causing the problem. ;; - Personal Emacs customisations, if any. +;; Patch suggestions are welcome at: +;; https://github.com/erlang/otp/pulls ;; ;; Should the Emacs generate an error, please set the Emacs variable ;; `debug-on-error' to `t'. Repeat the error and enclose the debug diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 6e4ee2fd14..33ff4b4a7c 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 3.4.3 +TOOLS_VSN = 3.4.4 diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index fe5a9966f8..62f333f0c1 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -32,6 +32,24 @@ <p>This document describes the changes made to the wxErlang application.</p> +<section><title>Wx 1.9.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed wx initialization on mac, top level menus did not + always work on newer MacOS versions. The menues will not + work until wxWidgets-3.1.5 is released and used on these + MacOS versions.</p> + <p> + Own Id: OTP-17187</p> + </item> + </list> + </section> + +</section> + <section><title>Wx 1.9.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index e2ff4812e9..42776cc17a 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.9.2 +WX_VSN = 1.9.3 diff --git a/otp_versions.table b/otp_versions.table index fd46539fba..f6ccfd0612 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,6 @@ +OTP-23.3.1 : ssh-4.11.1 # asn1-5.0.14 common_test-1.20 compiler-7.6.7 crypto-4.9 debugger-5.0 dialyzer-4.3.1 diameter-2.2.3 edoc-0.12 eldap-1.2.9 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11.1 kernel-7.3 megaco-3.19.5 mnesia-4.19 observer-2.9.5 odbc-2.13.3 os_mon-2.6.1 parsetools-2.2 public_key-1.10 reltool-0.8 runtime_tools-1.16 sasl-4.0.2 snmp-5.8 ssl-10.3 stdlib-3.14.1 syntax_tools-2.5 tftp-1.0.2 tools-3.4.4 wx-1.9.3 xmerl-1.3.26 : +OTP-23.3 : common_test-1.20 compiler-7.6.7 crypto-4.9 dialyzer-4.3.1 eldap-1.2.9 erts-11.2 jinterface-1.11.1 kernel-7.3 mnesia-4.19 odbc-2.13.3 public_key-1.10 runtime_tools-1.16 sasl-4.0.2 snmp-5.8 ssh-4.11 ssl-10.3 stdlib-3.14.1 syntax_tools-2.5 tools-3.4.4 wx-1.9.3 # asn1-5.0.14 debugger-5.0 diameter-2.2.3 edoc-0.12 erl_docgen-1.0.2 erl_interface-4.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 megaco-3.19.5 observer-2.9.5 os_mon-2.6.1 parsetools-2.2 reltool-0.8 tftp-1.0.2 xmerl-1.3.26 : +OTP-23.2.7.1 : ssl-10.2.4.1 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11 kernel-7.2.1 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.8 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.7 : kernel-7.2.1 ssl-10.2.4 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.2 jinterface-1.11 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.8 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.6 : inets-7.3.2 ssh-4.10.8 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.8 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssl-10.2.3 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.3 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.5 : erts-11.1.8 ssl-10.2.3 tools-3.4.3 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.7 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 wx-1.9.2 xmerl-1.3.26 : @@ -17,6 +20,7 @@ OTP-23.0.3 : compiler-7.6.2 erts-11.0.3 # asn1-5.0.13 common_test-1.19 crypto-4. OTP-23.0.2 : erts-11.0.2 megaco-3.19.1 # asn1-5.0.13 common_test-1.19 compiler-7.6.1 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0.1 : compiler-7.6.1 erts-11.0.1 # asn1-5.0.13 common_test-1.19 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0 : asn1-5.0.13 common_test-1.19 compiler-7.6 crypto-4.7 debugger-5.0 dialyzer-4.2 edoc-0.12 erl_docgen-1.0 erl_interface-4.0 erts-11.0 eunit-2.5 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tools-3.4 wx-1.9.1 xmerl-1.3.25 # diameter-2.2.3 eldap-1.2.8 et-1.6.4 ftp-1.0.4 reltool-0.8 tftp-1.0.2 : +OTP-22.3.4.17 : erts-10.7.2.9 kernel-6.5.2.2 ssh-4.9.1.3 tools-3.3.1.1 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 crypto-4.6.5.2 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.16 : erts-10.7.2.8 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 crypto-4.6.5.2 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.15 : crypto-4.6.5.2 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 erts-10.7.2.7 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.14 : compiler-7.5.4.3 erts-10.7.2.7 # asn1-5.0.12 common_test-1.18.2 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : @@ -65,6 +69,7 @@ OTP-22.0.3 : compiler-7.4.2 dialyzer-4.0.1 erts-10.4.2 ssl-9.3.2 stdlib-3.9.2 # OTP-22.0.2 : compiler-7.4.1 crypto-4.5.1 erts-10.4.1 stdlib-3.9.1 # asn1-5.0.9 common_test-1.17.3 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3.1 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.1 : ssl-9.3.1 # asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 stdlib-3.9 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0 : asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3 stdlib-3.9 syntax_tools-2.2 tools-3.2 wx-1.8.8 xmerl-1.3.21 # diameter-2.2.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2 parsetools-2.1.8 tftp-1.0.1 : +OTP-21.3.8.22 : erts-10.3.5.17 ssh-4.7.6.6 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.21 : erts-10.3.5.16 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.20 : erl_interface-3.11.3.1 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erts-10.3.5.15 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.19 : crypto-4.4.2.3 erts-10.3.5.15 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : diff --git a/scripts/build-otp b/scripts/build-otp index f3f7f8ee86..938a6809bb 100755 --- a/scripts/build-otp +++ b/scripts/build-otp @@ -60,7 +60,7 @@ if [ "$1" = "docs" ]; then do_and_log "Linting documentation" make xmllint do_and_log "Checking html links" scripts/otp_html_check $PWD/$DOC_TARGET doc/index.html do_and_log "Test chunks" erlc lib/stdlib/test/shell_docs_SUITE.erl && - ct_run -no_auto_compile -suite shell_docs_SUITE -case render + ct_run -no_auto_compile -suite shell_docs_SUITE -case render_smoke # The code below prepares this build to be used as a deploy to # github pages for documentation. if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_TAG" = "" -a "$TRAVIS_REPO_SLUG" = "erlang/otp" ]; then diff --git a/system/doc/general_info/Makefile b/system/doc/general_info/Makefile index f7e1a86c5a..7dc5811134 100644 --- a/system/doc/general_info/Makefile +++ b/system/doc/general_info/Makefile @@ -85,13 +85,13 @@ debug opt: DEPRECATIONS_SCRIPT = $(ERL_TOP)/lib/stdlib/scripts/update_deprecations -$(XMLDIR)/deprecations.xml: DEPRECATIONS +$(XMLDIR)/deprecations.xml: DEPRECATIONS $(wildcard deprecations_*.inc) $(gen_verbose)escript $(DEPRECATIONS_SCRIPT) make_xml deprecations $(ERL_TOP) $@ -$(XMLDIR)/scheduled_for_removal.xml: DEPRECATIONS +$(XMLDIR)/scheduled_for_removal.xml: DEPRECATIONS $(wildcard scheduled_*.inc) $(gen_verbose)escript $(DEPRECATIONS_SCRIPT) make_xml scheduled_for_removal $(ERL_TOP) $@ -$(XMLDIR)/removed.xml: DEPRECATIONS +$(XMLDIR)/removed.xml: DEPRECATIONS $(wildcard removed_*.inc) $(gen_verbose)escript $(DEPRECATIONS_SCRIPT) make_xml removed $(ERL_TOP) $@ clean clean_docs: diff --git a/system/doc/general_info/deprecations_22.inc b/system/doc/general_info/deprecations_22.inc index 53da6df594..57ba7d8f33 100644 --- a/system/doc/general_info/deprecations_22.inc +++ b/system/doc/general_info/deprecations_22.inc @@ -2,7 +2,8 @@ <title>VxWorks Support</title> <p>Some parts of OTP has had limited VxWorks support, such as for example <seeapp marker="erl_interface:index"><c>erl_interface</c></seeapp>. - This support is now deprecated and has also been scheduled for removal in OTP 23.</p> + This support is as of OTP 22 formally deprecated and has also been + <seeguide marker="removed#otp-23">removed in OTP 23</seeguide>.</p> </section> <section> <title>Legacy parts of erl_interface</title> @@ -13,7 +14,8 @@ the <c>erl_interface</c> library with the use of the <c>ei</c> library which also is part of the <c>erl_interface</c> application. The old legacy <seeapp marker="erl_interface:index"><c>erl_interface</c></seeapp> - library has also been scheduled for removal in OTP 23.</p> + library has also been + <seeguide marker="removed#otp-23">removed in OTP 23</seeguide>.</p> </section> <section> <title>System Events</title> diff --git a/system/doc/general_info/deprecations_23.inc b/system/doc/general_info/deprecations_23.inc index 0d968a537f..ef1c6ff12f 100644 --- a/system/doc/general_info/deprecations_23.inc +++ b/system/doc/general_info/deprecations_23.inc @@ -1,8 +1,7 @@ <section> <title>Crypto Old API</title> - <p>The <seeguide marker="crypto:new_api#the-old-api">Old API</seeguide> is now deprecated - and has also been - <seeguide marker="scheduled_for_removal#otp-24">scheduled for removal</seeguide>.</p> + <p>The <seeguide marker="crypto:new_api#the-old-api">Old API</seeguide> is deprecated + as of OTP 23 and has been <seeguide marker="removed#otp-24">removed in OTP 24</seeguide>.</p> <p>For replacement functions see the <seeguide marker="crypto:new_api#the-new-api">New API</seeguide>.</p> </section> @@ -35,8 +34,7 @@ As of OTP 23, the distributed <seeerl marker="kernel:disk_log"><c>disk_log</c></seeerl> feature has been deprecated and it has also been - <seeguide marker="scheduled_for_removal#otp-24">scheduled for removal</seeguide> - in OTP 24. + <seeguide marker="removed#otp-24">removed in OTP 24</seeguide>. </p> </section> @@ -45,6 +43,6 @@ <p> As of OTP 23, the <c>registry</c> functionality part of <c>erl_interface</c> has been deprecated and it has also been - removed in OTP 24. + <seeguide marker="removed#otp-24">removed in OTP 24</seeguide>. </p> </section> diff --git a/system/doc/general_info/deprecations_24.inc b/system/doc/general_info/deprecations_24.inc new file mode 100644 index 0000000000..0eb859789a --- /dev/null +++ b/system/doc/general_info/deprecations_24.inc @@ -0,0 +1,30 @@ + <section> + <title>Erlang Distribution Without Large Node Container Support</title> + <p> + Communication over the Erlang distribution without support for large + <seeguide marker="erts:erl_dist_protocol#DFLAG_V4_NC">node container + data types (version 4)</seeguide> is as of OTP 24 deprecated and is + <seeguide marker="scheduled_for_removal#otp-26">scheduled for removal + in OTP 26</seeguide>. That is, as of OTP 26, support for large + node container data types will become mandatory. + </p> + </section> + + <section> + <title>Old Link Protocol</title> + <p> + The <seeguide marker="erts:erl_dist_protocol#old_link_protocol">old + link protocol</seeguide> used when communicating over the Erlang + distribution is as of OTP 24 deprecated and support for it is + <seeguide marker="scheduled_for_removal#otp-26">scheduled for removal + in OTP 26</seeguide>. As of OTP 26, the + <seeguide marker="erts:erl_dist_protocol#new_link_protocol">new + link protocol</seeguide> will become mandatory. That is, Erlang nodes + will then refuse to connect to nodes not implementing the new + link protocol. If you implement the Erlang distribution yourself, you + are, however, encouraged to implement the new link protocol as soon as + possible since the old protocol can cause links to enter an + inconsistent state. + </p> + </section> + diff --git a/system/doc/general_info/scheduled_for_removal_23.inc b/system/doc/general_info/removed_23.inc index 692d51d09c..258218d460 100644 --- a/system/doc/general_info/scheduled_for_removal_23.inc +++ b/system/doc/general_info/removed_23.inc @@ -2,14 +2,15 @@ <title>VxWorks Support</title> <p>Some parts of OTP has had limited VxWorks support, such as <seeapp marker="erl_interface:index"><c>erl_interface</c></seeapp>. - This support will be removed as of OTP 23. This limited support + This support was removed in OTP 23. This limited support was formally deprecated as of OTP 22.</p> </section> + <section> <title>Legacy parts of erl_interface</title> <p>The old legacy <seeapp marker="erl_interface:index"><c>erl_interface</c></seeapp> - library (functions with prefix <c>erl_</c>) will be removed as of - OTP 23. These parts of <c>erl_interface</c> has been informally deprecated + library (functions with prefix <c>erl_</c>) was removed in OTP 23. + These parts of <c>erl_interface</c> has been informally deprecated for a very long time, and was formally deprecated in OTP 22. You typically want to replace the usage of the <c>erl_interface</c> library with the use of the <c>ei</c> library which also is part of the <c>erl_interface</c> @@ -28,7 +29,7 @@ <section> <title>inets - httpd Apache config files</title> <p> - Support for the Apache-compatible config files will be removed in + Support for the Apache-compatible config files was removed in OTP 23. A new config file format was introduced in OTP 12. </p> </section> diff --git a/system/doc/general_info/removed_24.inc b/system/doc/general_info/removed_24.inc index 37d514efa4..0274c8dec4 100644 --- a/system/doc/general_info/removed_24.inc +++ b/system/doc/general_info/removed_24.inc @@ -21,3 +21,29 @@ and <url href="https://github.com/richcarl/erl_tidy">github.com/richcarl/erl_tidy</url>, respectively.</p> </section> + <section> + <title>Distributed Disk Logs</title> + <p> + The distributed + <seeerl marker="kernel:disk_log"><c>disk_log</c></seeerl> + feature was as of + <seeguide marker="deprecations#otp-23">OTP 23 deprecated</seeguide> + and was removed in OTP 24. + </p> + </section> + <section> + <title>Old Crypto API</title> + <p>The <seeguide marker="crypto:new_api#the-old-api">Old API</seeguide> was + removed in OTP 24. The support was formally deprecated as of OTP 23.</p> + <p>For replacement functions see the + <seeguide marker="crypto:new_api#the-new-api">New API</seeguide>.</p> + </section> + <section> + <title>Megaco version 3 encoding config</title> + <p>The pre-release version 3 encoding configs; + <c>prev3a</c>, <c>prev3b</c> and <c>prev3c</c> was removed in OTP 24. + Use the full version instead. </p> + <p>The (encoding) config option for the full version, <c>{version3, 3}</c>, + will still be supported, even though its no longer necessary to specify + it this way. </p> + </section> diff --git a/system/doc/general_info/scheduled_for_removal_24.inc b/system/doc/general_info/scheduled_for_removal_24.inc deleted file mode 100644 index 6fddc033b6..0000000000 --- a/system/doc/general_info/scheduled_for_removal_24.inc +++ /dev/null @@ -1,26 +0,0 @@ - <section> - <title>Old Crypto API</title> - <p>The <seeguide marker="crypto:new_api#the-old-api">Old API</seeguide> will be - removed as of OTP 24. The support was formally deprecated as of OTP 23.</p> - <p>For replacement functions see the - <seeguide marker="crypto:new_api#the-new-api">New API</seeguide>.</p> - </section> - <section> - <title>Distributed Disk Logs</title> - <p> - The distributed - <seeerl marker="kernel:disk_log"><c>disk_log</c></seeerl> - feature is as of - <seeguide marker="deprecations#otp-23">OTP 23 deprecated</seeguide> - and will be removed in OTP 24. - </p> - </section> - <section> - <title>Megaco version 3 encoding config</title> - <p>As of OTP 24, the pre-release version 3 encoding configs; - <c>prev3a</c>, <c>prev3b</c> and <c>prev3c</c> will be removed. - Use the full version instead. </p> - <p>The (encoding) config option for the full version, <c>{version3, 3}</c>, - will still be supported, even though its no longer necessary to specify - it this way. </p> - </section> diff --git a/system/doc/general_info/scheduled_for_removal_26.inc b/system/doc/general_info/scheduled_for_removal_26.inc new file mode 100644 index 0000000000..1d59136ce5 --- /dev/null +++ b/system/doc/general_info/scheduled_for_removal_26.inc @@ -0,0 +1,30 @@ + <section> + <title>Erlang Distribution Without Large Node Container Support</title> + <p> + Communication over the Erlang distribution without support for large + <seeguide marker="erts:erl_dist_protocol#DFLAG_V4_NC"> node container + data types (version 4)</seeguide> is as of + <seeguide marker="deprecations#otp-24">OTP 24 deprecated</seeguide> + and support for it is scheduled for removal in OTP 26. That is, as + of OTP 26, support for large node container data types will become + mandatory. + </p> + </section> + + <section> + <title>Old Link Protocol</title> + <p> + The <seeguide marker="erts:erl_dist_protocol#old_link_protocol">old + link protocol</seeguide> used when communicating over the Erlang + distribution is as of <seeguide marker="deprecations#otp-24"> + OTP 24 deprecated</seeguide> and support for it is scheduled for + removal in OTP 26. As of OTP 26 the + <seeguide marker="erts:erl_dist_protocol#new_link_protocol">new + link protocol</seeguide> will become mandatory. That is, Erlang nodes + will then refuse to connect to nodes not implementing the new + link protocol. If you implement the Erlang distribution yourself, you + are, however, encouraged to implement the new link protocol as soon as + possible since the old protocol can cause links to enter an + inconsistent state. + </p> + </section> diff --git a/system/doc/reference_manual/expressions.xml b/system/doc/reference_manual/expressions.xml index 38b8a7a241..b21ab8eb96 100644 --- a/system/doc/reference_manual/expressions.xml +++ b/system/doc/reference_manual/expressions.xml @@ -402,12 +402,15 @@ Expr1 ! Expr2</pre> <p>Sends the value of <c>Expr2</c> as a message to the process specified by <c>Expr1</c>. The value of <c>Expr2</c> is also the return value of the expression.</p> - <p><c>Expr1</c> must evaluate to a pid, a registered name (atom), or + <p><c>Expr1</c> must evaluate to a pid, an alias (reference), + a port, a registered name (atom), or a tuple <c>{Name,Node}</c>. <c>Name</c> is an atom and <c>Node</c> is a node name, also an atom.</p> <list type="bulleted"> <item>If <c>Expr1</c> evaluates to a name, but this name is not registered, a <c>badarg</c> run-time error occurs.</item> + <item>Sending a message to a reference never fails, even if the + reference is no longer (or never was) an alias.</item> <item>Sending a message to a pid never fails, even if the pid identifies a non-existing process.</item> <item>Distributed message sending, that is, if <c>Expr1</c> diff --git a/system/doc/reference_manual/processes.xml b/system/doc/reference_manual/processes.xml index 2b68be4b31..597258b9fd 100644 --- a/system/doc/reference_manual/processes.xml +++ b/system/doc/reference_manual/processes.xml @@ -122,14 +122,39 @@ spawn(Module, Name, Args) -> pid() <section> <title>Links</title> - <p>Two processes can be <em>linked</em> to each other. A link - between two processes <c>Pid1</c> and <c>Pid2</c> is created - by <c>Pid1</c> calling the BIF <c>link(Pid2)</c> (or conversely). - There also exist a number of <c>spawn_link</c> BIFs, which spawn - and link to a process in one operation.</p> - <p>Links are bidirectional and there can only be one link between - two processes. Repeated calls to <c>link(Pid)</c> have no effect.</p> - <p>A link can be removed by calling the BIF <c>unlink(Pid)</c>.</p> + <p> + Two processes can be <em>linked</em> to each other. Also a + process and a port that reside on the same node can be linked + to each other. A link beteen two processes can be created + if one of them calls the + <seemfa marker="erts:erlang#link/1"><c>link/1</c></seemfa> BIF + with the process identifier of the other process as argument. + Links can also be created using one the following spawn BIFs + <seemfa marker="erts:erlang#spawn_link/4"><c>spawn_link()</c></seemfa>, + <seemfa marker="erts:erlang#spawn_opt/5"><c>spawn_opt()</c></seemfa>, or + <seemfa marker="erts:erlang#spawn_request/5"><c>spawn_request()</c></seemfa>. + The spawn operation and the link operation will + be performed atomically, in these cases. + </p> + <p> + If one of the participants of a link terminates, it will + <seeguide marker="system/reference_manual:processes#sending_exit_signals">send + an exit signal</seeguide> to the other participant. The exit + signal will contain the + <seeguide marker="system/reference_manual:processes#link_exit_signal_reason">exit + reason</seeguide> of the terminated participant. + </p> + <p> + A link can be removed by calling the + <seemfa marker="erts:erlang#unlink/1"><c>unlink/1</c></seemfa> + BIF. + </p> + <p> + Links are bidirectional and there can only be one link between + two processes. Repeated calls to <c>link()</c> have no effect. + Either one of the involved processes may create or remove a + link. + </p> <p>Links are used to monitor the behaviour of other processes, see <seeguide marker="#errors">Error Handling</seeguide>.</p> </section> @@ -149,35 +174,198 @@ spawn(Module, Name, Args) -> pid() OTP supervision trees, which use this feature.</p> <section> - <title>Emitting Exit Signals</title> - <p>When a process terminates, it terminates with an - <em>exit reason</em> as explained in <seeguide marker="#term"> - Process Termination</seeguide>. This exit reason is emitted in - an <em>exit signal</em> to all linked processes.</p> - <p>A process can also call the function <c>exit(Pid,Reason)</c>. - This results in an exit signal with exit reason - <c>Reason</c> being emitted to <c>Pid</c>, but does not affect - the calling process.</p> + <marker id="sending_exit_signals"/> + <title>Sending Exit Signals</title> + <p> + When a process or port + <seeguide marker="#term">terminates</seeguide> it will + send exit signals to all processes and ports that it + is <seeguide marker="#links">linked</seeguide> to. + The exit signal will contain the following information: + </p> + <taglist> + <tag>Sender identifier</tag> + <item><p> + The process or port identifier of the process or port + that terminated. + </p></item> + <tag>Receiver identifier</tag> + <item><p> + The process or port identifier of the process or port + which the exit signal is sent to. + </p></item> + <tag>The <c>link</c> flag</tag> + <item><p> + This flag will be set indicating that the exit signal + was sent due to a link. + </p></item> + <tag><marker id="link_exit_signal_reason"/>Exit reason</tag> + <item><p> + The exit reason of the process or port that + terminated or the atom:</p> + <list> + <item><p> + <c>noproc</c> in case no process or port was + found when setting up a link in a preceeding + call to the + <seemfa marker="erts:erlang#link/1"><c>link(PidOrPort)</c></seemfa> + BIF. The process or port identified as sender + of the exit signal will equal the <c>PidOrPort</c> + argument passed to <c>link/1</c>. + </p></item> + <item><p> + <c>noconnection</c> in case the linked + processes resides on different nodes and + the connection between the nodes was lost or + could not be established. The process or port + identified as sender of the exit signal might + in this case still be alive. + </p></item> + </list> + </item> + </taglist> + + <p> + Exit signals can also be sent explicitly by calling the + <seemfa marker="erts:erlang#exit/2"><c>exit(PidOrPort, + Reason)</c></seemfa> BIF. The exit signal is sent to the + process or port identified by the <c>PidOrPort</c> argument. + The exit signal sent will contain the following information: + </p> + <taglist> + <tag>Sender identifier</tag> + <item><p> + The process identifier of the process that called + <c>exit/2</c>. + </p></item> + <tag>Receiver identifier</tag> + <item><p> + The process or port identifier of the process or port + which the exit signal is sent to. + </p></item> + <tag>The <c>link</c> flag</tag> + <item><p> + This flag will not be set, indicating that this exit + signal was not sent due to a link. + </p></item> + <tag>Exit reason</tag> + <item><p> + The term passed as <c>Reason</c> in the call to + <c>exit/2</c>. If <c>Reason</c> is the atom <c>kill</c>, + the receiver cannot + <seeerl marker="erts:erlang#process_flag_trap_exit">trap + the exit</seeerl> signal and will unconditionally + terminate when it receives the signal. + </p></item> + </taglist> </section> <section> + <marker id="receiving_exit_signals"/> <title>Receiving Exit Signals</title> - <p>The default behaviour when a process receives an exit signal - with an exit reason other than <c>normal</c>, is to terminate - and in turn emit exit signals with the same exit reason to its - linked processes. An exit signal with reason <c>normal</c> is - ignored.</p> - <p>A process can be set to trap exit signals by calling:</p> - <pre> -process_flag(trap_exit, true)</pre> - <p>When a process is trapping exits, it does not terminate when - an exit signal is received. Instead, the signal is transformed - into a message <c>{'EXIT',FromPid,Reason}</c>, which is put into - the mailbox of the process, just like a regular message.</p> - <p>An exception to the above is if the exit reason is <c>kill</c>, - that is if <c>exit(Pid,kill)</c> has been called. This - unconditionally terminates the process, regardless of if it is - trapping exit signals.</p> + + <p>What happens when a process receives an exit signal depends on:</p> + <list> + <item><p> + The <seeerl marker="erts:erlang#process_flag_trap_exit">trap exit</seeerl> + state of the receiver at the time when the exit signal is received. + </p></item> + <item><p> + The exit reason of the exit signal. + </p></item> + <item><p> + The sender of the exit signal. + </p></item> + <item><p> + The state of the <c>link</c> flag of the exit signal. If the + <c>link</c> flag is set, the exit signal was sent due to a + link; otherwise, the exit signal was sent by a call to the + <seemfa marker="erts:erlang#exit/2"><c>exit/2</c></seemfa> BIF. + </p></item> + <item><p> + If the <c>link</c> flag is set, what happens also depends on + whether the <seemfa marker="erts:erlang#unlink/1">link is still active + or not</seemfa> when the exit signal is received. + </p></item> + </list> + + <p> + Based on the above states, the following will happen when an + exit signal is received by a process: + </p> + <list> + <item> + <p>The exit signal is silently dropped if:</p> + <list> + <item><p> + the <c>link</c> flag of the exit signal is set and + the corresponding link has been deactivated. + </p></item> + <item><p> + the exit reason of the exit signal is the atom <c>normal</c>, + the receiver is not trapping exits, and the receiver and + sender are not the same process. + </p></item> + </list> + </item> + <item> + <p>The receiving process is terminated if:</p> + <list> + <item><p> + the <c>link</c> flag of the exit signal + is not set, and the exit reason of the exit signal + is the atom <c>kill</c>. The receiving process will + terminate with the atom <c>killed</c> as exit reason. + </p></item> + <item><p> + the receiver is not trapping exits, and the exit + reason is something other than the atom <c>normal</c>. + Also, if the <c>link</c> flag of the exit signal + is set, the link also needs to be active otherwise the + exit signal will be dropped. The exit reason of the + receiving process will equal the exit reason of the + exit signal. Note that if the <c>link</c> flag + is set, an exit reason of <c>kill</c> will <em>not</em> + be converted to <c>killed</c>. + </p></item> + <item><p> + the exit reason of the exit signal is the atom + <c>normal</c> and the sender of the exit signal is + the same process as the receiver. The <c>link</c> + flag cannot be set in this case. The exit reason + of the receiving process will be the atom <c>normal</c>. + </p></item> + </list> + </item> + <item> + <p> + The exit signal is converted to a message signal and + moved into the message queue of the receiver, if the + receiver is trapping exits, the <c>link</c> flag + of the exit signal is: + </p> + <list> + <item><p> + not set, and the exit reason of the signal is not + the atom <c>kill</c>. + </p></item> + <item><p> + set, and the corresponding link is active. + Note that an exit reason of <c>kill</c> will + <em>not</em> terminate the process in this + case and it will not be converted to + <c>killed</c>. + </p></item> + </list> + <p> + The converted message will be on the form + <c>{'EXIT', SenderID, Reason}</c> where <c>Reason</c> + equals the exit reason of the exit signal and + <c>SenderID</c> is the identifier of the process + or port that sent the exit signal. + </p> + </item> + </list> </section> </section> @@ -216,4 +404,3 @@ erase(Key) erase()</pre> </section> </chapter> - |