diff options
248 files changed, 8973 insertions, 2697 deletions
diff --git a/.github/dockerfiles/Dockerfile.cross-compile b/.github/dockerfiles/Dockerfile.cross-compile index 868f8beed1..98f7f0e576 100644 --- a/.github/dockerfiles/Dockerfile.cross-compile +++ b/.github/dockerfiles/Dockerfile.cross-compile @@ -3,9 +3,6 @@ ## FROM docker.pkg.github.com/erlang/otp/i386-debian-base as build -ARG HOST_ARCH=amd64 -ARG HOST_TRIP=x86_64-linux-gnu - ARG MAKEFLAGS=-j4 ENV MAKEFLAGS=$MAKEFLAGS \ ERLC_USE_SERVER=yes \ @@ -21,7 +18,8 @@ WORKDIR /buildroot/otp/ RUN ./configure && make && make install ## Build pre-build tar ball -RUN scripts/build-otp-tar -o /buildroot/otp_clean_src.tar.gz /buildroot/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz +RUN scripts/build-otp-tar -o /buildroot/otp_clean_src.tar.gz /buildroot/otp_src.tar.gz \ + -b /buildroot/otp/ /buildroot/otp.tar.gz ## Prepare for a new build using pre-built tar ball RUN cd .. && rm -rf otp && tar -xzf ./otp_src.tar.gz @@ -43,7 +41,9 @@ RUN ./configure --prefix=/otp/ --host=$HOST --build=`erts/autoconf/config.guess` ## Build the cross tests RUN ./otp_build tests RUN cd release/tests/test_server && \ - erl -sname test@docker -noshell -eval "ts:install([{cross,\"yes\"},{crossflags,[{\"host\",\"$HOST\"}]},{crossroot,\"/$ERL_TOP\"}])." -s ts compile_testcases -s init stop + erl -sname test@docker -noshell \ + -eval "ts:install([{cross,\"yes\"},{crossflags,[{\"host\",\"$HOST\"}]},{crossroot,\"/$ERL_TOP\"}])." \ + -s ts compile_testcases -s init stop FROM debian as install diff --git a/.github/dockerfiles/Dockerfile.debian-base b/.github/dockerfiles/Dockerfile.debian-base index 1c26677959..416edd97c9 100644 --- a/.github/dockerfiles/Dockerfile.debian-base +++ b/.github/dockerfiles/Dockerfile.debian-base @@ -6,29 +6,38 @@ FROM $BASE ## Need to have a second arg here as the first does not expose the $BASE in the script below ARG BASE=debian -ARG HOST_ARCH=amd64 ARG HOST_TRIP=x86_64-linux-gnu +ENV HOST_TRIP=$HOST_TRIP -ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssh-dev unixodbc-dev libgmp3-dev libwxbase3.0-dev libwxgtk3.0-dev libsctp-dev lksctp-tools" +ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssh-dev unixodbc-dev libgmp3-dev libwxbase3.0-dev libwxgtk3.0-dev libwxgtk-webview3.0-gtk3-dev libsctp-dev lksctp-tools" ## See https://wiki.debian.org/Multiarch/HOWTO for details on how to install things -RUN if [ "$BASE" = "i386/debian" ]; then BUILD_ARCH=`dpkg --print-architecture` && \ - dpkg --add-architecture $HOST_ARCH && \ - sed -i "s:deb http:deb [arch=$BUILD_ARCH,$HOST_ARCH] http:g" /etc/apt/sources.list; \ - fi - -RUN apt-get update && \ - apt-get -y upgrade && \ - apt-get install -y build-essential m4 autoconf fop xsltproc \ - default-jdk libxml2-utils $INSTALL_LIBS - -RUN if [ "$BASE" = "i386/debian" ]; then apt-get install -y \ - crossbuild-essential-$HOST_ARCH \ - $(for LIB in $INSTALL_LIBS; do echo "$LIB:$HOST_ARCH"; done) && \ - for dir in `find / -type d -name $HOST_TRIP`; do \ - echo -n "$dir: /buildroot/sysroot"; \ - echo `dirname $dir`; \ - mkdir -p /buildroot/sysroot$dir; \ - cp -r `dirname $dir`/* `dirname /buildroot/sysroot$dir`; \ - cp -r $dir/* `dirname /buildroot/sysroot$dir`; \ - done; fi +## +## 1. Install build-essential to get access to dpkg-architecture +## 2. Use dpkg-architecture to figure out what we are runnon on +## 3. If the HOST_TRIP does not equal BUILD_TRIP we should cross compile +RUN apt-get update && apt-get -y upgrade && apt-get install -y build-essential && \ + BUILD_TRIP=`dpkg-architecture -t${HOST_TRIP} -qDEB_BUILD_MULTIARCH` && \ + BUILD_ARCH=`dpkg-architecture -t${HOST_TRIP} -qDEB_BUILD_ARCH` && \ + if [ "$HOST_TRIP" != "$BUILD_TRIP" ]; then \ + HOST_ARCH=`dpkg-architecture -t${HOST_TRIP} -qDEB_HOST_ARCH` && \ + dpkg --add-architecture $HOST_ARCH && \ + sed -i "s:deb http:deb [arch=$BUILD_ARCH,$HOST_ARCH] http:g" /etc/apt/sources.list; \ + fi && \ + apt-get update && \ + apt-get install -y build-essential m4 autoconf fop xsltproc default-jdk libxml2-utils \ + $INSTALL_LIBS && \ + if [ "$HOST_TRIP" != "$BUILD_TRIP" ]; then \ + apt-get install -y \ + crossbuild-essential-$HOST_ARCH \ + $(for LIB in $INSTALL_LIBS; do echo "$LIB:$HOST_ARCH"; done) && \ + for dir in `find / -type d -name $HOST_TRIP`; do \ + echo -n "$dir: /buildroot/sysroot"; \ + echo `dirname $dir`; \ + mkdir -p /buildroot/sysroot$dir; \ + cp -r `dirname $dir`/* `dirname /buildroot/sysroot$dir`; \ + cp -r $dir/* `dirname /buildroot/sysroot$dir`; \ + done; \ + fi && \ + update-alternatives --set wx-config /usr/lib/${BUILD_TRIP}/wx/config/gtk3-unicode-3.0 && \ + rm -rf /var/lib/apt/lists/* diff --git a/.github/scripts/base-tag b/.github/scripts/base-tag new file mode 100755 index 0000000000..6683793762 --- /dev/null +++ b/.github/scripts/base-tag @@ -0,0 +1,20 @@ +#!/bin/bash + +set -x + +case "$1" in + *i386-debian-base) + BASE="i386/debian" + BASE_TYPE=debian-base + ;; + *debian-base) + BASE="debian" + BASE_TYPE=debian-base + ;; + *ubuntu-base) + BASE="ubuntu" + BASE_TYPE=ubuntu-base + ;; +esac +echo "::set-output name=BASE::${BASE}" +echo "::set-output name=BASE_TYPE::${BASE_TYPE}" diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1eddf8f43c..988bf3b07e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -10,6 +10,12 @@ ## Also once the windows runner supports WSL we should implement ## support for building Erlang/OTP here. ## +## When ghcr.io support using the GITHUB_TOKEN we should migrate +## over to use it instead as that should allow us to use the +## built-in caching mechanisms of docker/build-push-action@v2. +## However as things are now we use docker directly to make things +## work. +## name: Build and check Erlang/OTP @@ -62,20 +68,35 @@ jobs: uses: actions/download-artifact@v2 with: name: otp_git_archive - ## We need to login to the package registry in order to pull - ## the base debian image. - name: Docker login - run: docker login https://docker.pkg.github.com -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + uses: docker/login-action@v1 + with: + registry: docker.pkg.github.com + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Calculate BASE image + id: base + run: | + BASE_TAG=$(grep "^FROM" .github/dockerfiles/Dockerfile.${{ matrix.type }} | head -1 | awk '{print $2}') + echo "::set-output name=BASE_TAG::${BASE_TAG}" + .github/scripts/base-tag "${BASE_TAG}" + - name: Pull BASE image + run: docker pull ${{ steps.base.outputs.BASE_TAG }} + - name: Build BASE image + run: | + docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \ + --cache-from ${{ steps.base.outputs.BASE_TAG }} \ + --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \ + --build-arg BASE=${{ steps.base.outputs.BASE }} . - name: Build ${{ matrix.type }} image run: | - docker build -t otp --build-arg ARCHIVE=otp_src.tar.gz \ - -f .github/dockerfiles/Dockerfile.${{ matrix.type }} . + docker build --tag otp --file .github/dockerfiles/Dockerfile.${{ matrix.type }} \ + --build-arg ARCHIVE=otp_src.tar.gz . ## Smoke build tests - if: matrix.type == '32-bit' || matrix.type == '64-bit' || matrix.type == 'cross-compile' name: Run smoke test - run: | - docker run -v $PWD/scripts:/scripts otp "cd /tests && /scripts/run-smoke-tests" + run: docker run -v $PWD/scripts:/scripts otp "cd /tests && /scripts/run-smoke-tests" ## Documentation checks - if: matrix.type == 'documentation' @@ -162,7 +183,7 @@ jobs: name: otp_doc_man ## We add the correct version name into the file names - ## and create the MD5 file for all assets + ## and create the hash files for all assets - name: Create pre-build and doc archives run: | mkdir artifacts @@ -177,12 +198,13 @@ jobs: run: | scripts/bundle-otp ${{ steps.tag.outputs.tag }} - ## Create md5sum + ## Create hash files - name: Create pre-build and doc archives run: | shopt -s nullglob cd artifacts md5sum {*.tar.gz,*.txt} > MD5.txt + sha256sum {*.tar.gz,*.txt} > SHA256.txt - name: Upload pre-built and doc tar archives uses: softprops/action-gh-release@v1 diff --git a/.github/workflows/update-base.yaml b/.github/workflows/update-base.yaml index ed250262e7..6cf2eafac2 100644 --- a/.github/workflows/update-base.yaml +++ b/.github/workflows/update-base.yaml @@ -9,48 +9,33 @@ on: ## Build base images to be used by other github workflows jobs: - build-debian-64bit: + build: + name: Update base Erlang/OTP build images if: github.repository == 'erlang/otp' runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build and push 64-bit base image - uses: docker/build-push-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - dockerfile: .github/dockerfiles/Dockerfile.debian-base - repository: erlang/otp/debian-base - tags: latest - build-debian-32bit: - if: github.repository == 'erlang/otp' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build and push 32-bit base image - uses: docker/build-push-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - dockerfile: .github/dockerfiles/Dockerfile.debian-base - build_args: "BASE=i386/debian" - repository: erlang/otp/i386-debian-base - tags: latest + strategy: + matrix: + type: [debian-base,ubuntu-base,i386-debian-base] - build-ubuntu-64bit: - if: github.repository == 'erlang/otp' - runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Build and push 64-bit base image - uses: docker/build-push-action@v1 + - name: Docker login + uses: docker/login-action@v1 with: + registry: docker.pkg.github.com username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - dockerfile: .github/dockerfiles/Dockerfile.ubuntu-base - repository: erlang/otp/ubuntu-base - tags: latest + - name: Calculate BASE image + id: base + run: | + echo "::set-output name=BASE_TAG::docker.pkg.github.com/erlang/otp/${{ matrix.type }}" + .github/scripts/base-tag "${{ matrix.type }}" + - name: Build base image + run: | + docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \ + --cache-from ${{ steps.base.outputs.BASE_TAG }} \ + --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \ + --build-arg BASE=${{ steps.base.outputs.BASE }} . + - name: Push base image + run: docker push ${{ steps.base.outputs.BASE_TAG }} diff --git a/HOWTO/INSTALL-ANDROID.md b/HOWTO/INSTALL-ANDROID.md index 24e8b0d658..7d5af0b0ec 100644 --- a/HOWTO/INSTALL-ANDROID.md +++ b/HOWTO/INSTALL-ANDROID.md @@ -27,22 +27,76 @@ to generate the configure scripts. $ ./otp_build autoconf -Use the following when compiling a 64-bit version. +Use the following commands when compiling a 64-bit version. $ export NDK_ABI_PLAT=android21 # When targeting Android 5.0 Lollipop + + + $ # Either without OpenSSL support: + $ $ ./otp_build configure \ --xcomp-conf=./xcomp/erl-xcomp-arm64-android.conf \ --without-ssl -Use the following instead when compiling a 32-bit version. + $ # Or with OpenSSL linked statically: + $ + $ cd /path/to/OpenSSL/source/dir/built/for/android-arm64 + $ # First follow the NOTES.ANDROID build instructions from OpenSSL + $ + $ # Then to avoid the full installation of this cross-compiled build, + $ # manually create a 'lib' directory at the root of the OpenSSL directory + $ # (at the same level as 'include') and link 'libcrypto.a' inside it. + $ + $ mkdir lib + $ ln -s ../libcrypto.a lib/libcrypto.a + $ cd - # Return to the Erlang/OTP directory + $ + $ # This previous step is needed for the OpenSSL static linking to work as + $ # the --with-ssl option expects a path with both the 'lib' and 'include' + $ # directories. Otherwise the Erlang/OTP build will fallback to dynamic + $ # linking if it doesn't find 'libcrypto.a' in its expected location. + $ ./otp_build configure \ + --xcomp-conf=./xcomp/erl-xcomp-arm64-android.conf \ + --with-ssl=/path/to/OpenSSL/source/dir/built/for/android-arm64 \ + --disable-dynamic-ssl-lib + + +Use the following commands instead when compiling a 32-bit version. $ export NDK_ABI_PLAT=androideabi16 # When targeting Android 4.1 Jelly Bean + + + $ # Either without OpenSSL support: + $ $ ./otp_build configure \ --xcomp-conf=./xcomp/erl-xcomp-arm-android.conf \ --without-ssl + $ # Or with OpenSSL linked statically: + $ + $ cd /path/to/OpenSSL/source/dir/built/for/android-arm + $ # First follow the NOTES.ANDROID build instructions from OpenSSL + $ + $ # Then to avoid the full installation of this cross-compiled build, + $ # manually create a 'lib' directory at the root of the OpenSSL directory + $ # (at the same level as 'include') and link 'libcrypto.a' inside it. + $ + $ mkdir lib + $ ln -s ../libcrypto.a lib/libcrypto.a + $ cd - # Return to the Erlang/OTP directory + $ + $ # This previous step is needed for the OpenSSL static linking to work as + $ # the --with-ssl option expects a path with both the 'lib' and 'include' + $ # directories. Otherwise the Erlang/OTP build will fallback to dynamic + $ # linking if it doesn't find 'libcrypto.a' in its expected location. + $ ./otp_build configure \ + --xcomp-conf=./xcomp/erl-xcomp-arm-android.conf \ + --with-ssl=/path/to/OpenSSL/source/dir/built/for/android-arm \ + --disable-dynamic-ssl-lib + + ### Compile Erlang/OTP ### $ make noboot [-j4] diff --git a/HOWTO/INSTALL.md b/HOWTO/INSTALL.md index c39ff38b22..fab973e5e4 100644 --- a/HOWTO/INSTALL.md +++ b/HOWTO/INSTALL.md @@ -607,7 +607,7 @@ using the similar steps just described. $ (cd $ERL_TOP/erts/emulator && make $TYPE) -where `$TYPE` is `opt`, `gcov`, `gprof`, `debug`, `valgrind`, or `lcnt`. +where `$TYPE` is `opt`, `gcov`, `gprof`, `debug`, `valgrind`, `asan` or `lcnt`. These different beam types are useful for debugging and profiling purposes. diff --git a/HOWTO/TESTING.md b/HOWTO/TESTING.md index 020be0309c..7a7f6982f2 100644 --- a/HOWTO/TESTING.md +++ b/HOWTO/TESTING.md @@ -185,6 +185,52 @@ examine the results so far for the currently executing test suite (in R14B02 and later you want to open the `release/tests/test_server/all_runs.html` file to get to the currently running test) + +Run tests with Address Sanitizer +-------------------------------- + +First build emulator with `asan` build target. +See [$ERL_TOP/HOWTO/INSTALL.md][]. + +Set environment variable `ASAN_LOG_DIR` to the directory +where the error logs will be generated. + + export ASAN_LOG_DIR=$TESTROOT/test_server/asan_logs + mkdir $ASAN_LOG_DIR + +Set environment variable `TS_RUN_EMU` to `asan`. + + export TS_RUN_EMU=asan + +Then run the tests you want with `ts:run` as described above. Either +inspect the log files directly or use the script at +`$ERL_TOP/erts/emulator/asan/asan_logs_to_html` to read all log files +in `$ASAN_LOG_DIR` and distill them into one html page +`asan_summary.html`. Repeated reports from the same memory leak will +for example be ignored by the script and make it easier to analyze. + + +Run tests with Valgrind +----------------------- + +First make sure [valgrind][] is installed, then build OTP from source +and build the emulator with `valgrind` build target. See +[$ERL_TOP/HOWTO/INSTALL.md][]. + +Set environment variable `VALGRIND_LOG_DIR` to the directory +where the valgrind error logs will be generated. + + export VALGRIND_LOG_DIR=$TESTROOT/test_server/vg_logs + mkdir $VALGRIND_LOG_DIR + +Set environment variable `TS_RUN_EMU` to `valgrind`. + + export TS_RUN_EMU=valgrind + +Then run the tests you want with `ts:run` as described above and +inspect the log file(s) in `$VALGRIND_LOG_DIR`. + + [ct_run]: http://www.erlang.org/doc/man/ct_run.html [ct hook]: http://www.erlang.org/doc/apps/common_test/ct_hooks_chapter.html [$ERL_TOP/HOWTO/INSTALL.md]: INSTALL.md @@ -192,5 +238,6 @@ get to the currently running test) [common_test]: http://www.erlang.org/doc/man/ct.html [data_dir]: http://www.erlang.org/doc/apps/common_test/write_test_chapter.html#data_priv_dir [configuring the tests]: #configuring-the-test-environment + [valgrind]: https://valgrind.org [?TOC]: true diff --git a/OTP_VERSION b/OTP_VERSION index 3f833b5b53..f657ed524a 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -23.2.1 +23.2.5 diff --git a/bootstrap/bin/no_dot_erlang.boot b/bootstrap/bin/no_dot_erlang.boot Binary files differindex 607a341cd5..1434a96636 100644 --- a/bootstrap/bin/no_dot_erlang.boot +++ b/bootstrap/bin/no_dot_erlang.boot diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 607a341cd5..1434a96636 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 607a341cd5..1434a96636 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex e0574f3124..047f014022 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex 6b1b04e153..48dc4f8096 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index bae37d87d1..a85ab33c24 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -19,7 +19,7 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "7.6.5"}, + {vsn, "7.6.6"}, {modules, [ beam_a, beam_asm, diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex 66003e4474..ecabcd9569 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index 80f68e62d6..ac5d0dbc49 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -22,7 +22,7 @@ {application, kernel, [ {description, "ERTS CXC 138 10"}, - {vsn, "7.1"}, + {vsn, "7.2"}, {modules, [application, application_controller, application_master, diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex 5eddd13cfc..489c7e65b6 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex 9ef9945d43..90850f2828 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/stdlib.app b/bootstrap/lib/stdlib/ebin/stdlib.app index 62ad6044ba..11754bf301 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.app +++ b/bootstrap/lib/stdlib/ebin/stdlib.app @@ -20,7 +20,7 @@ %% {application, stdlib, [{description, "ERTS CXC 138 10"}, - {vsn, "3.13.2"}, + {vsn, "3.14"}, {modules, [array, base64, beam_lib, diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index d237f7ae08..cf22456518 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1374,28 +1374,55 @@ AC_DEFUN(ETHR_CHK_GCC_ATOMIC_OPS, ETHR_CHK_GCC_ATOMIC_OP__(__atomic_compare_exchange_n) ethr_have_gcc_native_atomics=no - ethr_arm_dbm_instr_val=0 + ethr_arm_dbm_sy_instr_val=0 + ethr_arm_dbm_st_instr_val=0 + ethr_arm_dbm_ld_instr_val=0 case "$GCC-$host_cpu" in yes-arm*) - AC_CACHE_CHECK([for ARM DMB instruction], ethr_cv_arm_dbm_instr, + AC_CACHE_CHECK([for ARM 'dmb sy' instruction], ethr_cv_arm_dbm_sy_instr, [ - ethr_cv_arm_dbm_instr=no + ethr_cv_arm_dbm_sy_instr=no AC_TRY_LINK([], [ __asm__ __volatile__("dmb sy" : : : "memory"); - __asm__ __volatile__("dmb st" : : : "memory"); ], - [ethr_cv_arm_dbm_instr=yes]) + [ethr_cv_arm_dbm_sy_instr=yes]) ]) - if test $ethr_cv_arm_dbm_instr = yes; then + if test $ethr_cv_arm_dbm_sy_instr = yes; then ethr_arm_dbm_instr_val=1 test $ethr_cv_64bit___atomic_compare_exchange_n = yes && ethr_have_gcc_native_atomics=yes + fi + AC_CACHE_CHECK([for ARM 'dmb st' instruction], ethr_cv_arm_dbm_st_instr, + [ + ethr_cv_arm_dbm_st_instr=no + AC_TRY_LINK([], + [ + __asm__ __volatile__("dmb st" : : : "memory"); + ], + [ethr_cv_arm_dbm_st_instr=yes]) + ]) + if test $ethr_cv_arm_dbm_sy_instr = yes; then + ethr_arm_dbm_st_instr_val=1 + fi + AC_CACHE_CHECK([for ARM 'dmb ld' instruction], ethr_cv_arm_dbm_ld_instr, + [ + ethr_cv_arm_dbm_ld_instr=no + AC_TRY_LINK([], + [ + __asm__ __volatile__("dmb ld" : : : "memory"); + ], + [ethr_cv_arm_dbm_ld_instr=yes]) + ]) + if test $ethr_cv_arm_dbm_ld_instr = yes; then + ethr_arm_dbm_ld_instr_val=1 fi;; *) ;; esac - AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION], [$ethr_arm_dbm_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM DMB instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION], [$ethr_arm_dbm_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb sy' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION], [$ethr_arm_dbm_st_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb st' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION], [$ethr_arm_dbm_ld_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb ld' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) test $ethr_cv_32bit___sync_val_compare_and_swap = yes && ethr_have_gcc_native_atomics=yes test $ethr_cv_64bit___sync_val_compare_and_swap = yes && @@ -1512,6 +1539,33 @@ AC_ARG_WITH(with_sparc_memory_order, AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], [specify sparc memory order (defaults to RMO)])) +AC_ARG_ENABLE(ppc-lwsync-instruction, +AS_HELP_STRING([--enable-ppc-lwsync-instruction], [enable use of powerpc lwsync instruction]) +AS_HELP_STRING([--disable-ppc-lwsync-instruction], [disable use of powerpc lwsync instruction]), +[ case "$enableval" in + no) enable_lwsync=no ;; + *) enable_lwsync=yes ;; + esac ], +[ + AC_CHECK_SIZEOF(void *) + case $host_cpu-$ac_cv_sizeof_void_p in + macppc-8|powerpc-8|ppc-8|powerpc64-8|ppc64-8|powerpc64le-8|ppc64le-8|"Power Macintosh"-8) + enable_lwsync=yes;; + *) + enable_lwsync=undefined;; + esac ]) + +case $enable_lwsync in + no) + AC_DEFINE(ETHR_PPC_HAVE_NO_LWSYNC, [1], [Define if you do not have the powerpc lwsync instruction]) + ;; + yes) + AC_DEFINE(ETHR_PPC_HAVE_LWSYNC, [1], [Define if you have the powerpc lwsync instruction]) + ;; + *) + ;; +esac + LM_CHECK_THR_LIB ERL_INTERNAL_LIBS @@ -2823,6 +2877,8 @@ AC_DEFUN([LM_HARDWARE_ARCH], [ ppc) ARCH=ppc;; ppc64) ARCH=ppc64;; ppc64le) ARCH=ppc64le;; + powerpc64) ARCH=ppc64;; + powerpc64le) ARCH=ppc64le;; "Power Macintosh") ARCH=ppc;; arm64) ARCH=arm64;; armv5b) ARCH=arm;; diff --git a/erts/configure.in b/erts/configure.in index 12dfb9b19b..94297df77e 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -3551,6 +3551,26 @@ AH_BOTTOM([ #endif ]) + +dnl ---------------------------------------------------------------------- +dnl Check for GCC diagnostic ignored "-Waddress-of-packed-member" +dnl ---------------------------------------------------------------------- +saved_CFLAGS="$CFLAGS" +CFLAGS="-Werror $CFLAGS" +AC_TRY_COMPILE([], + [_Pragma("GCC diagnostic push") + _Pragma("GCC diagnostic ignored \"-Waddress-of-packed-member\"") + _Pragma("GCC diagnostic pop") + ], + AC_DEFINE(HAVE_GCC_DIAG_IGNORE_WADDRESS_OF_PACKED_MEMBER,[1], + [define if compiler support _Pragma('GCC diagnostic ignored '-Waddress-of-packed-member'')])) +CFLAGS="$saved_CFLAGS" + + +dnl ---------------------------------------------------------------------- +dnl Enable any -Werror flags +dnl ---------------------------------------------------------------------- + if test "x$GCC" = xyes; then CFLAGS="$WERRORFLAGS $CFLAGS" fi diff --git a/erts/doc/src/erl_cmd.xml b/erts/doc/src/erl_cmd.xml index eb1e9e2de7..3dbcbe1973 100644 --- a/erts/doc/src/erl_cmd.xml +++ b/erts/doc/src/erl_cmd.xml @@ -371,7 +371,7 @@ the default. In <c><![CDATA[embedded]]></c> mode modules are not auto loaded. The latter is recommended when the boot script preloads all modules, as conventionally happens in OTP releases. See - <seeerl marker="kernel:code"><c>code(3)</c></seeerl></p>. + <seeerl marker="kernel:code"><c>code(3)</c></seeerl>.</p> </item> <tag><marker id="name"/><c><![CDATA[-name Name]]></c></tag> <item> @@ -1406,8 +1406,8 @@ <note> <p>This feature has been introduced as a temporary workaround for long-executing native code, and native code that does not - bump reductions properly in OTP. When these bugs have be fixed, - this flag will be removed.</p> + bump reductions properly in OTP. When these bugs have been + fixed, this flag will be removed.</p> </note> </item> <tag><marker id="+spp"/><c>+spp Bool</c></tag> @@ -1424,7 +1424,7 @@ <seeerl marker="erlang#open_port_parallelism"> <c>parallelism</c></seeerl> to <seemfa marker="erlang#open_port/2"> - <c>erlang:open_port/2</c></seemfa></p>. + <c>erlang:open_port/2</c></seemfa>.</p> </item> <tag><marker id="sched_thread_stack_size"/> <c><![CDATA[+sss size]]></c></tag> diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 8cb8e09615..a8ec5bbaeb 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -430,9 +430,6 @@ io:format("old/unused name ~ts at port ~p, fd = ~p ~n", <p>where n = <c>Length</c> - 1.</p> - <p>The current implementation of Erlang does not care if the connection - to the EPMD is broken.</p> - <p>The response for a <c>STOP_REQ</c> is as follows:</p> <table align="left"> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index d2a1581e35..57bff58db4 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -140,6 +140,21 @@ $> erl However, unused local stub functions will be optimized away by the compiler, causing loading of the NIF library to fail.</p> </note> + <warning> + <p> + There is a known limitation for Erlang fallback functions of NIFs. Avoid + functions involved in traversal of binaries by matching and + recursion. If a NIF is loaded over such function, binary arguments to + the NIF may get corrupted and cause VM crash or other misbehavior. + </p> + <p>Example of such bad fallback function:</p> + <code type="none"> +skip_until(Byte, <<Byte, Rest/binary>>) -> + Rest; +skip_until(Byte, <<_, Rest/binary>>) -> + skip_until(Byte, Rest). +</code> + </warning> </section> <section> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 1200afce20..04edba9bef 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,86 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 11.1.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug that could cause some work scheduled for + execution on scheduler threads to be delayed until other + similar work appeared. Beside delaying various cleanup of + internal data structures also the following could be + delayed:</p> <list> <item>Termination of a distribution + controller process</item> <item>Disabling of the + distribution on a node</item> <item>Gathering of memory + allocator information using the <c>instrument</c> + module</item> <item>Enabling, disabling, and gathering of + <c>msacc</c> information</item> <item>Delivery of + <c>'CHANGE'</c> messages when time offset is + monitored</item> <item>A call to + <c>erlang:cancel_timer()</c></item> <item>A call to + <c>erlang:read_timer()</c></item> <item>A call to + <c>erlang:statistics(io | garbage_collection | + scheduler_wall_time)</c></item> <item>A call to + <c>ets:all()</c></item> <item>A call to + <c>erlang:memory()</c></item> <item>A call to + <c>erlang:system_info({allocator | allocator_sizes, + _})</c></item> <item>A call to + <c>erlang:trace_delivered()</c></item> </list> <p>The bug + existed on runtime systems running on all types of + hardware except for x86/x86_64.</p> + <p> + Own Id: OTP-17185</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 11.1.7</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Make windows installer remove write access rights for non + admin users when installing to a non default directory. + Reduces the risk for DLL sideloading, but the user should + always be aware of the access rights for the + installation.</p> + <p> + Own Id: OTP-17097</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 11.1.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The <c>suspend_process()</c> and <c>resume_process()</c> + BIFs did not check their arguments properly which could + cause an emulator crash.</p> + <p> + Own Id: OTP-17080</p> + </item> + <item> + <p> + The runtime system would get into an infinite loop if the + runtime system was started with more than 1023 file + descriptors already open.</p> + <p> + Own Id: OTP-17088 Aux Id: ERIERL-580 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 11.1.5</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -1136,6 +1216,67 @@ </section> +<section><title>Erts 10.7.2.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug that could cause some work scheduled for + execution on scheduler threads to be delayed until other + similar work appeared. Beside delaying various cleanup of + internal data structures also the following could be + delayed:</p> <list> <item>Termination of a distribution + controller process</item> <item>Disabling of the + distribution on a node</item> <item>Gathering of memory + allocator information using the <c>instrument</c> + module</item> <item>Enabling, disabling, and gathering of + <c>msacc</c> information</item> <item>Delivery of + <c>'CHANGE'</c> messages when time offset is + monitored</item> <item>A call to + <c>erlang:cancel_timer()</c></item> <item>A call to + <c>erlang:read_timer()</c></item> <item>A call to + <c>erlang:statistics(io | garbage_collection | + scheduler_wall_time)</c></item> <item>A call to + <c>ets:all()</c></item> <item>A call to + <c>erlang:memory()</c></item> <item>A call to + <c>erlang:system_info({allocator | allocator_sizes, + _})</c></item> <item>A call to + <c>erlang:trace_delivered()</c></item> </list> <p>The bug + existed on runtime systems running on all types of + hardware except for x86/x86_64.</p> + <p> + Own Id: OTP-17185</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.7.2.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The <c>suspend_process()</c> and <c>resume_process()</c> + BIFs did not check their arguments properly which could + cause an emulator crash.</p> + <p> + Own Id: OTP-17080</p> + </item> + <item> + <p> + The runtime system would get into an infinite loop if the + runtime system was started with more than 1023 file + descriptors already open.</p> + <p> + Own Id: OTP-17088 Aux Id: ERIERL-580 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.7.2.6</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3095,6 +3236,75 @@ </section> +<section><title>Erts 10.3.5.16</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug that could cause some work scheduled for + execution on scheduler threads to be delayed until other + similar work appeared. Beside delaying various cleanup of + internal data structures also the following could be + delayed:</p> <list> <item>Termination of a distribution + controller process</item> <item>Disabling of the + distribution on a node</item> <item>Gathering of memory + allocator information using the <c>instrument</c> + module</item> <item>Enabling, disabling, and gathering of + <c>msacc</c> information</item> <item>Delivery of + <c>'CHANGE'</c> messages when time offset is + monitored</item> <item>A call to + <c>erlang:cancel_timer()</c></item> <item>A call to + <c>erlang:read_timer()</c></item> <item>A call to + <c>erlang:statistics(io | garbage_collection | + scheduler_wall_time)</c></item> <item>A call to + <c>ets:all()</c></item> <item>A call to + <c>erlang:memory()</c></item> <item>A call to + <c>erlang:system_info({allocator | allocator_sizes, + _})</c></item> <item>A call to + <c>erlang:trace_delivered()</c></item> </list> <p>The bug + existed on runtime systems running on all types of + hardware except for x86/x86_64.</p> + <p> + Own Id: OTP-17185</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.3.5.15</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed rare distribution bug in race between received + signal (link/monitor/spawn_request/spawn_reply) and + disconnection. Symptom: VM crash. Since: OTP 21.0.</p> + <p> + Own Id: OTP-16869 Aux Id: ERL-1337 </p> + </item> + <item> + <p> + The <c>suspend_process()</c> and <c>resume_process()</c> + BIFs did not check their arguments properly which could + cause an emulator crash.</p> + <p> + Own Id: OTP-17080</p> + </item> + <item> + <p> + The runtime system would get into an infinite loop if the + runtime system was started with more than 1023 file + descriptors already open.</p> + <p> + Own Id: OTP-17088 Aux Id: ERIERL-580 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.3.5.14</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 59bc1eecd2..98dd6ea669 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -137,6 +137,14 @@ TYPE_FLAGS = $(DEBUG_CFLAGS) -DVALGRIND -DNO_JUMP_TABLE ENABLE_ALLOC_TYPE_VARS += valgrind else +ifeq ($(TYPE),asan) +PURIFY = +TYPEMARKER = .asan +TYPE_FLAGS = $(DEBUG_CFLAGS) -fsanitize=address -fsanitize-recover=address -fno-omit-frame-pointer -DADDRESS_SANITIZER +LDFLAGS += -fsanitize=address +ENABLE_ALLOC_TYPE_VARS += asan +else + ifeq ($(TYPE),gprof) PURIFY = TYPEMARKER = .gprof @@ -181,6 +189,7 @@ endif endif endif endif +endif LIBS += $(TYPE_LIBS) diff --git a/erts/emulator/asan/asan_logs_to_html b/erts/emulator/asan/asan_logs_to_html new file mode 100755 index 0000000000..14c9b7fcde --- /dev/null +++ b/erts/emulator/asan/asan_logs_to_html @@ -0,0 +1,453 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +%% Parse address sanitizer log files generated from test runs with +%% with environment variables ASAN_LOG_DIR and TS_RUN_EMU=asan set. + +%% Repeated leak reports are ignored and additional leaks of same type +%% as seen before are identified as such. + +-mode(compile). + +main([]) -> + help(); +main(["--help"]) -> + help(); +main([OutDir]) -> + case os:getenv("ASAN_LOG_DIR") of + false -> + io:format(standard_error, + "\nMissing asan log directory argument and environment\n" + "variable ASAN_LOG_DIR is not set.\n\n",[]), + help(); + InDir -> + run(OutDir, InDir) + end; +main([OutDir, InDir]) -> + run(OutDir, InDir). + + +help() -> + io:format("\nSyntax: asan_log_to_html OutDir [InDir]\n" + "\nParses all address-sanetizer log files in InDir\n" + "and generates a summary file OutDir/asan_summary.html.\n" + "Environment variable ASAN_LOG_DIR is used if InDir\n" + "is not specified\n\n", []). + +-record(logacc, {srcfile, % full path of current log file + did_output = false, % output contribution from srcfile + obuf = [], % output buffer + app = none, % current application + app_err = 0, % nr of reports from application + tc_err = 0, % nr of reports from srcfile (test case) + app_stat_bytes = 0, % total leaked bytes from app + app_stat_blocks = 0, % total leaked blocks from app + app_stat_errors = 0}). % total errors from app + +run(OutDir, InDir) -> + StartTime = erlang:monotonic_time(millisecond), + + {ok, InFilesUS} = file:list_dir(InDir), + InFiles = lists:sort(InFilesUS), + + OutFile = filename:join(OutDir, "asan_summary.html"), + {ok, Out} = file:open(OutFile, [write]), + + ok = file:write(Out, <<"<!DOCTYPE html>\n" + "<html>\n" + "<head><title>Address Sanitizer</title>\n">>), + ok = file:write(Out, style_block()), + ok = file:write(Out, <<"</head><body>\n" + "<h1>Address Sanitizer</h1>\n">>), + + {_, _, LogAcc2} = + lists:foldl(fun(File, {LM, RegEx, LogAcc}) -> + analyze_log_file(Out, filename:join(InDir,File), + {LM, RegEx, LogAcc}) + end, + {#{}, none, #logacc{}}, + InFiles), + + LogAcc3 = app_end(Out, LogAcc2), + try_delete_srcfile(LogAcc3), + + Time = calendar:system_time_to_rfc3339(erlang:system_time(second), + [{time_designator, 32}]), + %%{_, _, ThisFile} = code:get_object_code(?MODULE), + ThisFile = escript:script_name(), + User = string:trim(os:cmd("whoami")), + {ok, Host} = inet:gethostname(), + Seconds = (erlang:monotonic_time(millisecond) - StartTime) / 1000, + ok = io:format(Out, "\n<hr><p><small>This page was generated ~s\n" + " by <tt>~s</tt>\n" + " run by ~s@~s in ~.1f seconds.</small></p>\n", + [Time, ThisFile, User, Host, Seconds]), + + ok = file:write(Out, script_block()), + ok = file:write(Out, <<"</body>\n</html>\n">>), + ok = file:close(Out), + io:format("Generated file ~s\n", [OutFile]), + ok. + +analyze_log_file(Out, SrcFile, {LeakMap0, RegEx0, LogAcc0}=Acc) -> + + #logacc{app=PrevApp} = LogAcc0, + + case filelib:is_regular(SrcFile) of + false -> + Acc; + true -> + FileName = filename:basename(SrcFile), + %%io:format("analyze ~s\n", [FileName]), + + %% Is it a new application? + LogAcc2 = case string:lexemes(FileName, "-") of + [_Exe, PrevApp | _] -> + try_delete_srcfile(LogAcc0), + LogAcc0#logacc{srcfile=SrcFile, + tc_err=0, + did_output=false}; + [_Exe, NewApp | _] -> + LogAcc1 = app_end(Out, LogAcc0), + try_delete_srcfile(LogAcc1), + LogAcc1#logacc{srcfile=SrcFile, + obuf=[], + app=NewApp, + app_err=0, + tc_err=0, + did_output=false, + app_stat_bytes=0, + app_stat_blocks=0, + app_stat_errors=0} + end, + + case LogAcc2#logacc.app_err of + truncated -> + {LeakMap0, RegEx0, LogAcc2}; + _ -> + {ok, Bin} = file:read_file(SrcFile), + match_loop(Out, Bin, RegEx0, LogAcc2, 0, [], LeakMap0) + end + end. + +-define(APP_ERR_LIMIT, 200). + +match_loop(Out, _, RegEx, #logacc{app_err=AppErr}=LogAcc0, _, _, LM) + when AppErr >= ?APP_ERR_LIMIT -> + + Txt = [io_format("<h2>WARNING!!! Log truncated for application ~p," + " more than ~p errors found.</h2>\n", + [LogAcc0#logacc.app, ?APP_ERR_LIMIT])], + LogAcc1 = log_error(Out, LogAcc0, Txt), + {LM, RegEx, LogAcc1#logacc{app_err=truncated}}; + +match_loop(Out, Bin, RegEx0, LogAcc0, PrevEnd, Unmatched0, LM0) -> + {Match, RegEx1} = + run_regex(Bin, RegEx0, + %% LeakReport + "(?:(Direct|Indirect) leak of ([0-9]+) byte\\(s\\) " + "in ([0-9]+) object\\(s\\) allocated from:\n" + "((?:[ \t]*#[0-9]+.+\n)+))" % Call stack + "|" + %% ErrorReport + "(?:(==ERROR: AddressSanitizer:.*\n" + "(?:.*\n)+?)" % any lines (non-greedy) + "^(?:==|--))" % stop at line begining with == or -- + "|" + %% Skipped + "(?:^[=-]+$)" % skip lines consisting only of = or - + "|" + "Objects leaked above:\n" % if LSAN_OPTIONS="report_objects=1" + "(?:0x.+\n)+" + "|" + "^\n", % empty lines + [multiline], + [{offset, PrevEnd}, {capture, all, index}]), + + + BP = fun(PartIx) -> binary:part(Bin, PartIx) end, + + case Match of + [ErrorReport, {-1,0}, {-1,0}, {-1,0}, {-1,0}, Captured] -> + {Start,MatchLen} = ErrorReport, + Txt = [io_format("<pre~s>\n", [style(error)]), + file_write(BP(Captured)), + io_format("</pre>\n", [])], + LogAcc1 = log_error(Out, LogAcc0, Txt), + Unmatched1 = [BP({PrevEnd, Start-PrevEnd}) | Unmatched0], + End = Start + MatchLen, + match_loop(Out, Bin, RegEx1, app_stats(LogAcc1,0,0,1), + End, Unmatched1, LM0); + + [LeakReport, TypeIx, BytesIx, BlocksIx, StackIx | _] -> + {Start, MatchLen} = LeakReport, + Bytes = binary_to_integer(BP(BytesIx)), + Blocks = binary_to_integer(BP(BlocksIx)), + End = Start + MatchLen, + Unmatched1 = [BP({PrevEnd, Start-PrevEnd})|Unmatched0], + TypeBin = BP(TypeIx), + + %% We indentify a leak by its type (direct or indirect) + %% and its full call stack. + Key = {TypeBin, BP(StackIx)}, + {LogAcc2, LM2} = + case lookup_leak(LM0, Key) of + undefined -> + %% A new leak + LM1 = insert_leak(LM0, Key, Bytes, Blocks), + Txt = [io_format("<pre~s>\n", [style(new, TypeBin)]), + file_write(BP(LeakReport)), + io_format("</pre>\n", [])], + LogAcc1 = log_error(Out, LogAcc0, Txt), + {app_stats(LogAcc1,Bytes,Blocks,0), LM1}; + + {Bytes, Blocks} -> + %% Exact same leak(s) repeated, ignore + {LogAcc0, LM0}; + + {OldBytes, OldBlocks} -> + %% More leaked bytes/blocks of same type&stack as before + LM1 = insert_leak(LM0, Key, Bytes, Blocks), + ByteDiff = Bytes - OldBytes, + BlockDiff = Blocks - OldBlocks, + Txt = [io_format("<pre~s>\n", [style(more, TypeBin)]), + io_format("More ~s leak of ~w(~w) byte(s) " + "in ~w(~w) object(s) allocated from:\n", + [TypeBin, ByteDiff, Bytes, BlockDiff, Blocks]), + file_write(BP(StackIx)), + io_format("</pre>\n", [])], + LogAcc1 = log_error(Out, LogAcc0, Txt), + {app_stats(LogAcc1, ByteDiff, BlockDiff, 0), LM1} + end, + match_loop(Out, Bin, RegEx1, LogAcc2, End, Unmatched1, LM2); + + [SkipLine] -> + {Start, MatchLen} = SkipLine, + %%nomatch = binary:match(BP(SkipLine), <<"\n">>), % Assert single line + End = Start + MatchLen, + Unmatched1 = [BP({PrevEnd, Start-PrevEnd})|Unmatched0], + match_loop(Out, Bin, RegEx1, LogAcc0, End, Unmatched1, LM0); + + nomatch -> + Unmatched1 = [BP({PrevEnd, byte_size(Bin)-PrevEnd}) | Unmatched0], + + LogAcc1 = + case iolist_size(Unmatched1) > 500 of + true -> + Txt = [io_format("<h2>WARNING!!! May be unmatched error reports" + " in file ~s:</h2>\n<pre>~s</pre>", + [LogAcc0#logacc.srcfile, Unmatched1])], + log_error(Out, LogAcc0, Txt); + false -> + LogAcc0 + end, + {LM0, RegEx1, LogAcc1} + end. + +lookup_leak(LeakMap, Key) -> + maps:get(Key, LeakMap, undefined). + +insert_leak(LeakMap, Key, Bytes, Blocks) -> + LeakMap#{Key => {Bytes, Blocks}}. + +log_error(_Out, #logacc{app_err=AppErr, tc_err=TcErr}=LogAcc, Txt0) -> + {DidTc, Txt1} = + case TcErr of + 0 -> + %% First error in test case, print test case header + SrcFile = LogAcc#logacc.srcfile, + TcFile = filename:basename(SrcFile), + Hdr = case string:lexemes(TcFile, "-") of + [_Exe, App, _Rest] -> + io_format("<h3>Before first test case of ~s</h3>\n", + [App]); + [_Exe, _App, "tc", Num, Mod, Rest] -> + [Func | _] = string:lexemes(Rest, "."), + io_format("<h3>Test case #~s ~s:~s</h3>\n", + [Num, Mod, Func]); + _ -> + io_format("<h3>Strange log file name '~s'</h3>\n", + [SrcFile]) + end, + {true, [Hdr | Txt0]}; + _ -> + {false, Txt0} + end, + LogAcc#logacc{app_err=AppErr+1, tc_err=TcErr+1, + obuf = [Txt1 | LogAcc#logacc.obuf], + did_output = (LogAcc#logacc.did_output or DidTc)}. + +app_stats(#logacc{}=LogAcc, Bytes, Blocks, Errors) -> + LogAcc#logacc{app_stat_bytes = LogAcc#logacc.app_stat_bytes + Bytes, + app_stat_blocks = LogAcc#logacc.app_stat_blocks + Blocks, + app_stat_errors = LogAcc#logacc.app_stat_errors + Errors}. + + +app_end(Out, LogAcc) -> + case LogAcc of + #logacc{app=none} -> + LogAcc; + #logacc{app_err=0} -> + ok = io:format(Out, "<button class=\"app_ok\" disabled>~s</button>\n", + [LogAcc#logacc.app]), + LogAcc#logacc{did_output = true}; + #logacc{} -> + %% Print red clickable app button with stats + %% and all the buffered logs. + ok = io:format(Out, "<button type=\"button\" class=\"app_err\">~s" + "<span class=\"stats\">Leaks: ~p bytes in ~p blocks</span>", + [LogAcc#logacc.app, + LogAcc#logacc.app_stat_bytes, + LogAcc#logacc.app_stat_blocks]), + case LogAcc#logacc.app_stat_errors of + 0 -> ignore; + _ -> + ok = io:format(Out, "<span class=\"stats\">Errors: ~p</span>", + [LogAcc#logacc.app_stat_errors]) + end, + ok = io:format(Out, "</button>\n" + "<div class=\"content\">", []), + + flush_obuf(Out, LogAcc#logacc.obuf), + + ok = io:format(Out, "<button type=\"button\" " + "class=\"app_err_end\">" + "end of ~s</button>\n", [LogAcc#logacc.app]), + ok = io:format(Out, "</div>", []), + LogAcc + end. + +flush_obuf(Out, Obuf) -> + flush_obuf_rev(Out, lists:reverse(Obuf)). + +flush_obuf_rev(_Out, []) -> ok; +flush_obuf_rev(Out, [Txt | T]) -> + [OutFun(Out) || OutFun <- Txt], + flush_obuf_rev(Out, T). + +io_format(Frmt, List) -> + fun(Out) -> io:format(Out, Frmt, List) end. + +file_write(Bin) -> + fun(Out) -> file:write(Out, Bin) end. + +style(error) -> + " style=\"background-color:Tomato;\"". + +style(new, <<"Direct">>) -> + " style=\"background-color:orange;\""; +style(new, <<"Indirect">>) -> + ""; +style(more, _) -> + " style=\"background-color:yellow;\"". + + +run_regex(Bin, none, RegExString, CompileOpts, RunOpts) -> + {ok, RegEx} = re:compile(RegExString, CompileOpts), + run_regex(Bin, RegEx, none, none, RunOpts); +run_regex(Bin, RegEx, _, _, RunOpts) -> + case re:run(Bin, RegEx, RunOpts) of + nomatch -> + {nomatch, RegEx}; + {match, Match} -> + {Match, RegEx} + end. + +try_delete_srcfile(LogAcc) -> + case LogAcc of + #logacc{srcfile=undefined} -> + ignore; + #logacc{did_output=false} -> + %% This file did not contribute any output. + %% Optimize future script invokations by removing it. + delete_file(LogAcc#logacc.srcfile); + _ -> + keep + end. + +delete_file(File) -> + case filelib:is_regular(File) of + true -> + io:format("Delete file ~p\n", [File]), + Dir = filename:dirname(File), + Name = filename:basename(File), + Trash = filename:join([Dir, "DELETED", Name]), + ok = filelib:ensure_dir(Trash), + ok = file:rename(File, Trash); + false -> + ignore + end. + +style_block() -> + <<"<style> + +.app_err, .app_err_end, .app_ok { + color: white; + padding: 10px; + /*border: none;*/ + text-align: left; + /*outline: none;*/ + font-size: 15px; +} + +.app_err { + width: 100%; + background-color: #D11; + cursor: pointer; +} +.app_err:hover { + background-color: #F11; +} +.app_err_end { + background-color: #D11; + cursor: pointer; +} +.app_err_end:hover { + background-color: #F11; +} + +.app_ok { + width: 100%; + background-color: #292; +} + +.stats { + font-style: italic; + margin-left: 50px; +} + +.content { + padding: 0 18px; + display: none; + overflow: hidden; + background-color: #f1f1f1; +} +</style> +">>. + +script_block() -> + <<"<script> +var app_err = document.getElementsByClassName(\"app_err\"); +var i; + +for (i = 0; i < app_err.length; i++) { + app_err[i].addEventListener(\"click\", function() { + var content = this.nextElementSibling; + if (content.style.display === \"block\") { + content.style.display = \"none\"; + } else { + content.style.display = \"block\"; + } + }); +} + +var app_err_end = document.getElementsByClassName(\"app_err_end\"); +for (i = 0; i < app_err_end.length; i++) { + app_err_end[i].addEventListener(\"click\", function() { + var content = this.parentElement; + content.style.display = \"none\"; + }); +} + +</script> +">>. diff --git a/erts/emulator/asan/suppress b/erts/emulator/asan/suppress new file mode 100644 index 0000000000..5625938f37 --- /dev/null +++ b/erts/emulator/asan/suppress @@ -0,0 +1,18 @@ +leak:erts_alloc_permanent_cache_aligned + +# Harmless leak of ErtsThrPrgrData from async threads in exiting emulator +leak:erts_thr_progress_register_unmanaged_thread + +# Block passed to sigaltstack() +leak:sys_thread_init_signal_stack + +#Copied from valgrind/suppress.standard: +#Crypto internal... loading gives expected errors when curves are tried. +#But including <openssl/err.h> and removing them triggers compiler errors on Windows +#fun:valid_curve +#fun:init_curves +leak:init_curve_types +#fun:init_algorithms_types +#fun:initialize +#fun:load +#fun:erts_load_nif diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 47150b2aea..4b90fff688 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -66,7 +66,7 @@ #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS -#if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) +#if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) || defined(ADDRESS_SANITIZER) #define AU_ALLOC_DEFAULT_ENABLE(X) 0 #else #define AU_ALLOC_DEFAULT_ENABLE(X) (X) @@ -289,7 +289,11 @@ static void set_default_literal_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); +#ifdef ADDRESS_SANITIZER + ip->enable = 0; +#else ip->enable = 1; +#endif ip->thr_spec = 0; ip->disable_allowed = 0; ip->thr_spec_allowed = 0; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index c13cf3f5b0..831e7ab0a7 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -358,24 +358,11 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \ (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) +#if !defined(VALGRIND) && !defined(ADDRESS_SANITIZER) + #define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0) -#define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ -ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) - -#define ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ) \ -static erts_spinlock_t NAME##_lck; \ -ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, \ - erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock", NIL, \ - ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\ - erts_spin_lock(&NAME##_lck), \ - erts_spin_unlock(&NAME##_lck)) - - -#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ) \ - ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ) - #define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK) \ ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, ILCK, LCK, ULCK) \ @@ -606,6 +593,69 @@ NAME##_free(TYPE *p) \ (char *) p); \ } +#else /* !defined(VALGRIND) && !defined(ADDRESS_SANITIZER) */ + +/* + * For VALGRIND and ADDRESS_SANITIZER we short circuit all preallocation + * with dummy wrappers around malloc and free. + */ + +#define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ + ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0) + +#define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK) \ +static void init_##NAME##_alloc(void) \ +{ \ +} \ +static ERTS_INLINE TYPE* NAME##_alloc(void) \ +{ \ + return malloc(sizeof(TYPE)); \ +} \ +static ERTS_INLINE void NAME##_free(TYPE *p) \ +{ \ + free((void *) p); \ +} + +#define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ) \ + ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) + +#define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \ +ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ) + +#define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ + ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) + +#define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ +void erts_##NAME##_pre_alloc_init_thread(void) \ +{ \ +} \ +static void init_##NAME##_alloc(int nthreads) \ +{ \ +} \ +static ERTS_INLINE TYPE* NAME##_alloc(void) \ +{ \ + return malloc(sizeof(TYPE)); \ +} \ +static ERTS_INLINE void NAME##_free(TYPE *p) \ +{ \ + free(p); \ +} + +#define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \ +static void init_##NAME##_alloc(void) \ +{ \ +} \ +static TYPE* NAME##_alloc(void) \ +{ \ + return (TYPE *) malloc(sizeof(TYPE)); \ +} \ +static int NAME##_free(TYPE *p) \ +{ \ + free(p); \ + return 1; \ +} + +#endif /* VALGRIND || ADDRESS_SANITIZER */ #ifdef DEBUG #define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2)) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index d65aa71085..d4d1264916 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -62,8 +62,11 @@ #endif #ifdef VALGRIND -#include <valgrind/valgrind.h> -#include <valgrind/memcheck.h> +# include <valgrind/valgrind.h> +# include <valgrind/memcheck.h> +#endif +#ifdef ADDRESS_SANITIZER +# include <sanitizer/lsan_interface.h> #endif static Export* alloc_info_trap = NULL; @@ -127,6 +130,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef VALGRIND " [valgrind-compiled]" #endif +#ifdef ADDRESS_SANITIZER + " [address-sanitizer]" +#endif #ifdef ERTS_FRMPTR " [frame-pointer]" #endif @@ -2120,6 +2126,28 @@ current_stacktrace(Process *p, ErtsHeapFactory *hfact, Process* rp, return res; } +#if defined(VALGRIND) || defined(ADDRESS_SANITIZER) +static int iolist_to_tmp_buf(Eterm iolist, char** bufp) +{ + ErlDrvSizeT buf_size = 1024; /* Try with 1KB first */ + char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); + ErlDrvSizeT r = erts_iolist_to_buf(iolist, (char*) buf, buf_size - 1); + if (ERTS_IOLIST_TO_BUF_FAILED(r)) { + erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (erts_iolist_size(iolist, &buf_size)) { + return 0; + } + buf_size++; + buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); + r = erts_iolist_to_buf(iolist, (char*) buf, buf_size - 1); + ASSERT(r == buf_size - 1); + } + buf[buf_size - 1 - r] = '\0'; + *bufp = buf; + return 1; +} +#endif + /* * This function takes care of calls to erlang:system_info/1 when the argument * is a tuple. @@ -2182,59 +2210,45 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ goto badarg; ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res); return ret; -#if defined(PURIFY) || defined(VALGRIND) - } else if (ERTS_IS_ATOM_STR("error_checker", sel) -#if defined(PURIFY) - || sel == am_purify -#elif defined(VALGRIND) - || ERTS_IS_ATOM_STR("valgrind", sel) + } else if (ERTS_IS_ATOM_STR("memory_checker", sel)) { + if (arity == 2 && ERTS_IS_ATOM_STR("test_leak", *tp)) { +#if defined(VALGRIND) || defined(ADDRESS_SANITIZER) + erts_alloc(ERTS_ALC_T_HEAP , 100); #endif - ) { - if (*tp == am_memory) { -#if defined(PURIFY) - BIF_RET(erts_make_integer(purify_new_leaks(), BIF_P)); -#elif defined(VALGRIND) -# ifdef VALGRIND_DO_ADDED_LEAK_CHECK + BIF_RET(am_ok); + } + else if (arity == 2 && ERTS_IS_ATOM_STR("test_overflow", *tp)) { + static int test[2]; + BIF_RET(make_small(test[2])); + } +#if defined(VALGRIND) || defined(ADDRESS_SANITIZER) + if (arity == 2 && *tp == am_running) { +# if defined(VALGRIND) + if (RUNNING_ON_VALGRIND) + BIF_RET(ERTS_MAKE_AM("valgrind")); +# elif defined(ADDRESS_SANITIZER) + BIF_RET(ERTS_MAKE_AM("asan")); +# endif + } + else if (arity == 2 && ERTS_IS_ATOM_STR("check_leaks", *tp)) { +# if defined(VALGRIND) +# ifdef VALGRIND_DO_ADDED_LEAK_CHECK VALGRIND_DO_ADDED_LEAK_CHECK; -# else +# else VALGRIND_DO_LEAK_CHECK; +# endif + BIF_RET(am_ok); +# elif defined(ADDRESS_SANITIZER) + __lsan_do_recoverable_leak_check(); + BIF_RET(am_ok); # endif - BIF_RET(make_small(0)); -#endif - } else if (*tp == am_fd) { -#if defined(PURIFY) - BIF_RET(erts_make_integer(purify_new_fds_inuse(), BIF_P)); -#elif defined(VALGRIND) - /* Not present in valgrind... */ - BIF_RET(make_small(0)); -#endif - } else if (*tp == am_running) { -#if defined(PURIFY) - BIF_RET(purify_is_running() ? am_true : am_false); -#elif defined(VALGRIND) - BIF_RET(RUNNING_ON_VALGRIND ? am_true : am_false); -#endif - } else if (is_list(*tp)) { -#if defined(PURIFY) -# define ERTS_ERROR_CHECKER_PRINTF purify_printf -#elif defined(VALGRIND) -# define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF -#endif - ErlDrvSizeT buf_size = 8*1024; /* Try with 8KB first */ - char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); - ErlDrvSizeT r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1); - if (ERTS_IOLIST_TO_BUF_FAILED(r)) { - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (erts_iolist_size(*tp, &buf_size)) { - goto badarg; - } - buf_size++; - buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); - r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1); - ASSERT(r == buf_size - 1); - } - buf[buf_size - 1 - r] = '\0'; - ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); + } +# if defined(VALGRIND) + if (arity == 3 && tp[0] == am_print && is_list(tp[1])) { + char* buf; + if (!iolist_to_tmp_buf(tp[1], &buf)) + goto badarg; + VALGRIND_PRINTF("%s\n", buf); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(am_true); #undef ERTS_ERROR_CHECKER_PRINTF @@ -2254,6 +2268,30 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else if (*tp == am_running) { BIF_RET(quantify_is_running() ? am_true : am_false); } +# endif +# if defined(ADDRESS_SANITIZER) + if (arity == 3 && ERTS_IS_ATOM_STR("log",tp[0]) && is_list(tp[1])) { + static char *active_log = NULL; + static int active_log_len; + Eterm ret = NIL; + char* buf; + if (!iolist_to_tmp_buf(tp[1], &buf)) + goto badarg; + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); /* random lock abuse */ + __sanitizer_set_report_path(buf); + if (active_log) { + Eterm *hp = HAlloc(BIF_P, 2 * active_log_len); + ret = erts_bld_string_n(&hp, 0, active_log, active_log_len); + erts_free(ERTS_ALC_T_DEBUG, active_log); + } + active_log_len = sys_strlen(buf); + active_log = erts_alloc(ERTS_ALC_T_DEBUG, active_log_len + 1); + sys_memcpy(active_log, buf, active_log_len + 1); + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_RET(ret); + } +# endif #endif #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON) } else if (ERTS_IS_ATOM_STR("ultrasparc_set_pcr", sel)) { @@ -2459,6 +2497,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #elif defined(VALGRIND) ERTS_DECL_AM(valgrind); BIF_RET(AM_valgrind); +#elif defined(ADDRESS_SANITIZER) + ERTS_DECL_AM(asan); + BIF_RET(AM_asan); #elif defined(GPROF) ERTS_DECL_AM(gprof); BIF_RET(AM_gprof); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 568534cab2..0428b4e348 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1502,7 +1502,7 @@ re_inspect_2(BIF_ALIST_2) tp = tuple_val(BIF_ARG_1); if (tp[1] != am_re_pattern || is_not_small(tp[2]) || is_not_small(tp[3]) || is_not_small(tp[4]) || - is_not_binary(tp[5])) { + is_not_binary(tp[5]) || binary_size(tp[5]) < 4) { goto error; } if (BIF_ARG_2 != am_namelist) { diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7708e0755c..36cad53ce4 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1841,10 +1841,13 @@ new_seq_trace_token(Process* p, int ensure_new_heap) make_small(p->seq_trace_lastcnt)); } else if (ensure_new_heap) { + Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; + Uint mature_size = p->high_water - mature; Eterm* tpl = tuple_val(SEQ_TRACE_TOKEN(p)); ASSERT(arityval(tpl[0]) == 5); if (ErtsInArea(tpl, OLD_HEAP(p), - (OLD_HEND(p) - OLD_HEAP(p))*sizeof(Eterm))) { + (OLD_HEND(p) - OLD_HEAP(p))*sizeof(Eterm)) || + ErtsInArea(tpl, mature, mature_size*sizeof(Eterm))) { hp = HAlloc(p, 6); sys_memcpy(hp, tpl, 6*sizeof(Eterm)); SEQ_TRACE_TOKEN(p) = make_tuple(hp); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index f3e3890e94..20344bbdcd 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -369,7 +369,7 @@ erts_free_aligned_binary_bytes(byte* buf) ** These extra bytes where earlier (< R13B04) added by an alignment-bug ** in this code. Do we dare remove this in some major release (R14?) maybe? */ -#if defined(DEBUG) || defined(VALGRIND) +#if defined(DEBUG) || defined(VALGRIND) || defined(ADDRESS_SANITIZER) # define CHICKEN_PAD 0 #else # define CHICKEN_PAD (sizeof(void*) - 1) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5d68bd1ae0..3b9ee7b4fb 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1634,6 +1634,11 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } +static ERTS_INLINE erts_aint32_t +unset_aux_work_flags_mb(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_band_mb(&ssi->aux_work, ~flgs); +} static ERTS_INLINE void haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) @@ -1729,9 +1734,7 @@ handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, in if (!waiting && awdp->delayed_wakeup.next > awdp->esdp->reductions) return aux_work; - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); - - ERTS_THR_MEMORY_BARRIER; + unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); max_jix = awdp->delayed_wakeup.jix; awdp->delayed_wakeup.jix = -1; @@ -1855,7 +1858,7 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, { ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q; - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); while (1) { erts_misc_aux_work_t *mawp = erts_thr_q_dequeue(q); if (!mawp) @@ -1957,7 +1960,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); - unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); + unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); if (erts_check_async_ready(awdp->async_ready.queue)) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { @@ -2013,8 +2016,8 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); - unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); + unset_aux_work_flags_mb(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC); res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work); @@ -2062,7 +2065,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); - unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_DD); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, @@ -2158,7 +2161,7 @@ handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); - unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); erts_handle_canceled_timers((void *) awdp->esdp, &need_thr_progress, &wakeup, @@ -2328,7 +2331,7 @@ handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int w awdp->debug.wait_completed.callback = NULL; awdp->debug.wait_completed.arg = NULL; - unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); + unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; } @@ -2464,7 +2467,7 @@ int erts_halt_code; static ERTS_INLINE erts_aint32_t handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); + unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING); if (erts_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { @@ -2559,7 +2562,7 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) static ERTS_INLINE erts_aint32_t handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); + unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); erts_mseg_cache_check(); return aux_work & ~ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; } @@ -2570,7 +2573,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); + unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); setup_aux_work_timer(awdp->esdp); return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } @@ -11936,6 +11939,28 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->fp_exception = 0; #endif + if (parent && IS_TRACED(parent)) { + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); + } + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { + /* Overrides TRACE_CHILDREN */ + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + } + if (so->flags & SPO_LINK && ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); + erts_tracer_replace(&p->common, ERTS_TRACER(parent)); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + } + } + } + /* seq_trace is handled before regular tracing as the latter may touch the * trace token. */ if (!have_seqtrace(token)) { @@ -12036,39 +12061,17 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } } - if (parent && IS_TRACED(parent)) { - if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { - ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); - erts_tracer_replace(&p->common, ERTS_TRACER(parent)); - } - if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { - /* Overrides TRACE_CHILDREN */ - ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); - erts_tracer_replace(&p->common, ERTS_TRACER(parent)); - ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); - ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); - } - if (so->flags & SPO_LINK && ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { - ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); - erts_tracer_replace(&p->common, ERTS_TRACER(parent)); - if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ - ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - } - } - if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - /* The locks may already be released if seq_trace is enabled as - * well. */ - if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) - == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { - locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - } - trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); - if (so->flags & SPO_LINK) - trace_proc(parent, locks, parent, am_link, p->common.id); + if (parent && IS_TRACED_FL(parent, F_TRACE_PROCS)) { + /* The locks may already be released if seq_trace is enabled as well. */ + if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) + == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); } + trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); + if (so->flags & SPO_LINK) + trace_proc(parent, locks, parent, am_link, p->common.id); } if (IS_TRACED_FL(p, F_TRACE_PROCS)) { diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index 9766e76a83..d24bb727ce 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -32,6 +32,7 @@ # include "config.h" #endif +#if !defined(VALGRIND) && !defined(ADDRESS_SANITIZER) #include "erl_process.h" #include "erl_thr_progress.h" @@ -347,3 +348,4 @@ erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, return res; } +#endif /* !defined(VALGRIND) && !defined(ADDRESS_SANITIZER) */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index d7fa2f2696..a3f3124fee 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1229,14 +1229,9 @@ erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp, int task_flags, ErtsProc2PortSigCallback callback) { - Port *prt = erts_port_lookup_raw(port_num); - - if (!prt) - return -1; - sigdp->caller = ERTS_INVALID_PID; - return erts_port_task_schedule(prt->common.id, + return erts_port_task_schedule(port_num, NULL, ERTS_PORT_TASK_PROC_SIG, sigdp, diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 00708f1478..5f396b209e 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -82,6 +82,32 @@ /* All platforms fail on malloc errors. */ #define FATAL_MALLOC +/* The linux kernel sctp include files have an alignment bug + that causes warnings of this type to appear: + + drivers/common/inet_drv.c:3196:47: error: taking address of packed member of 'struct sctp_paddr_change' may result in an unaligned pointer value [-Werror=address-of-packed-member] + 3196 | i = load_inet_get_address(spec, i, desc, &sptr->spc_aaddr); + + So we need to suppress those, without disabling all warning + diagnostics of that type. + + See https://lore.kernel.org/patchwork/patch/1108122/ for the + patch that fixes this bug. In a few years we should be able to + remove this suppression. */ +#ifdef HAVE_GCC_DIAG_IGNORE_WADDRESS_OF_PACKED_MEMBER +#define PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER() \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Waddress-of-packed-member\"") \ + do { } while(0) +#define POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER() \ + _Pragma("GCC diagnostic pop") \ + do { } while(0) +#else +#define PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER() \ + do { } while(0) +#define POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER() \ + do { } while(0) +#endif #include "erl_driver.h" @@ -601,15 +627,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #include "packet_parser.h" -#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \ - (((unsigned char*) (s))[1] << 8) | \ - (((unsigned char*) (s))[2])) - -#define get_little_int32(s) ((((unsigned char*) (s))[3] << 24) | \ - (((unsigned char*) (s))[2] << 16) | \ - (((unsigned char*) (s))[1] << 8) | \ - (((unsigned char*) (s))[0])) - #if defined(HAVE_SYS_UN_H) || defined(SO_BINDTODEVICE) /* strnlen doesn't exist everywhere */ @@ -3193,7 +3210,9 @@ static int sctp_parse_async_event ASSERT(sptr->spc_length <= sz); /* No buffer overrun */ i = LOAD_ATOM (spec, i, am_sctp_paddr_change); + PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = load_inet_get_address(spec, i, desc, &sptr->spc_aaddr); + POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); switch (sptr->spc_state) { @@ -8150,7 +8169,9 @@ static int load_paddrinfo (ErlDrvTermData * spec, int i, { i = LOAD_ATOM (spec, i, am_sctp_paddrinfo); i = LOAD_ASSOC_ID (spec, i, pai->spinfo_assoc_id); + PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = load_inet_get_address(spec, i, desc, &pai->spinfo_address); + POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); switch(pai->spinfo_state) { case SCTP_ACTIVE: @@ -8670,7 +8691,9 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, ASSERT(0); } i = LOAD_ASSOC_ID (spec, i, sp.sspp_assoc_id); + PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = load_inet_get_address(spec, i, desc, &sp.sspp_addr); + POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = LOAD_TUPLE (spec, i, 3); i = LOAD_TUPLE (spec, i, 2); break; @@ -8730,7 +8753,9 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, i = LOAD_ATOM (spec, i, am_sctp_peer_addr_params); i = LOAD_ATOM (spec, i, am_sctp_paddrparams); i = LOAD_ASSOC_ID (spec, i, ap.spp_assoc_id); + PUSH_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = load_inet_get_address(spec, i, desc, &ap.spp_address); + POP_SUPPRESS_ADDRESS_OF_PACKED_MEMBER(); i = LOAD_INT (spec, i, ap.spp_hbinterval); i = LOAD_INT (spec, i, ap.spp_pathmaxrxt); @@ -10499,6 +10524,11 @@ static void tcp_inet_send_timeout(ErlDrvData e, ErlDrvTermData dummy) if (desc->send_timeout_close) { tcp_desc_close(desc); } + /* Q: Why not keep port busy as send queue may still be full (ERL-1390)? + * + * A: If kept busy, a following send call would hang without a timeout + * as it would get suspended in erlang:port_command waiting on busy port. + */ } /* diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index b0d9fc0776..78c20ea98d 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -2130,13 +2130,18 @@ void erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) { static int is_first_call = 1; - int virtual_map = 0; char *start = NULL, *end = NULL; UWord pagesize; + int virtual_map = 0; + + (void)virtual_map; + #if defined(__WIN32__) - SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - pagesize = (UWord) sysinfo.dwPageSize; + { + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + pagesize = (UWord) sysinfo.dwPageSize; + } #elif defined(_SC_PAGESIZE) pagesize = (UWord) sysconf(_SC_PAGESIZE); #elif defined(HAVE_GETPAGESIZE) diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 1a6de44dfd..7a3fdd0aa9 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -49,7 +49,8 @@ * See the following message on how MAP_NORESERVE was treated on FreeBSD: * <http://lists.llvm.org/pipermail/cfe-commits/Week-of-Mon-20150202/122958.html> */ -# if defined(MAP_FIXED) && (defined(MAP_NORESERVE) || defined(__FreeBSD__)) +# if (defined(MAP_FIXED) && (defined(MAP_NORESERVE) || defined(__FreeBSD__)) \ + && !defined(ADDRESS_SANITIZER)) # define ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION 1 # endif #endif @@ -207,20 +208,16 @@ ERTS_GLB_INLINE void erts_mem_discard(void *p, UWord size); #include <sys/mman.h> ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { + /* Note that we don't fall back to MADV_DONTNEED since it promises that + * the given region will be zeroed on access, which turned out to be + * too much of a performance hit. */ #ifdef MADV_FREE - /* This is preferred as it doesn't necessarily free the pages right - * away, which is a bit faster than MADV_DONTNEED. */ madvise(ptr, size, MADV_FREE); #else - madvise(ptr, size, MADV_DONTNEED); + (void)ptr; + (void)size; #endif } -#elif defined(HAVE_SYS_MMAN_H) && defined(HAVE_POSIX_MADVISE) && !(defined(__sun) || defined(__sun__)) - #include <sys/mman.h> - - ERTS_GLB_INLINE void erts_mem_discard(void *ptr, UWord size) { - posix_madvise(ptr, size, POSIX_MADV_DONTNEED); - } #elif defined(_WIN32) #include <winbase.h> diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index beaa466f81..46a035214b 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -49,6 +49,10 @@ #include <sys/ioctl.h> #endif +#ifdef ADDRESS_SANITIZER +# include <sanitizer/asan_interface.h> +#endif + #define ERTS_WANT_BREAK_HANDLING #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" @@ -386,6 +390,9 @@ void erts_sys_sigsegv_handler(int signo) { */ int erts_sys_is_area_readable(char *start, char *stop) { +#ifdef ADDRESS_SANITIZER + return __asan_region_is_poisoned(start, stop-start) == NULL; +#else int fds[2]; if (!pipe(fds)) { /* We let write try to figure out if the pointers are readable */ @@ -400,7 +407,7 @@ erts_sys_is_area_readable(char *start, char *stop) { return 1; } return 0; - +#endif } static ERTS_INLINE int diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 51406c6934..97cd90d72c 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -19,8 +19,8 @@ -module(alloc_SUITE). -author('rickard.green@uab.ericsson.se'). --export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). - +-export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2, + init_per_suite/1, end_per_suite/1]). -export([basic/1, coalesce/1, threads/1, @@ -47,6 +47,18 @@ all() -> bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration, cpool_opt]. +init_per_suite(Config) -> + case test_server:memory_checker() of + MC when MC =:= valgrind; MC =:= asan -> + %% No point testing own allocators under valgrind or asan. + {skip, "Memory checker " ++ atom_to_list(MC)}; + none -> + Config + end. + +end_per_suite(_Config) -> + ok. + init_per_testcase(Case, Config) when is_list(Config) -> [{testcase, Case},{debug,false}|Config]. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index a5f2e70d71..924c24b2fb 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -790,7 +790,12 @@ erlang_halt(Config) when is_list(Config) -> % This test triggers a segfault when dumping a crash dump % to make sure that we can handle it properly. + + %% Prevent address sanitizer from catching SEGV in slave node + AsanOpts = add_asan_opt("handle_segv=0"), {ok,N4} = slave:start(H, halt_node4), + reset_asan_opts(AsanOpts), + CrashDump = filename:join(proplists:get_value(priv_dir,Config), "segfault_erl_crash.dump"), true = rpc:call(N4, os, putenv, ["ERL_CRASH_DUMP",CrashDump]), @@ -808,6 +813,25 @@ erlang_halt(Config) when is_list(Config) -> ok end. +add_asan_opt(Opt) -> + case test_server:is_asan() of + true -> + case os:getenv("ASAN_OPTIONS") of + false -> + os:putenv("ASAN_OPTIONS", Opt), + undefined; + AO -> + os:putenv("ASAN_OPTIONS", AO ++ [$: | Opt]), + AO + end; + _ -> + false + end. + +reset_asan_opts(false) -> ok; +reset_asan_opts(undefined) -> os:unsetenv("ASAN_OPTIONS"); +reset_asan_opts(AO) -> os:putenv("ASAN_OPTIONS", AO). + wait_until_stable_size(_File,-10) -> {error,enoent}; wait_until_stable_size(File,PrevSz) -> diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index 32efd6cf84..01473874e8 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -229,7 +229,10 @@ alloc_blocks_size(Config) when is_list(Config) -> ok = rpc:call(Node, ?MODULE, do_alloc_blocks_size, []), true = test_server:stop_node(Node) end, - F("+Meamax"), + case test_server:is_asan() of + false -> F("+Meamax"); + true -> skip + end, F("+Meamin"), F(""), ok. diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl index c4a700d1a7..86b4460b38 100644 --- a/erts/emulator/test/hash_SUITE.erl +++ b/erts/emulator/test/hash_SUITE.erl @@ -640,13 +640,18 @@ test_phash2_plus_bin_helper2(Bin, TransformerFun, ExtraBytes, ExtraBits, Expecte end. run_when_enough_resources(Fun) -> - case {total_memory(), erlang:system_info(wordsize)} of - {Mem, 8} when is_integer(Mem) andalso Mem >= 31 -> + Bits = 8 * erlang:system_info({wordsize,external}), + Mem = total_memory(), + Build = erlang:system_info(build_type), + + if Bits =:= 64, is_integer(Mem), Mem >= 31, + Build =/= valgrind, Build =/= asan -> Fun(); - {Mem, WordSize} -> + + true -> {skipped, - io_lib:format("Not enough resources (System Memory >= ~p, Word Size = ~p)", - [Mem, WordSize])} + io_lib:format("Not enough resources (System Memory = ~p, Bits = ~p, Build = ~p)", + [Mem, Bits, Build])} end. %% Total memory in GB diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl index 6bafb0e18c..7bd8985dc7 100644 --- a/erts/emulator/test/os_signal_SUITE.erl +++ b/erts/emulator/test/os_signal_SUITE.erl @@ -275,6 +275,15 @@ t_sigalrm(_Config) -> ok. t_sigchld_fork(_Config) -> + case test_server:is_asan() of + true -> + %% Avoid false leak reports from forked process + {skip, "Address sanitizer"}; + false -> + sigchld_fork() + end. + +sigchld_fork() -> Pid1 = setup_service(), ok = os:set_signal(sigchld, handle), {ok,OsPid} = os_signal_SUITE:fork(), diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 539425b954..3889308a67 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -43,6 +43,7 @@ # -purecov Run emulator compiled for purecov # -gcov Run emulator compiled for gcov # -valgrind Run emulator compiled for valgrind +# -asan Run emulator compiled for address-sanitizer # -lcnt Run emulator compiled for lock counting # -icount Run emulator compiled for instruction counting # -rr Run emulator under "rr record" @@ -75,8 +76,8 @@ GDB= GDBBP= GDBARGS= TYPE= -debug= run_valgrind=no +run_asan=no run_rr=no skip_erlexec=no @@ -220,6 +221,12 @@ while [ $# -gt 0 ]; do run_valgrind=yes skip_erlexec=yes ;; + "-asan") + shift + cargs="$cargs -asan" + run_asan=yes + TYPE=.asan + ;; "-emu_type") shift cargs="$cargs -$1" @@ -271,6 +278,28 @@ if [ $skip_erlexec = yes ]; then set -- $beam_args IFS="$SAVE_IFS" fi +if [ $run_asan = yes ]; then + # Leak sanitizer options + if [ "x${LSAN_OPTIONS#*suppressions=}" = "x$LSAN_OPTIONS" ]; then + export LSAN_OPTIONS + if [ "x$ERL_TOP" != "x" ]; then + LSAN_OPTIONS="$LSAN_OPTIONS:suppressions=$ERL_TOP/erts/emulator/asan/suppress" + else + echo "No leak-sanitizer suppression file found in \$LSAN_OPTIONS" + echo "and \$ERL_TOP not set." + fi + fi + # Address sanitizer options + export ASAN_OPTIONS + if [ "x$ASAN_LOG_DIR" != "x" ]; then + if [ "x${ASAN_OPTIONS#*log_path=}" = "x$ASAN_OPTIONS" ]; then + ASAN_OPTIONS="$ASAN_OPTIONS:log_path=$ASAN_LOG_DIR/$EMU_NAME-$ASAN_LOGFILE_PREFIX-0" + fi + fi + if [ "x${ASAN_OPTIONS#*halt_on_error=}" = "x$ASAN_OPTIONS" ]; then + ASAN_OPTIONS="$ASAN_OPTIONS:halt_on_error=false" + fi +fi if [ "x$GDB" = "x" ]; then if [ $run_valgrind = yes ]; then valversion=`valgrind --version` diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi index 5a79101b5d..907a64b89c 100644 --- a/erts/etc/win32/nsis/erlang20.nsi +++ b/erts/etc/win32/nsis/erlang20.nsi @@ -144,7 +144,21 @@ SubSection /e "Erlang" SecErlang Section "Development" SecErlangDev
SectionIn 1 RO
+ SetOutPath "$INSTDIR"
+ +; Don't let Users nor Autenticated Users group create new files +; Avoid dll injection when installing to non /Program Files/ dirs + + StrCmp $INSTDIR $InstallDir cp_files
+ ; Remove ANY inherited access control + ExecShellWait "open" "$SYSDIR\icacls.exe" '"$INSTDIR" /inheritance:r' SW_HIDE
+ ; Grant Admin full control + ExecShellWait "open" "$SYSDIR\icacls.exe" '"$INSTDIR" /grant:r *S-1-5-32-544:(OI)(CI)F' SW_HIDE
+ ; Grant Normal Users read+execute control + ExecShellWait "open" "$SYSDIR\icacls.exe" '"$INSTDIR" /grant:r *S-1-1-0:(OI)(CI)RX' SW_HIDE
+
+cp_files:
File "${TESTROOT}\Install.ini"
File "${TESTROOT}\Install.exe"
SetOutPath "$INSTDIR\releases"
diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 6309f10439..300068b952 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -76,6 +76,12 @@ /* Define if x86/x86_64 out of order instructions should be synchronized */ #undef ETHR_X86_OUT_OF_ORDER +/* Define if you have the powerpc lwsync instruction */ +#undef ETHR_PPC_HAVE_LWSYNC + +/* Define if you do not have the powerpc lwsync instruction */ +#undef ETHR_PPC_HAVE_NO_LWSYNC + /* Define if only run in Sparc TSO mode */ #undef ETHR_SPARC_TSO @@ -86,10 +92,20 @@ #undef ETHR_SPARC_RMO /* Define as a boolean indicating whether you have a gcc compatible compiler - capable of generating the ARM DMB instruction, and are compiling for an ARM - processor with ARM DMB instruction support, or not */ + capable of generating the ARM 'dmb sy' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ #undef ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION +/* Define as a boolean indicating whether you have a gcc compatible compiler + capable of generating the ARM 'dmb ld' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ +#undef ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION + +/* Define as a boolean indicating whether you have a gcc compatible compiler + capable of generating the ARM 'dmb st' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ +#undef ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION + /* Define as a bitmask corresponding to the word sizes that __sync_synchronize() can handle on your system */ #undef ETHR_HAVE___sync_synchronize diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h index 643b243683..4e1eb1117e 100644 --- a/erts/include/internal/gcc/ethr_membar.h +++ b/erts/include/internal/gcc/ethr_membar.h @@ -149,14 +149,51 @@ ethr_full_fence__(void) __asm__ __volatile__("dmb sy" : : : "memory"); } +#if ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION static __inline__ __attribute__((__always_inline__)) void ethr_store_fence__(void) { + /* StoreStore */ __asm__ __volatile__("dmb st" : : : "memory"); } +#endif + +#if ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +static __inline__ __attribute__((__always_inline__)) void +ethr_load_fence__(void) +{ + /* LoadLoad and LoadStore */ + __asm__ __volatile__("dmb ld" : : : "memory"); +} +#endif -#define ETHR_MEMBAR(B) \ - ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, ethr_store_fence__(), ethr_full_fence__()) +#if ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION && ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +/* sy, st & ld */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_store_fence__(), \ + ETHR_CHOOSE_EXPR((B) & (ETHR_StoreStore \ + | ETHR_StoreLoad), \ + ethr_full_fence__(), \ + ethr_load_fence__())) +#elif ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION +/* sy & st */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_store_fence__(), \ + ethr_full_fence__()) +#elif ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +/* sy & ld */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & (ETHR_StoreStore \ + | ETHR_StoreLoad), \ + ethr_full_fence__(), \ + ethr_load_fence__()) +#else +/* sy */ +#define ETHR_MEMBAR(B) \ + ethr_full_fence__() +#endif #elif ETHR_HAVE___sync_synchronize @@ -205,9 +242,12 @@ ethr_full_fence__(void) /* * Define ETHR_READ_DEPEND_MEMORY_BARRIER for all architechtures * not known to order data dependent loads + * + * This is a bit too conservative, but better safe than sorry... + * Add more archs as needed... */ -#if !defined(__ia64__) && !defined(__arm__) +#if !defined(__ia64__) && !defined(__arm__) && !defined(__arm64__) # define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) #endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index 12b41f8704..300a8c6922 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -44,6 +44,10 @@ #undef ETHR_GCC_RELB_VERSIONS__ #undef ETHR_GCC_RELB_MOD_VERSIONS__ #undef ETHR_GCC_MB_MOD_VERSIONS__ +#undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ + +#define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ \ + ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS /* * True GNU GCCs before version 4.8 do not emit a memory barrier @@ -52,18 +56,38 @@ */ #undef ETHR___atomic_load_ACQUIRE_barrier_bug #if ETHR_GCC_COMPILER != ETHR_GCC_COMPILER_TRUE + +#if ETHR_GCC_COMPILER == ETHR_GCC_COMPILER_CLANG \ + && defined(__apple_build_version__) \ + && __clang_major__ >= 12 +/* Apples clang verified not to have this bug */ +# define ETHR___atomic_load_ACQUIRE_barrier_bug 0 +/* Also trust builtin barriers */ +# undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ +# define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 +# else /* - * A gcc compatible compiler. We have no information + * Another gcc compatible compiler. We have no information * about the existence of this bug, but we assume * that it is not impossible that it could have * been "inherited". Therefore, until we are certain * that the bug does not exist, we assume that it * does. */ -# define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +# define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +# endif + #elif !ETHR_AT_LEAST_GCC_VSN__(4, 8, 0) /* True gcc of version < 4.8, i.e., bug exist... */ # define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +#elif ETHR_AT_LEAST_GCC_VSN__(9, 3, 0) \ + && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)) \ + && ETHR_SIZEOF_PTR == 8 +/* Verified not to have this bug */ +# define ETHR___atomic_load_ACQUIRE_barrier_bug 0 +/* Also trust builtin barriers */ +# undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ +# define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 #else /* True gcc of version >= 4.8 */ /* * Sizes less than or equal to word size have been fixed, @@ -87,7 +111,7 @@ #define ETHR_GCC_RELAXED_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ #define ETHR_GCC_RELAXED_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ -#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS +#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ # define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ # define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ # define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index bb43d51d97..8fddd3479e 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -81,6 +81,11 @@ CFLAGS=@DEBUG_CFLAGS@ -DVALGRIND TYPE_SUFFIX=.valgrind PRE_LD= else +ifeq ($(TYPE),asan) +CFLAGS=@DEBUG_CFLAGS@ +TYPE_SUFFIX=.asan +PRE_LD= +else ifeq ($(TYPE),gprof) CFLAGS += -DGPROF -pg TYPE_SUFFIX=.gprof @@ -117,6 +122,7 @@ endif endif endif endif +endif OPSYS=@OPSYS@ sol2CFLAGS= diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index 17506b87ef..b35d53be7d 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -30,6 +30,7 @@ #include "erl_misc_utils.h" #if !defined(__WIN32__) /* UNIX */ +# include <stdarg.h> # include <stdio.h> # include <sys/types.h> # include <sys/param.h> @@ -1095,15 +1096,21 @@ get_cgroup_v1_base_dir(const char *controller) { return NULL; } -static const char* -get_cgroup_path(const char *controller) { +enum cgroup_version_t { + ERTS_CGROUP_NONE, + ERTS_CGROUP_V1, + ERTS_CGROUP_V2 +}; + +static enum cgroup_version_t +get_cgroup_path(const char *controller, const char **path) { char line_buf[10 << 10]; FILE *var_file; var_file = fopen("/proc/self/mountinfo", "r"); if (var_file == NULL) { - return NULL; + return ERTS_CGROUP_NONE; } while (fgets(line_buf, sizeof(line_buf), var_file)) { @@ -1138,7 +1145,9 @@ get_cgroup_path(const char *controller) { if (csv_contains(controllers, controller, ' ')) { free((void*)cgc_path); fclose(var_file); - return strdup(mount_path); + + *path = strdup(mount_path); + return ERTS_CGROUP_V2; } } free((void*)cgc_path); @@ -1147,42 +1156,51 @@ get_cgroup_path(const char *controller) { const char *base_dir = get_cgroup_v1_base_dir(controller); if (base_dir) { - const char *result; - if (strcmp(root_path, base_dir)) { - result = str_combine(mount_path, base_dir); + *path = str_combine(mount_path, base_dir); } else { - result = strdup(mount_path); + *path = strdup(mount_path); } free((void*)base_dir); fclose(var_file); - return result; + + return ERTS_CGROUP_V1; } } } } fclose(var_file); - return NULL; + + return ERTS_CGROUP_NONE; } -static int read_cgroup_var(const char *group_path, const char *var_name, - ssize_t *out) { +static int read_cgroup_interface(const char *group_path, const char *if_name, + int arg_count, const char *format, ...) { const char *var_path; int res; - var_path = str_combine(group_path, var_name); + var_path = str_combine(group_path, if_name); res = 0; if (var_path) { - FILE *var_file = fopen(var_path, "r"); + FILE *var_file; + + var_file = fopen(var_path, "r"); free((void*)var_path); if (var_file) { - if (fscanf(var_file, "%zi", out) == 1) { + va_list va_args; + + va_start(va_args, format); + + if (vfscanf(var_file, format, va_args) == arg_count) { res = 1; } + + va_end(va_args); + fclose(var_file); } } @@ -1197,35 +1215,44 @@ static int read_cgroup_var(const char *group_path, const char *var_name, static int read_cpu_quota(int limit) { - const char *cgroup_path = get_cgroup_path("cpu"); + ssize_t cfs_period_us, cfs_quota_us; + const char *cgroup_path; + int succeeded; - if (cgroup_path) { - ssize_t cfs_period_us, cfs_quota_us; - int succeeded; + switch (get_cgroup_path("cpu", &cgroup_path)) { + case ERTS_CGROUP_V1: + succeeded = read_cgroup_interface(cgroup_path, "/cpu.cfs_quota_us", + 1, "%zi", &cfs_quota_us) && + read_cgroup_interface(cgroup_path, "/cpu.cfs_period_us", + 1, "%zi", &cfs_period_us); - cfs_period_us = -1; - cfs_quota_us = -1; - - succeeded = - read_cgroup_var(cgroup_path, "/cpu.cfs_quota_us", &cfs_quota_us) && - read_cgroup_var(cgroup_path, "/cpu.cfs_period_us", &cfs_period_us); + free((void*)cgroup_path); + break; + case ERTS_CGROUP_V2: + succeeded = read_cgroup_interface(cgroup_path, "/cpu.max", + 2, "%zi %zi", &cfs_quota_us, &cfs_period_us); free((void*)cgroup_path); + break; + default: + succeeded = 0; + break; + } - if (succeeded) { - if (cfs_period_us > 0 && cfs_quota_us > 0) { - size_t quota = cfs_quota_us / cfs_period_us; + if (succeeded) { + if (cfs_period_us > 0 && cfs_quota_us > 0) { + size_t quota = cfs_quota_us / cfs_period_us; - if (quota == 0) { - quota = 1; - } - if (quota > 0 && quota <= (size_t)limit) { - return quota; - } + if (quota == 0) { + quota = 1; } - return limit; + if (quota > 0 && quota <= (size_t)limit) { + return quota; + } } + + return limit; } return 0; diff --git a/erts/vsn.mk b/erts/vsn.mk index 5cf93de819..cc7958578f 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 11.1.5 +VSN = 11.1.8 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/common_test/doc/src/ct.xml b/lib/common_test/doc/src/ct.xml index c8b780db66..366d856e08 100644 --- a/lib/common_test/doc/src/ct.xml +++ b/lib/common_test/doc/src/ct.xml @@ -166,7 +166,7 @@ <seemfa marker="#break/2"><c>ct:break/2</c></seemfa> is to be called instead.</p> <p>A cancelled timetrap is not automatically reactivated after the - break, but must be started exlicitly with + break, but must be started explicitly with <seemfa marker="#timetrap/1"><c>ct:timetrap/1</c></seemfa>.</p> <p>In order for the break/continue functionality to work, <c>Common Test</c> must release the shell process controlling <c>stdin</c>. @@ -1090,7 +1090,7 @@ <v>Reason = term()</v> </type> <desc><marker id="remove_config-2"/> - <p>Removes configuration variables (together wih their aliases) + <p>Removes configuration variables (together with their aliases) that were loaded with specified callback module and configuration string.</p> </desc> @@ -1435,7 +1435,7 @@ </type> <desc><marker id="step-4"/> <p>Steps through a test case with the debugger. If option - <c>config</c> has been specifed, breakpoints are also set on + <c>config</c> has been specified, breakpoints are also set on the configuration functions in <c>Suite</c>.</p> <p>See also <seemfa marker="#run/3"><c>ct:run/3</c></seemfa>.</p> diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 20c229dde4..b3cc2af145 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -21,7 +21,7 @@ -define(DEFAULT_TIMETRAP_SECS, 60). %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([run_test_case_apply/1,init_target_info/0,init_valgrind/0]). +-export([run_test_case_apply/1,init_target_info/0,init_memory_checker/0]). -export([cover_compile/1,cover_analyse/2]). %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -48,10 +48,8 @@ -export([is_cover/0,is_debug/0,is_commercial/0]). -export([break/1,break/2,break/3,continue/0,continue/1]). +-export([memory_checker/0, is_valgrind/0, is_asan/0]). -%%% DEBUGGER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([valgrind_new_leaks/0, valgrind_format/2, - is_valgrind/0]). %%% PRIVATE EXPORTED %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([]). @@ -60,6 +58,7 @@ -include("test_server_internal.hrl"). -include_lib("kernel/include/file.hrl"). + init_target_info() -> [$.|Emu] = code:objfile_extension(), {_, OTPRel} = init:script_id(), @@ -73,8 +72,8 @@ init_target_info() -> username=test_server_sup:get_username(), cookie=atom_to_list(erlang:get_cookie())}. -init_valgrind() -> - valgrind_new_leaks(). +init_memory_checker() -> + check_memory_leaks(). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -367,19 +366,50 @@ stick_all_sticky(Node,Sticky) -> %% cover. run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit,TimetrapData}) -> - case is_valgrind() of - false -> - ok; - true -> - valgrind_format("Test case #~w ~w:~w/1", [CaseNum, Mod, Func]), - os:putenv("VALGRIND_LOGFILE_INFIX",atom_to_list(Mod)++"."++ - atom_to_list(Func)++"-") - end, + MC = case {Func, memory_checker()} of + {init_per_suite, _} -> none; % skip init/end_per_suite/group + {init_per_group, _} -> none; % as CaseNum is always 0 + {end_per_group, _} -> none; + {end_per_suite, _} -> none; + {_, valgrind} -> + valgrind_format("Test case #~w ~w:~w/1", [CaseNum, Mod, Func]), + os:putenv("VALGRIND_LOGFILE_INFIX",atom_to_list(Mod)++"."++ + atom_to_list(Func)++"-"), + valgrind; + {_, asan} -> + %% Address sanitizer does not support printf in log file + %% but it lets us change the log file on the fly. So we use + %% that to give each test case its own log file. + case asan_take_logpath() of + false -> false; + {LogPath, OtherOpts} -> + LogDir = filename:dirname(LogPath), + LogFile = filename:basename(LogPath), + [Exe, App | _ ] = string:lexemes(LogFile, "-"), + NewLogFile = io_lib:format("~s-~s-tc-~4..0w-~w-~w", + [Exe,App,CaseNum, Mod, Func]), + NewLogPath = filename:join(LogDir, NewLogFile), + + %% Do leak check and then change asan log file + %% for this running beam executable. + erlang:system_info({memory_checker, check_leaks}), + _PrevLog = erlang:system_info({memory_checker, log, NewLogPath}), + + %% Set log file name for subnodes + %% that may be created by this test case + NewOpts = asan_make_opts(["log_path="++NewLogPath++".subnode" + | OtherOpts]), + os:putenv("ASAN_OPTIONS", NewOpts) + end, + asan; + {_, none} -> + node + end, ProcBef = erlang:system_info(process_count), Result = run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData), ProcAft = erlang:system_info(process_count), - valgrind_new_leaks(), + check_memory_leaks(MC), DetFail = get(test_server_detected_fail), {Result,DetFail,ProcBef,ProcAft}. @@ -2053,7 +2083,8 @@ timetrap_scale_factor() -> { 3, fun() -> has_superfluous_schedulers() end}, { 6, fun() -> is_debug() end}, {10, fun() -> is_cover() end}, - {10, fun() -> is_valgrind() end} + {10, fun() -> is_valgrind() end}, + {2, fun() -> is_asan() end} ]). timetrap_scale_factor(Scales) -> @@ -2962,22 +2993,36 @@ is_commercial() -> %% %% Returns true if valgrind is running, else false is_valgrind() -> - case catch erlang:system_info({valgrind, running}) of - {'EXIT', _} -> false; - Res -> Res + memory_checker() =:= valgrind. + +%% Returns true if address-sanitizer is running, else false +is_asan() -> + memory_checker() =:= asan. + +%% Returns the error checker running (valgrind | asan | none). +memory_checker() -> + case catch erlang:system_info({memory_checker, running}) of + {'EXIT', _} -> none; + EC -> EC end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% DEBUGGER INTERFACE %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% valgrind_new_leaks() -> ok +%% check_memory_leaks() -> ok %% -%% Checks for new memory leaks if Valgrind is active. -valgrind_new_leaks() -> - catch erlang:system_info({valgrind, memory}), +%% Checks for memory leaks if Valgrind or Address-sanitizer is active. +check_memory_leaks() -> + check_memory_leaks(memory_checker()). + +check_memory_leaks(valgrind) -> + catch erlang:system_info({memory_checker, check_leaks}), + ok; +check_memory_leaks(_) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2987,9 +3032,31 @@ valgrind_new_leaks() -> %% %% Outputs the formatted string to Valgrind's logfile,if Valgrind is active. valgrind_format(Format, Args) -> - (catch erlang:system_info({valgrind, io_lib:format(Format, Args)})), + (catch erlang:system_info({memory_checker, print, io_lib:format(Format, Args)})), ok. +asan_take_logpath() -> + case os:getenv("ASAN_OPTIONS") of + false -> false; + S -> + Opts = string:lexemes(S, ":"), + asan_take_logpath_loop(Opts, []) + end. + +asan_take_logpath_loop(["log_path="++LogPath | T], Acc) -> + {LogPath, T ++ Acc}; +asan_take_logpath_loop([Opt | T], Acc) -> + asan_take_logpath_loop(T, [Opt | Acc]); +asan_take_logpath_loop([], _) -> + false. + +asan_make_opts([A|T]) -> + asan_make_opts(T, A). + +asan_make_opts([], Acc) -> + Acc; +asan_make_opts([A|T], Acc) -> + asan_make_opts(T, A ++ [$: | Acc]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 995594dd59..dbd5537206 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2195,7 +2195,7 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> %% Runs the specified tests, then displays/logs the summary. run_test_cases(TestSpec, Config, TimetrapData) -> - test_server:init_valgrind(), + test_server:init_memory_checker(), case lists:member(no_src, get(test_server_logopts)) of true -> ok; diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index 8deea353d7..edfb1fbd92 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -645,7 +645,38 @@ find_release(latest) -> find_release(previous) -> "kaka"; find_release(Rel) -> - find_release(os:type(), Rel). + case find_release(os:type(), Rel) of + none -> + find_release_path(Rel); + Else -> + Else + end. + +find_release_path(Rel) -> + Paths = string:lexemes(os:getenv("PATH"), ":"), + find_release_path(Paths, Rel). +find_release_path([Path|T], Rel) -> + case os:find_executable("erl", Path) of + false -> + find_release_path(T, Rel); + ErlExec -> + Pattern = filename:join([Path,"..","releases","*","OTP_VERSION"]), + case filelib:wildcard(Pattern) of + [VersionFile] -> + {ok, VsnBin} = file:read_file(VersionFile), + [MajorVsn|_] = string:lexemes(VsnBin, "."), + case unicode:characters_to_list(MajorVsn) of + Rel -> + ErlExec; + _Else -> + find_release_path(T, Rel) + end; + _Else -> + find_release_path(T, Rel) + end + end; +find_release_path([], _) -> + none. find_release({unix,sunos}, Rel) -> case os:cmd("uname -p") of diff --git a/lib/common_test/test_server/ts_run.erl b/lib/common_test/test_server/ts_run.erl index 7e12b9652c..ce454dce9c 100644 --- a/lib/common_test/test_server/ts_run.erl +++ b/lib/common_test/test_server/ts_run.erl @@ -197,17 +197,25 @@ make_command(Vars, Spec, State) -> {ok,Cwd} = file:get_cwd(), TestDir = State#state.test_dir, TestPath = filename:nativename(TestDir), - Erl = case os:getenv("TS_RUN_VALGRIND") of + Erl = case os:getenv("TS_RUN_EMU") of false -> ct:get_progname(); - _ -> + "valgrind" -> case State#state.file of Dir when is_list(Dir) -> os:putenv("VALGRIND_LOGFILE_PREFIX", Dir++"-"); _ -> ok end, - "cerl -valgrind" + "cerl -valgrind"; + "asan" -> + case State#state.file of + App when is_list(App) -> + os:putenv("ASAN_LOGFILE_PREFIX", App); + _ -> + ok + end, + "cerl -asan" end, Naming = case ts_lib:var(longnames, Vars) of @@ -261,9 +269,10 @@ run_batch(Vars, _Spec, State) -> ts_lib:progress(Vars, 1, "Command: ~ts~n", [Command]), io:format(user, "Command: ~ts~n",[Command]), Port = open_port({spawn, Command}, [stream, in, eof, exit_status]), - Timeout = 30000 * case os:getenv("TS_RUN_VALGRIND") of + Timeout = 30000 * case os:getenv("TS_RUN_EMU") of false -> 1; - _ -> 100 + "valgrind" -> 100; + "asan" -> 2 end, tricky_print_data(Port, Timeout). diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index b9978401a5..4938c6920a 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -287,6 +287,22 @@ </section> +<section><title>Compiler 7.5.4.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>Fixed a bug in the type optimization pass that could + yield incorrect values or cause the wrong clauses to be + executed.</p> + <p> + Own Id: OTP-17073</p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.5.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index b5295147e5..055debe620 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -191,8 +191,6 @@ validate_0([{function, Name, Arity, Entry, Code} | Fs], Module, Level, Ft) -> h=0, %%Available heap size for floats. hf=0, - %% Floating point state. - fls=undefined, %% List of hot catch/try tags ct=[], %% Previous instruction was setelement/3. @@ -303,8 +301,9 @@ init_function_args(-1, Vst) -> init_function_args(X, Vst) -> init_function_args(X - 1, create_term(any, argument, [], {x,X}, Vst)). -kill_heap_allocation(St) -> - St#st{h=0,hf=0}. +kill_heap_allocation(#vst{current=St0}=Vst) -> + St = St0#st{h=0,hf=0}, + Vst#vst{current=St}. validate_branches(MFA, Vst) -> #vst{ branched=Targets0, labels=Labels0 } = Vst, @@ -393,7 +392,6 @@ vi({fmove,Src,{fr,_}=Dst}, Vst) -> set_freg(Dst, Vst); vi({fmove,{fr,_}=Src,Dst}, Vst0) -> assert_freg_set(Src, Vst0), - assert_fls(checked, Vst0), Vst = eat_heap_float(Vst0), create_term(#t_float{}, fmove, [], Dst, Vst); vi({kill,Reg}, Vst) -> @@ -680,9 +678,8 @@ vi({gc_bif,Op,{f,Fail},Live,Ss,Dst}, Vst0) -> %% Heap allocations and X registers are killed regardless of whether we %% fail or not, as we may fail after GC. - #vst{current=St0} = Vst0, - St = kill_heap_allocation(St0), - Vst = prune_x_regs(Live, Vst0#vst{current=St}), + Vst1 = kill_heap_allocation(Vst0), + Vst = prune_x_regs(Live, Vst1), validate_bif(gc_bif, Op, Fail, Ss, Dst, Vst0, Vst); @@ -728,13 +725,10 @@ vi({wait,{f,Lbl}}, Vst) -> vi({wait_timeout,{f,Lbl},Src}, Vst0) -> assert_no_exception(Lbl), - %% Note that the receive marker is not cleared since we may re-enter the - %% loop while waiting. If we time out we'll be transferred to a timeout - %% instruction that clears the marker. assert_term(Src, Vst0), verify_y_init(Vst0), - Vst = branch(Lbl, prune_x_regs(0, Vst0)), + Vst = branch(Lbl, schedule_out(0, Vst0)), branch(?EXCEPTION_LABEL, Vst); %% @@ -771,14 +765,15 @@ vi({try_end,Reg}, #vst{current=#st{ct=[Tag|_]}}=Vst) -> vi({try_case,Reg}, #vst{current=#st{ct=[Tag|_]}}=Vst0) -> case get_tag_type(Reg, Vst0) of {trytag,_Fail}=Tag -> - %% Kill the catch tag, all x registers, and the receive marker. + %% Kill the catch tag and all other state (as if we've been + %% scheduled out with no live registers). Only previously allocated + %% Y registers are alive at this point. Vst1 = kill_catch_tag(Reg, Vst0), - Vst2 = prune_x_regs(0, Vst1), - Vst3 = set_receive_marker(none, Vst2), + Vst2 = schedule_out(0, Vst1), %% Class:Error:Stacktrace - Vst4 = create_term(#t_atom{}, try_case, [], {x,0}, Vst3), - Vst = create_term(any, try_case, [], {x,1}, Vst4), + Vst3 = create_term(#t_atom{}, try_case, [], {x,0}, Vst2), + Vst = create_term(any, try_case, [], {x,1}, Vst3), create_term(any, try_case, [], {x,2}, Vst); Type -> error({wrong_tag_type,Type}) @@ -932,26 +927,13 @@ vi({fconv,Src,{fr,_}=Dst}, Vst) -> assert_term(Src, Vst), branch(?EXCEPTION_LABEL, Vst, - fun(FailVst) -> - %% This is a hack to supress assert_float_checked/1 in - %% fork_state/2, since this instruction is legal even when - %% the state is unchecked. - set_fls(checked, FailVst) - end, fun(SuccVst0) -> SuccVst = update_type(fun meet/2, number, Src, SuccVst0), set_freg(Dst, SuccVst) end); vi(fclearerror, Vst) -> - case get_fls(Vst) of - undefined -> ok; - checked -> ok; - Fls -> error({bad_floating_point_state,Fls}) - end, - set_fls(cleared, Vst); -vi({fcheckerror,_}, Vst0) -> - assert_fls(cleared, Vst0), - Vst = set_fls(checked, Vst0), + Vst; +vi({fcheckerror, _}, Vst) -> branch(?EXCEPTION_LABEL, Vst); %% @@ -1125,8 +1107,6 @@ validate_var_info([], _Reg, Vst) -> %% The stackframe must have a known size and be initialized. %% Does not return to the instruction following the call. validate_tail_call(Deallocate, Func, Live, #vst{current=#st{numy=NumY}}=Vst0) -> - assert_float_checked(Vst0), - verify_y_init(Vst0), verify_live(Live, Vst0), verify_call_args(Func, Live, Vst0), @@ -1153,18 +1133,15 @@ validate_tail_call(Deallocate, Func, Live, #vst{current=#st{numy=NumY}}=Vst0) -> %% The instruction will return to the instruction following the call. validate_body_call(Func, Live, #vst{current=#st{numy=NumY}}=Vst) when is_integer(NumY)-> - assert_float_checked(Vst), - verify_y_init(Vst), verify_live(Live, Vst), verify_call_args(Func, Live, Vst), - SuccFun = fun(#vst{current=St0}=SuccVst0) -> + SuccFun = fun(SuccVst0) -> {RetType, _, _} = call_types(Func, Live, SuccVst0), true = RetType =/= none, %Assertion. - St = St0#st{f=init_fregs()}, - SuccVst = prune_x_regs(0, SuccVst0#vst{current=St}), + SuccVst = schedule_out(0, SuccVst0), create_term(RetType, call, [], {x,0}, SuccVst) end, @@ -1180,13 +1157,6 @@ validate_body_call(Func, Live, validate_body_call(_, _, #vst{current=#st{numy=NumY}}) -> error({allocated, NumY}). -assert_float_checked(Vst) -> - case get_fls(Vst) of - undefined -> ok; - checked -> ok; - Fls -> error({unsafe_instruction,{float_error_state,Fls}}) - end. - init_try_catch_branch(Kind, Dst, Fail, Vst0) -> assert_no_exception(Fail), @@ -1336,7 +1306,6 @@ verify_return(#vst{current=#st{recv_marker=Mark}}) when Mark =/= none -> %% the message. error({return_with_receive_marker,Mark}); verify_return(Vst) -> - assert_float_checked(Vst), verify_no_ct(Vst), kill_state(Vst). @@ -1349,7 +1318,6 @@ verify_return(Vst) -> %% validate_bif(Kind, Op, Fail, Ss, Dst, OrigVst, Vst) -> - assert_float_checked(Vst), case {will_bif_succeed(Op, Ss, Vst), Fail} of {yes, _} -> %% This BIF cannot fail (neither throw nor branch), make sure it's @@ -1734,22 +1702,29 @@ test_heap(Heap, Live, Vst0) -> heap_alloc(Heap, Vst). heap_alloc(Heap, #vst{current=St0}=Vst) -> - St1 = kill_heap_allocation(St0), - St = heap_alloc_1(Heap, St1), + {HeapWords, Floats} = heap_alloc_1(Heap), + + St = St0#st{h=HeapWords,hf=Floats}, + Vst#vst{current=St}. -heap_alloc_1({alloc,Alloc}, St) -> - heap_alloc_2(Alloc, St); -heap_alloc_1(HeapWords, St) when is_integer(HeapWords) -> - St#st{h=HeapWords}. +heap_alloc_1({alloc, Alloc}) -> + heap_alloc_2(Alloc, 0, 0); +heap_alloc_1(HeapWords) when is_integer(HeapWords) -> + {HeapWords, 0}. -heap_alloc_2([{words,HeapWords}|T], St0) -> - St = St0#st{h=HeapWords}, - heap_alloc_2(T, St); -heap_alloc_2([{floats,Floats}|T], St0) -> - St = St0#st{hf=Floats}, - heap_alloc_2(T, St); -heap_alloc_2([], St) -> St. +heap_alloc_2([{words, HeapWords} | T], 0, Floats) -> + heap_alloc_2(T, HeapWords, Floats); +heap_alloc_2([{floats, Floats} | T], HeapWords, 0) -> + heap_alloc_2(T, HeapWords, Floats); +heap_alloc_2([], HeapWords, Floats) -> + {HeapWords, Floats}. + +schedule_out(Live, Vst0) when is_integer(Live) -> + Vst1 = prune_x_regs(Live, Vst0), + Vst2 = kill_heap_allocation(Vst1), + Vst = kill_fregs(Vst2), + set_receive_marker(none, Vst). prune_x_regs(Live, #vst{current=St0}=Vst) when is_integer(Live) -> #st{fragile=Fragile0,xs=Xs0} = St0, @@ -1788,22 +1763,10 @@ assert_arities(_) -> error(bad_tuple_arity_list). %%% -%%% Floating point checking. -%%% -%%% Possible values for the fls field (=floating point error state). -%%% -%%% undefined - Undefined (initial state). No float operations allowed. -%%% -%%% cleared - fclearerror/0 has been executed. Float operations -%%% are allowed (such as fadd). -%%% -%%% checked - fcheckerror/1 has been executed. It is allowed to -%%% move values out of floating point registers. -%%% -%%% The following instructions may be executed in any state: +%%% Floating point helpers. %%% -%%% fconv Src {fr,_} -%%% fmove Src {fr,_} %% Move INTO floating point register. +%%% fconv Src {fr,_} +%%% fmove Src {fr,_} %% Move known float INTO floating point register. %%% is_float_arith_bif(fadd, [_, _]) -> true; @@ -1813,25 +1776,16 @@ is_float_arith_bif(fnegate, [_]) -> true; is_float_arith_bif(fsub, [_, _]) -> true; is_float_arith_bif(_, _) -> false. -validate_float_arith_bif(Ss, Dst, Vst0) -> - _ = [assert_freg_set(S, Vst0) || S <- Ss], - assert_fls(cleared, Vst0), - Vst = set_fls(cleared, Vst0), +validate_float_arith_bif(Ss, Dst, Vst) -> + _ = [assert_freg_set(S, Vst) || S <- Ss], set_freg(Dst, Vst). -assert_fls(Fls, Vst) -> - case get_fls(Vst) of - Fls -> ok; - OtherFls -> error({bad_floating_point_state,OtherFls}) - end. - -set_fls(Fls, #vst{current=#st{}=St}=Vst) when is_atom(Fls) -> - Vst#vst{current=St#st{fls=Fls}}. - -get_fls(#vst{current=#st{fls=Fls}}) when is_atom(Fls) -> Fls. - init_fregs() -> 0. +kill_fregs(#vst{current=St0}=Vst) -> + St = St0#st{f=init_fregs()}, + Vst#vst{current=St}. + set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst) -> check_limit(Freg), Bit = 1 bsl Fr, @@ -2276,7 +2230,7 @@ new_value(Type, Op, Ss, #vst{current=#st{vs=Vs0}=St,ref_ctr=Counter}=Vst) -> {Ref, Vst#vst{current=St#st{vs=Vs},ref_ctr=Counter+1}}. kill_catch_tag(Reg, #vst{current=#st{ct=[Tag|Tags]}=St}=Vst0) -> - Vst = Vst0#vst{current=St#st{ct=Tags,fls=undefined}}, + Vst = Vst0#vst{current=St#st{ct=Tags}}, Tag = get_tag_type(Reg, Vst), %Assertion. kill_tag(Reg, Vst). @@ -2535,10 +2489,6 @@ branch(Fail, Vst) -> fork_state(?EXCEPTION_LABEL, Vst0) -> #vst{current=#st{ct=CatchTags,numy=NumY}} = Vst0, - %% Floating-point exceptions must be checked before any other kind of - %% exception can be raised. - assert_float_checked(Vst0), - %% The stack will be scanned looking for a catch tag, so all Y registers %% must be initialized. verify_y_init(Vst0), diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 6f6391d3ba..38be30b165 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -28,7 +28,7 @@ dead_code/1, overwrite_catchtag/1,overwrite_trytag/1,accessing_tags/1,bad_catch_try/1, cons_guard/1, - freg_range/1,freg_uninit/1,freg_state/1, + freg_range/1,freg_uninit/1, bad_bin_match/1,bad_dsetel/1, state_after_fault_in_catch/1,no_exception_in_catch/1, undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, @@ -63,7 +63,7 @@ groups() -> unsafe_catch,dead_code, overwrite_catchtag,overwrite_trytag,accessing_tags, bad_catch_try,cons_guard,freg_range,freg_uninit, - freg_state,bad_bin_match,bad_dsetel, + bad_bin_match,bad_dsetel, state_after_fault_in_catch,no_exception_in_catch, undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, @@ -290,28 +290,6 @@ freg_uninit(Config) when is_list(Config) -> {uninitialized_reg,{fr,0}}}}] = Errors, ok. -freg_state(Config) when is_list(Config) -> - Errors = do_val(freg_state, Config), - [{{t,sum_1,2}, - {{bif,fmul,{f,0},[{fr,0},{fr,1}],{fr,0}}, - 6, - {bad_floating_point_state,undefined}}}, - {{t,sum_2,2}, - {{fmove,{fr,0},{x,0}}, - 8, - {bad_floating_point_state,cleared}}}, - {{t,sum_3,2}, - {{bif,'-',{f,0},[{x,1},{x,0}],{x,1}}, - 8, - {unsafe_instruction,{float_error_state,cleared}}}}, - {{t,sum_4,2}, - {{fcheckerror,{f,0}}, - 4, - {bad_floating_point_state,undefined}}}, - {{t,sum_5,2}, - {fclearerror,5,{bad_floating_point_state,cleared}}}] = Errors, - ok. - bad_bin_match(Config) when is_list(Config) -> [{{t,t,1},{return,5,{match_context,{x,0}}}}] = do_val(bad_bin_match, Config), diff --git a/lib/compiler/test/beam_validator_SUITE_data/freg_state.S b/lib/compiler/test/beam_validator_SUITE_data/freg_state.S deleted file mode 100644 index 7466763482..0000000000 --- a/lib/compiler/test/beam_validator_SUITE_data/freg_state.S +++ /dev/null @@ -1,59 +0,0 @@ -{module, freg_state}. %% version = 0 - -{exports, [{sum_1,2},{sum_2,2},{sum_3,2},{sum_4,2},{sum_5,2}]}. - -{attributes, []}. - - -{function, sum_1, 2, 2}. - {label,1}. - {func_info,{atom,t},{atom,sum_1},2}. - {label,2}. - {fconv,{x,0},{fr,0}}. - {fconv,{x,1},{fr,1}}. - {bif,fmul,{f,0},[{fr,0},{fr,1}],{fr,0}}. - {'%live',1}. - return. - -{function, sum_2, 2, 4}. - {label,3}. - {func_info,{atom,t},{atom,sum_2},2}. - {label,4}. - {fconv,{x,0},{fr,0}}. - {fconv,{x,1},{fr,1}}. - fclearerror. - {bif,fmul,{f,0},[{fr,0},{fr,1}],{fr,0}}. - {fmove,{fr,0},{x,0}}. - {'%live',1}. - return. - -{function, sum_3, 2, 6}. - {label,5}. - {func_info,{atom,t},{atom,sum_3},2}. - {label,6}. - {fconv,{x,0},{fr,0}}. - {fconv,{x,1},{fr,1}}. - fclearerror. - {bif,fmul,{f,0},[{fr,0},{fr,1}],{fr,0}}. - {bif,'-',{f,0},[{x,1},{x,0}],{x,1}}. - {fcheckerror,{f,0}}. - {fmove,{fr,0},{x,0}}. - {'%live',1}. - return. - -{function, sum_4, 2, 8}. - {label,6}. - {func_info,{atom,t},{atom,sum_4},2}. - {label,8}. - {fcheckerror,{f,0}}. - {fmove,{fr,0},{x,0}}. - {'%live',1}. - return. - -{function, sum_5, 2, 10}. - {label,9}. - {func_info,{atom,t},{atom,sum_5},2}. - {label,10}. - fclearerror. - fclearerror. - return. diff --git a/lib/compiler/test/float_SUITE.erl b/lib/compiler/test/float_SUITE.erl index 586dfe8102..bf154eeb62 100644 --- a/lib/compiler/test/float_SUITE.erl +++ b/lib/compiler/test/float_SUITE.erl @@ -22,7 +22,7 @@ init_per_group/2,end_per_group/2, pending/1,bif_calls/1,math_functions/1,mixed_float_and_int/1, subtract_number_type/1,float_followed_by_guard/1, - fconv_line_numbers/1]). + fconv_line_numbers/1,exception_signals/1]). -include_lib("common_test/include/ct.hrl"). @@ -31,7 +31,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [pending, bif_calls, math_functions, mixed_float_and_int, subtract_number_type, - float_followed_by_guard,fconv_line_numbers]. + float_followed_by_guard,fconv_line_numbers, + exception_signals]. groups() -> []. @@ -220,5 +221,22 @@ fconv_line_numbers_1(A) -> false end, Stacktrace). +%% ERL-1471: compiler generated invalid 'fclearerror' / 'fcheckerror' +%% sequences. +exception_signals(Config) when is_list(Config) -> + 2.0 = exception_signals_1(id(25), id(true), []), + 2.0 = exception_signals_1(id(25), id(false), []), + 2.0 = exception_signals_1(id(25.0), id(true), []), + 2.0 = exception_signals_1(id(25.0), id(false), []), + ok. + +exception_signals_1(Width, Value, _Opts) -> + Height = Width / 25.0, + _Middle = case Value of + true -> Width / 2.0; + false -> 0 + end, + _More = Height + 1. + id(I) -> I. diff --git a/lib/crypto/c_src/Makefile.in b/lib/crypto/c_src/Makefile.in index ecccc33d8d..0821bd8d00 100644 --- a/lib/crypto/c_src/Makefile.in +++ b/lib/crypto/c_src/Makefile.in @@ -59,11 +59,17 @@ TYPEMARKER = .gprof TYPE_EXTRA_CFLAGS = -DGPROF -pg TYPE_FLAGS = $(CFLAGS) $(TYPE_EXTRA_CFLAGS) else +ifeq ($(TYPE),asan) +TYPEMARKER = .asan +TYPE_FLAGS = $(CFLAGS) -fsanitize=address -fsanitize-recover=address -fno-omit-frame-pointer -DADDRESS_SANITIZER +LDFLAGS += -fsanitize=address +else TYPEMARKER = TYPE_FLAGS = $(CFLAGS) endif endif endif +endif # ---------------------------------------------------- # Release directory specification @@ -159,7 +165,7 @@ ALL_STATIC_CFLAGS = @DED_STATIC_CFLAGS@ $(TYPE_EXTRA_CFLAGS) $(CONFIGURE_ARGS) $ _create_dirs := $(shell mkdir -p $(OBJDIR) $(LIBDIR)) -debug opt valgrind: $(NIF_LIB) $(CALLBACK_LIB) $(TEST_ENGINE_LIB) +debug opt valgrind asan: $(NIF_LIB) $(CALLBACK_LIB) $(TEST_ENGINE_LIB) static_lib: $(NIF_ARCHIVE) diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index b88413d873..4559a15b07 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -163,8 +163,8 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) const ERL_NIF_TERM* tpl_array; int vernum; ErlNifBinary lib_bin; - char lib_buf[1000]; #ifdef HAVE_DYNAMIC_CRYPTO_LIB + char lib_buf[1000]; void *handle; #endif diff --git a/lib/crypto/c_src/crypto_callback.c b/lib/crypto/c_src/crypto_callback.c index 0244952a65..53b4bbf1e0 100644 --- a/lib/crypto/c_src/crypto_callback.c +++ b/lib/crypto/c_src/crypto_callback.c @@ -106,9 +106,8 @@ static void crypto_free(void* ptr CCB_FILE_LINE_ARGS) #ifdef OPENSSL_THREADS /* vvvvvvvvvvvvvvv OPENSSL_THREADS vvvvvvvvvvvvvvvv */ -#if OPENSSL_VERSION_NUMBER < 0x10100000 +#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) static ErlNifRWLock** lock_vec = NULL; /* Static locks used by openssl */ -#endif #include <openssl/crypto.h> @@ -132,8 +131,6 @@ static INLINE void locking(int mode, ErlNifRWLock* lock) } } -#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) - /* TODO: there should be an enif_atomic32_add_return() */ typedef int (*add_lock_function_t)(int *var, int incr, int type, const char *file, int line); @@ -192,21 +189,18 @@ DLLEXPORT struct crypto_callbacks* get_crypto_callbacks(int nlocks) &crypto_realloc, &crypto_free, -#if OPENSSL_VERSION_NUMBER < 0x10100000 -#ifdef OPENSSL_THREADS +#if defined OPENSSL_THREADS && OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) NULL, /* add_lock_function, filled in below */ &locking_function, &id_function, &dyn_create_function, &dyn_lock_function, &dyn_destroy_function -#endif /* OPENSSL_THREADS */ -#endif +#endif /* OPENSSL_THREADS && PACKED_OPENSSL_VERSION_PLAIN(1,1,0) */ }; if (!is_initialized) { -#if OPENSSL_VERSION_NUMBER < 0x10100000 -#ifdef OPENSSL_THREADS +#if defined OPENSSL_THREADS && OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) the_struct.add_lock_function = get_add_lock_function(); if (nlocks > 0) { int i; @@ -223,18 +217,15 @@ DLLEXPORT struct crypto_callbacks* get_crypto_callbacks(int nlocks) goto err; } } -#endif -#endif +#endif /* OPENSSL_THREADS && PACKED_OPENSSL_VERSION_PLAIN(1,1,0) */ is_initialized = 1; } return &the_struct; -#if OPENSSL_VERSION_NUMBER < 0x10100000 -#ifdef OPENSSL_THREADS +#if defined OPENSSL_THREADS && OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) err: return NULL; #endif -#endif } #ifdef HAVE_DYNAMIC_CRYPTO_LIB diff --git a/lib/crypto/c_src/info.c b/lib/crypto/c_src/info.c index 573039203c..1d7e744995 100644 --- a/lib/crypto/c_src/info.c +++ b/lib/crypto/c_src/info.c @@ -26,6 +26,8 @@ char *crypto_callback_name = "crypto_callback.debug"; # elif defined(VALGRIND) char *crypto_callback_name = "crypto_callback.valgrind"; +# elif defined(ADDRESS_SANITIZER) +char *crypto_callback_name = "crypto_callback.asan"; # else char *crypto_callback_name = "crypto_callback"; # endif diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index cf63bd6051..1a11ab12fe 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -120,7 +120,7 @@ #endif #if defined(HAS_EVP_PKEY_CTX) \ - && OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,0,2) + && OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) /* EVP is slow on antique crypto libs. * DISABLE_EVP_* is 0 or 1 from the configure script */ diff --git a/lib/crypto/c_src/srp.c b/lib/crypto/c_src/srp.c index 2afd05d7b5..22fbedb4eb 100644 --- a/lib/crypto/c_src/srp.c +++ b/lib/crypto/c_src/srp.c @@ -57,6 +57,7 @@ ERL_NIF_TERM srp_value_B_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] goto err; /* g^b % N */ + BN_set_flags(bn_exponent, BN_FLG_CONSTTIME); if (!BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx)) goto err; @@ -154,6 +155,7 @@ ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar /* (B - (k * g^x)) */ if ((bn_base = BN_new()) == NULL) goto err; + BN_set_flags(bn_exponent, BN_FLG_CONSTTIME); if (!BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx)) goto err; if (!BN_mod_mul(bn_result, bn_multiplier, bn_result, bn_prime, bn_ctx)) @@ -170,6 +172,7 @@ ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar goto err; /* (B - (k * g^x)) ^ (a + (u * x)) % N */ + BN_set_flags(bn_exp2, BN_FLG_CONSTTIME); if (!BN_mod_exp(bn_result, bn_base, bn_exp2, bn_prime, bn_ctx)) goto err; @@ -258,12 +261,14 @@ ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar /* (A * v^u) */ if ((bn_base = BN_new()) == NULL) goto err; + BN_set_flags(bn_u, BN_FLG_CONSTTIME); if (!BN_mod_exp(bn_base, bn_verifier, bn_u, bn_prime, bn_ctx)) goto err; if (!BN_mod_mul(bn_base, bn_A, bn_base, bn_prime, bn_ctx)) goto err; /* (A * v^u) ^ b % N */ + BN_set_flags(bn_b, BN_FLG_CONSTTIME); if (!BN_mod_exp(bn_result, bn_base, bn_b, bn_prime, bn_ctx)) goto err; diff --git a/lib/crypto/doc/src/Makefile b/lib/crypto/doc/src/Makefile index f48a79e8d1..b4926d6d7c 100644 --- a/lib/crypto/doc/src/Makefile +++ b/lib/crypto/doc/src/Makefile @@ -48,4 +48,4 @@ TOP_SPECS_FILE = specs.xml include $(ERL_TOP)/make/doc.mk -valgrind: +valgrind asan: diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 690037ec05..aea2139351 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,37 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.8.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adding missing flag in BN-calls in SRP.</p> + <p> + Own Id: OTP-17107</p> + </item> + </list> + </section> + +</section> + +<section><title>Crypto 4.8.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed usage of <c>AC_CONFIG_AUX_DIRS()</c> macros in + configure script sources.</p> + <p> + Own Id: OTP-17093 Aux Id: ERL-1447, PR-2948 </p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.8.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -211,6 +242,21 @@ </section> +<section><title>Crypto 4.6.5.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adding missing flag in BN-calls in SRP.</p> + <p> + Own Id: OTP-17107</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.6.5.1</title> <section><title>Improvements and New Features</title> @@ -523,6 +569,21 @@ </section> +<section><title>Crypto 4.4.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adding missing flag in BN-calls in SRP.</p> + <p> + Own Id: OTP-17107</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.4.2.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/Makefile b/lib/crypto/src/Makefile index 1753ba4f36..c3f1c859e5 100644 --- a/lib/crypto/src/Makefile +++ b/lib/crypto/src/Makefile @@ -61,7 +61,7 @@ ERL_COMPILE_FLAGS += -DCRYPTO_VSN=\"$(VSN)\" -Werror -I../include # Targets # ---------------------------------------------------- -debug opt valgrind: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) +debug opt valgrind asan: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) clean: rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index d178ecc28e..b38d13d844 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -2985,10 +2985,10 @@ get_test_engine() -> end. check_otp_test_engine(LibDir) -> - case filelib:wildcard("otp_test_engine*", LibDir) of - [] -> + case choose_otp_test_engine(LibDir) of + false -> {error, notexist}; - [LibName|_] -> % In case of Valgrind there could be more than one + LibName -> LibPath = filename:join(LibDir,LibName), case filelib:is_file(LibPath) of true -> @@ -2999,3 +2999,20 @@ check_otp_test_engine(LibDir) -> end. +choose_otp_test_engine(LibDir) -> + LibNames = filelib:wildcard("otp_test_engine.*", LibDir), + Type = atom_to_list(erlang:system_info(build_type)), + choose_otp_test_engine(LibNames, Type, false). + +choose_otp_test_engine([LibName | T], Type, Acc) -> + case string:lexemes(LibName, ".") of + [_, Type, _SO] -> + LibName; %% Choose typed if exists (valgrind,asan) + [_, _SO] -> + %% Fallback on typeless (opt) + choose_otp_test_engine(T, Type, LibName); + _ -> + choose_otp_test_engine(T, Type, Acc) + end; +choose_otp_test_engine([], _, Acc) -> + Acc. diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index aab941b455..0fb42b3993 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.8.1 +CRYPTO_VSN = 4.8.3 diff --git a/lib/eldap/include/eldap.hrl b/lib/eldap/include/eldap.hrl index 7c12cd4f2b..b670de871f 100644 --- a/lib/eldap/include/eldap.hrl +++ b/lib/eldap/include/eldap.hrl @@ -7,6 +7,7 @@ -record(eldap_search, { base = [], % Baseobject filter = [], % Search conditions + size_limit = 0, % Setting default size limit to 0 makes it unlimited scope=wholeSubtree, % Search scope deref=derefAlways, % Dereference attributes = [], % Attributes to be returned diff --git a/lib/eldap/src/eldap.erl b/lib/eldap/src/eldap.erl index 9b7e254dfe..c647083024 100644 --- a/lib/eldap/src/eldap.erl +++ b/lib/eldap/src/eldap.erl @@ -326,6 +326,8 @@ parse_search_args([{base, Base}|T],A) -> parse_search_args(T,A#eldap_search{base = Base}); parse_search_args([{filter, Filter}|T],A) -> parse_search_args(T,A#eldap_search{filter = Filter}); +parse_search_args([{size_limit, SizeLimit}|T],A) when is_integer(SizeLimit) -> + parse_search_args(T,A#eldap_search{size_limit = SizeLimit}); parse_search_args([{scope, Scope}|T],A) -> parse_search_args(T,A#eldap_search{scope = Scope}); parse_search_args([{deref, Deref}|T],A) -> @@ -749,7 +751,7 @@ do_search_0(Data, A, Controls) -> Req = #'SearchRequest'{baseObject = A#eldap_search.base, scope = v_scope(A#eldap_search.scope), derefAliases = v_deref(A#eldap_search.deref), - sizeLimit = 0, % no size limit + sizeLimit = v_size_limit(A#eldap_search.size_limit), timeLimit = v_timeout(A#eldap_search.timeout), typesOnly = v_bool(A#eldap_search.types_only), filter = v_filter(A#eldap_search.filter), @@ -777,6 +779,9 @@ collect_search_responses(Data, S, ID, {ok,Msg}, Acc, Ref) success -> log2(Data, "search reply = searchResDone ~n", []), {ok,Acc,Ref,Data}; + sizeLimitExceeded -> + log2(Data, "[TRUNCATED] search reply = searchResDone ~n", []), + {ok,Acc,Ref,Data}; referral -> {{ok, {referral,R#'LDAPResult'.referral}}, Data}; Reason -> @@ -1088,6 +1093,9 @@ v_bool(true) -> true; v_bool(false) -> false; v_bool(_Bool) -> throw({error,concat(["not Boolean: ",_Bool])}). +v_size_limit(I) when is_integer(I), I>=0 -> I; +v_size_limit(_I) -> throw({error,concat(["size_limit not positive integer: ",_I])}). + v_timeout(I) when is_integer(I), I>=0 -> I; v_timeout(_I) -> throw({error,concat(["timeout not positive integer: ",_I])}). diff --git a/lib/eldap/test/eldap_basic_SUITE.erl b/lib/eldap/test/eldap_basic_SUITE.erl index a337fe1c21..683e1d5393 100644 --- a/lib/eldap/test/eldap_basic_SUITE.erl +++ b/lib/eldap/test/eldap_basic_SUITE.erl @@ -49,9 +49,12 @@ search_filter_and/1, search_filter_and_not/1, search_filter_equalityMatch/1, + search_filter_equalityMatch_objectClass_exists/1, search_filter_final/1, search_filter_initial/1, search_filter_or/1, + search_filter_or_sizelimit_ok/1, + search_filter_or_sizelimit_exceeded/1, search_filter_substring_any/1, search_non_existant/1, search_referral/1, @@ -118,6 +121,7 @@ groups() -> more_add, add_referral, search_filter_equalityMatch, + search_filter_equalityMatch_objectClass_exists, search_filter_substring_any, search_filter_initial, search_filter_final, @@ -126,6 +130,8 @@ groups() -> search_filter_and_not, search_two_hits, search_referral, + search_filter_or_sizelimit_ok, + search_filter_or_sizelimit_exceeded, modify, modify_referral, delete, @@ -569,6 +575,17 @@ search_filter_equalityMatch(Config) -> scope=eldap:singleLevel()}). %%%---------------------------------------------------------------- +search_filter_equalityMatch_objectClass_exists(Config) -> + BasePath = proplists:get_value(eldap_path, Config), + ExpectedDN = "cn=Jonas Jonsson," ++ BasePath, + {ok, #eldap_search_result{entries=[#eldap_entry{object_name=ExpectedDN}]}} = + eldap:search(proplists:get_value(handle, Config), + #eldap_search{base = BasePath, + filter = eldap:'and'([eldap:equalityMatch("sn", "Jonsson"), + eldap:present("objectclass")]), + scope=eldap:singleLevel()}). + +%%%---------------------------------------------------------------- search_filter_substring_any(Config) -> BasePath = proplists:get_value(eldap_path, Config), ExpectedDN = "cn=Jonas Jonsson," ++ BasePath, @@ -627,6 +644,39 @@ search_filter_or(Config) -> ExpectedDNs = lists:sort([DN || #eldap_entry{object_name=DN} <- Es]). %%%---------------------------------------------------------------- +search_filter_or_sizelimit_ok(Config) -> + H = proplists:get_value(handle, Config), + BasePath = proplists:get_value(eldap_path, Config), + ExpectedDNs = lists:sort(["cn=Foo Bar," ++ BasePath, + "ou=Team," ++ BasePath]), + {ok, #eldap_search_result{entries=Es}} = + eldap:search(H, + #eldap_search{base = BasePath, + filter = eldap:'or'([eldap:substrings("sn", [{any, "a"}]), + eldap:equalityMatch("ou","Team")]), + size_limit = 2, + scope=eldap:singleLevel()}), + ExpectedDNs = lists:sort([DN || #eldap_entry{object_name=DN} <- Es]). + +%%%---------------------------------------------------------------- +search_filter_or_sizelimit_exceeded(Config) -> + H = proplists:get_value(handle, Config), + BasePath = proplists:get_value(eldap_path, Config), + %% The quesry without the {size_limit,1} option would return two answers: + ExpectedDNs = ["cn=Foo Bar," ++ BasePath, + "ou=Team," ++ BasePath], + %% Expect exact one of the two answers, but we don't know which: + {ok, #eldap_search_result{entries=[E]}} = + eldap:search(H, + #eldap_search{base = BasePath, + filter = eldap:'or'([eldap:substrings("sn", [{any, "a"}]), + eldap:equalityMatch("ou","Team")]), + size_limit = 1, + scope=eldap:singleLevel()}), + #eldap_entry{object_name=DN} = E, + true = lists:member(DN, ExpectedDNs). + +%%%---------------------------------------------------------------- search_filter_and_not(Config) -> H = proplists:get_value(handle, Config), BasePath = proplists:get_value(eldap_path, Config), diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 97f4460e1d..fde35537ba 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,49 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 4.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Integers outside of the range [-(1 bsl 32) - 1, (1 bsl + 32) -1] were previously intended to be printed in an + internal bignum format by <c>ei_print_term()</c> and + <c>ei_s_print_term()</c>. Unfortunately the + implementation has been buggy since OTP R13B02 and since + then produced results with random content which also + could crash the calling program.</p> + <p> + This fix replaces the printing of the internal format + with printing in hexadecimal form and extend the range + for printing in decimal form. Currently integers in the + range [-(1 bsl 64), (1 bsl 64)] are printed in decimal + form and integers outside of this range in Erlang + hexadecimal form.</p> + <p> + Own Id: OTP-17099 Aux Id: ERIERL-585 </p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + The <c>ei</c> API for decoding/encoding terms is not + fully 64-bit compatible since terms that have a + representation on the external term format larger than 2 + GB cannot be handled.</p> + <p> + Own Id: OTP-16607 Aux Id: OTP-16608 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 4.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -410,6 +453,39 @@ </section> +<section><title>Erl_Interface 3.11.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix link error "multiple definition of + `ei_default_socket_callbacks'" for gcc version 10 or when + built with gcc option -fno-common. Error exists since + OTP-21.3.</p> + <p> + Own Id: OTP-16412 Aux Id: PR-2503 </p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + The <c>ei</c> API for decoding/encoding terms is not + fully 64-bit compatible since terms that have a + representation on the external term format larger than 2 + GB cannot be handled.</p> + <p> + Own Id: OTP-16607 Aux Id: OTP-16608 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.11.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c index d7d3c0e3e3..0bad730095 100644 --- a/lib/erl_interface/src/misc/ei_printterm.c +++ b/lib/erl_interface/src/misc/ei_printterm.c @@ -87,24 +87,73 @@ static char *ei_big_to_str(erlang_big *b) { int buf_len; char *s,*buf; + unsigned int no_digits; unsigned short *sp; int i; + int printed; - buf_len = 64+b->is_neg+10*b->arity; - if ( (buf=malloc(buf_len)) == NULL) return NULL; + /* Number of 16-bit digits */ + no_digits = (b->arity + 1) / 2; + + if (no_digits <= 4) { + EI_ULONGLONG val; + buf_len = 22; + s = buf = malloc(buf_len); + if (!buf) + return NULL; + val = 0; + sp=b->digits; + for (i = 0; i < no_digits; i++) + val |= ((EI_ULONGLONG) sp[i]) << (i*16); + if (b->is_neg) + s += sprintf(s,"-"); + sprintf(s, "%llu", val); + return buf; + } - memset(buf,(char)0,buf_len); + /* big nums this large gets printed in base 16... */ + buf_len = (!!b->is_neg /* "-" */ + + 3 /* "16#" */ + + 4*no_digits /* 16-bit digits in base 16 */ + + 1); /* \0 */ + if ( (buf=malloc(buf_len)) == NULL) return NULL; s = buf; if ( b->is_neg ) - s += sprintf(s,"-"); - s += sprintf(s,"#integer(%d) = {",b->arity); - for(sp=b->digits,i=0;i<b->arity;i++) { - s += sprintf(s,"%d",sp[i]); - if ( (i+1) != b->arity ) - s += sprintf(s,","); + *(s++) = '-'; + *(s++) = '1'; + *(s++) = '6'; + *(s++) = '#'; + + sp = b->digits; + printed = 0; + for (i = no_digits - 1; i >= 0; i--) { + unsigned short val = sp[i]; + int j; + + for (j = 3; j >= 0; j--) { + char c = (char) ((val >> (j*4)) & 0xf); + if (c < 10) + c += '0'; + else + c += 'A' - 10; + + if (printed) + *(s++) = c; + else if (c != '0') { + *(s++) = c; + printed = !0; + } + } } - s += sprintf(s,"}"); + + if (!printed) { + /* very strange to encode zero like this... */ + *(s++) = '0'; + } + + + *s = '\0'; return buf; } diff --git a/lib/erl_interface/test/ei_print_SUITE.erl b/lib/erl_interface/test/ei_print_SUITE.erl index 25dd95649d..43d74066a2 100644 --- a/lib/erl_interface/test/ei_print_SUITE.erl +++ b/lib/erl_interface/test/ei_print_SUITE.erl @@ -27,7 +27,8 @@ -export([all/0, suite/0, init_per_testcase/2, atoms/1, tuples/1, lists/1, strings/1, - maps/1, funs/1, binaries/1, bitstrings/1]). + maps/1, funs/1, binaries/1, bitstrings/1, + integers/1]). -import(runner, [get_term/1]). @@ -38,7 +39,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [atoms, tuples, lists, strings, maps, funs, binaries, bitstrings]. + [atoms, tuples, lists, strings, maps, funs, binaries, bitstrings, integers]. init_per_testcase(Case, Config) -> runner:init_per_testcase(?MODULE, Case, Config). @@ -198,6 +199,69 @@ bitstrings(Config) -> runner:recv_eot(P), ok. +integers(Config) -> + Port = runner:start(Config, ?integers), + + test_integers(Port, -1000, 1000), + test_integers(Port, (1 bsl 27) - 1000, (1 bsl 27) + 1000), + test_integers(Port, -(1 bsl 27) - 1000, -(1 bsl 27) + 1000), + test_integers(Port, (1 bsl 28) - 1000, (1 bsl 28) + 1000), + test_integers(Port, -(1 bsl 28) - 1000, -(1 bsl 28) + 1000), + test_integers(Port, (1 bsl 31) - 1000, (1 bsl 31) + 1000), + test_integers(Port, -(1 bsl 31) - 1000, -(1 bsl 31) + 1000), + test_integers(Port, (1 bsl 32) - 1000, (1 bsl 32) + 1000), + test_integers(Port, -(1 bsl 32) - 1000, -(1 bsl 32) + 1000), + test_integers(Port, (1 bsl 60) - 1000, (1 bsl 60) + 1000), + test_integers(Port, -(1 bsl 60) - 1000, -(1 bsl 60) + 1000), + test_integers(Port, 16#feeddeaddeadbeef - 1000, 16#feeddeaddeadbeef + 1000), + test_integers(Port, -16#feeddeaddeadbeef - 1000, -16#feeddeaddeadbeef + 1000), + test_integers(Port, (1 bsl 64) - 1000, (1 bsl 64) + 1000), + test_integers(Port, 16#addfeeddeaddeadbeef - 1000, 16#addfeeddeaddeadbeef + 1000), + test_integers(Port, -16#addfeeddeaddeadbeef - 1000, -16#addfeeddeaddeadbeef + 1000), + test_integers(Port, -(1 bsl 64) - 1000, -(1 bsl 64) + 1000), + test_integers(Port, (1 bsl 8192) - 1000, (1 bsl 8192) + 1000), + test_integers(Port, -(1 bsl 8192) - 1000, -(1 bsl 8192) + 1000), + + "done" = send_term_get_printed(Port, done), + + runner:recv_eot(Port), + + ok. + +test_integer(Port, Int, Print) when is_integer(Int) -> + Res = send_term_get_printed(Port, Int), + case Print of + true -> + io:format("Res: ~s~n", [Res]); + false -> + ok + end, + %% Large bignums are printed in base 16... + Exp = case Res of + "16#" ++ _ -> + "16#" ++ integer_to_list(Int, 16); + "-16#" ++ _ -> + "-16#" ++ integer_to_list(-1*Int, 16); + _ -> + integer_to_list(Int) + end, + case Exp =:= Res of + true -> + ok; + false -> + io:format("Exp: ~s~nRes: ~s~n", [Exp, Res]), + ct:fail({Exp, Res}) + end. + +test_integers(Port, FromInt, ToInt) -> + test_integers(Port, FromInt, ToInt, true). + +test_integers(_Port, FromInt, ToInt, _Print) when FromInt > ToInt -> + ok; +test_integers(Port, FromInt, ToInt, Print) -> + ok = test_integer(Port, FromInt, Print), + NewFromInt = FromInt + 1, + test_integers(Port, NewFromInt, ToInt, NewFromInt == ToInt). send_term_get_printed(Port, Term) -> diff --git a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c index 4b23701e82..b840c4aca0 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c +++ b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c @@ -345,3 +345,70 @@ TESTCASE(bitstrings) } report(1); } + +TESTCASE(integers) +{ + char *buf; + long len; + int err, n, index, done; + ei_x_buff x; + + ei_init(); + + done = 0; + do { + int type, type_len; + buf = read_packet(NULL); + + index = 0; + err = ei_decode_version(buf, &index, NULL); + if (err != 0) + fail1("ei_decode_version returned %d", err); + err = ei_get_type(buf, &index, &type, &type_len); + if (err) + fail1("ei_get_type() returned %d", err); + switch (type) { + case ERL_SMALL_INTEGER_EXT: + case ERL_INTEGER_EXT: { + long val; + err = ei_decode_long(buf, &index, &val); + if (err) + fail1("ei_decode_long() returned %d", err); + break; + } + case ERL_SMALL_BIG_EXT: + case ERL_LARGE_BIG_EXT: { + erlang_big *big = ei_alloc_big(type_len); + if (!big) + fail1("ei_alloc_big() failed %d", ENOMEM); + err = ei_decode_big(buf, &index, big); + if (err) + fail1("ei_decode_big() failed %d", err); + ei_free_big(big); + break; + } + case ERL_ATOM_EXT: { + char abuf[MAXATOMLEN]; + err = ei_decode_atom(buf, &index, &abuf[0]); + if (err) + fail1("ei_decode_atom() failed %d", err); + if (strcmp("done", &abuf[0]) == 0) + done = 1; + break; + } + default: + fail1("Unexpected type %d", type); + break; + } + + ei_x_new(&x); + ei_x_append_buf(&x, buf, index); + send_printed_buf(&x); + ei_x_free(&x); + + free_packet(buf); + + } while (!done); + + report(1); +} diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index e454f9e1d1..d928057ca4 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 4.0.1 +EI_VSN = 4.0.2 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index 7d82463d12..21d9a87304 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -137,7 +137,7 @@ be more that one server that has the same bind_address and port. If this property is not explicitly set, it is assumed that the <seeerl marker="#prop_bind_address"><c>bind_address</c></seeerl> and - <seeerl marker="#prop_port"><c>port</c></seeerl>uniquely identifies the HTTP server. + <seeerl marker="#prop_port"><c>port</c></seeerl> uniquely identifies the HTTP server. </p> </item> @@ -256,9 +256,9 @@ <tag><marker id="max_client_body_chunk"></marker>{max_client_body_chunk, integer()}</tag> <item> - <p>Enforces chunking of a HTTP PUT or POST body data to be deliverd + <p>Enforces chunking of a HTTP PUT or POST body data to be delivered to the mod_esi callback. Note this is not supported for mod_cgi. - Default is no limit e.i the whole body is deliverd as one entity, which could + Default is no limit e.i the whole body is delivered as one entity, which could be very memory consuming. <seeerl marker="mod_esi">mod_esi(3)</seeerl>. </p> </item> @@ -275,7 +275,7 @@ 1590. File suffixes are mapped to MIME types before file delivery. The mapping between file suffixes and MIME types can be specified as an Apache-like file or directly in the property list. Such - a file can look like the follwoing:</p> + a file can look like the following:</p> <pre> # MIME type Extension text/html html htm @@ -863,7 +863,7 @@ Transport: TLS <tag><marker id="prop_block_time"></marker>{block_time, integer()}</tag> <item> <p>Specifies the number of minutes a user is blocked. After - this timehas passed, the user automatically regains access. + this time has passed, the user automatically regains access. Default is <c>60</c>.</p> </item> @@ -1110,7 +1110,7 @@ Transport: TLS <p>If <c>Body</c> is returned and equal to <c>{Fun,Arg}</c>, the web server tries <c>apply/2</c> on <c>Fun</c> with <c>Arg</c> as argument. The web server expects that the fun either - returns a list <c>(Body)</c> that is an HTTP repsonse, or the + returns a list <c>(Body)</c> that is an HTTP response, or the atom <c>sent</c> if the HTTP response is sent back to the client. If <c>close</c> is returned from the fun, something has gone wrong and the server signals this to the client by diff --git a/lib/kernel/doc/src/erl_epmd.xml b/lib/kernel/doc/src/erl_epmd.xml index 03aa949516..f6fe3c0a9e 100644 --- a/lib/kernel/doc/src/erl_epmd.xml +++ b/lib/kernel/doc/src/erl_epmd.xml @@ -56,8 +56,10 @@ <desc> <p>Registers the node with <c>epmd</c> and tells epmd what port will be used for the current node. It returns a creation number. This number is - incremented on each register to help with identifying if a node is - reconnecting to epmd.</p> + incremented on each register to help differentiate a new node instance + connecting to epmd with the same name.</p> + <p>After the node has successfully registered with epmd it will automatically + attempt reconnect to the daemon if the connection is broken.</p> </desc> </func> diff --git a/lib/kernel/doc/src/erpc.xml b/lib/kernel/doc/src/erpc.xml index 64032a7f94..8b15c38a56 100644 --- a/lib/kernel/doc/src/erpc.xml +++ b/lib/kernel/doc/src/erpc.xml @@ -347,7 +347,7 @@ <desc> <p> The same as calling - <seemfa marker="#call/5"><c>erpc:multicall(<anno>Nodes</anno>,erlang,apply,[<anno>Fun</anno>,[]],<anno>Timeout</anno>)</c></seemfa>. + <seemfa marker="#multicall/5"><c>erpc:multicall(<anno>Nodes</anno>,erlang,apply,[<anno>Fun</anno>,[]],<anno>Timeout</anno>)</c></seemfa>. May raise all the same exceptions as <c>erpc:multicall/5</c> plus an <c>{erpc, badarg}</c> <c>error</c> exception if <c><anno>Fun</anno></c> is not a fun of @@ -471,7 +471,7 @@ my_multicall(Nodes, Module, Function, Args) -> <desc> <p> The same as calling - <seemfa marker="#cast/4"><c>erpc:multicast(<anno>Nodes</anno>,erlang,apply,[<anno>Fun</anno>,[]])</c></seemfa>. + <seemfa marker="#multicast/4"><c>erpc:multicast(<anno>Nodes</anno>,erlang,apply,[<anno>Fun</anno>,[]])</c></seemfa>. </p> <p><c>erpc:multicast/2</c> fails with an <c>{erpc, badarg}</c> <c>error</c> exception if:</p> @@ -586,7 +586,7 @@ my_call(Node, Module, Function, Args, Timeout) -> <desc> <p> The same as calling - <seemfa marker="#call/5"><c>erpc:send_request(<anno>Node</anno>,erlang,apply,[<anno>Fun</anno>,[]])</c></seemfa>. + <seemfa marker="#send_request/4"><c>erpc:send_request(<anno>Node</anno>,erlang,apply,[<anno>Fun</anno>,[]])</c></seemfa>. </p> <p><c>erpc:send_request/2</c> fails with an <c>{erpc, badarg}</c> <c>error</c> exception if:</p> diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index d4876e5e07..801a2c1193 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -156,7 +156,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro the field named <c>config</c>. See the <seeerl marker="logger_std_h"><c>logger_std_h(3)</c></seeerl> and <seeerl marker="logger_disk_log_h"><c>logger_disk_log_h(3)</c></seeerl> - manual pages for information about the specifc configuration + manual pages for information about the specific configuration for these handlers.</p> <p>See the <seetype marker="logger_formatter#config"> <c>logger_formatter(3)</c></seetype> manual page for @@ -913,7 +913,7 @@ start(_, []) -> <fsummary>Unset the log level for all modules in the specified application.</fsummary> <desc> <p>Unset the log level for all the modules of the specified application.</p> - <p>This function is a convinience function that calls + <p>This function is a utility function that calls <seemfa marker="#unset_module_level/1">logger:unset_module_level/2</seemfa> for each module associated with an application.</p> </desc> @@ -1160,7 +1160,7 @@ logger:set_proxy_config(maps:merge(Old, Config)). </type> <desc> <p>This callback function is optional.</p> - <p>The function is called on a temporary process when an new + <p>The function is called on a temporary process when a new handler is about to be added. The purpose is to verify the configuration and initiate all resources needed by the handler.</p> @@ -1199,7 +1199,7 @@ logger:set_proxy_config(maps:merge(Old, Config)). <c>set_handler_config/2,3</c></seemfa>, and <c>update</c> if it originates from <seemfa marker="#update_handler_config/2"> <c>update_handler_config/2,3</c></seemfa>. The handler can - use this parameteter to decide how to update the value of + use this parameter to decide how to update the value of the <c>config</c> field, that is, the handler specific configuration data. Typically, if <c>SetOrUpdate</c> equals <c>set</c>, values that are not specified must be diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml index 666aca988f..fc172b4306 100644 --- a/lib/kernel/doc/src/os.xml +++ b/lib/kernel/doc/src/os.xml @@ -37,27 +37,11 @@ use, these functions can be of help in enabling a program to run on most platforms.</p> - <note> - <p> - File operations used to accept filenames containing - null characters (integer value zero). This caused - the name to be truncated and in some cases arguments - to primitive operations to be mixed up. Filenames - containing null characters inside the filename - are now <em>rejected</em> and will cause primitive - file operations to fail. - </p> - <p> - Also environment variable operations used to accept - names and values of environment variables containing - null characters (integer value zero). This caused - operations to silently produce erroneous results. - Environment variable names and values containing - null characters inside the name or value are now - <em>rejected</em> and will cause environment variable - operations to fail. - </p> - </note> + <note> + <p>The functions in this module will raise a <c>badarg</c> exception + if their arguments contain invalid characters according to the + description in the "Data Types" section.</p> + </note> </description> <datatypes> @@ -67,11 +51,9 @@ <p>A string containing valid characters on the specific OS for environment variable names using <seemfa marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seemfa> - encoding. Note that specifically null characters (integer - value zero) and <c>$=</c> characters are not allowed. - However, note that not all invalid characters necessarily - will cause the primitiv operations to fail, but may instead - produce invalid results. + encoding. Null characters (integer value zero) are not allowed. On Unix, + <c>=</c> characters are not allowed. On Windows, a <c>=</c> character is only + allowed as the very first character in the string. </p> </desc> </datatype> @@ -81,10 +63,7 @@ <p>A string containing valid characters on the specific OS for environment variable values using <seemfa marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seemfa> - encoding. Note that specifically null characters (integer - value zero) are not allowed. However, note that not all - invalid characters necessarily will cause the primitiv - operations to fail, but may instead produce invalid results. + encoding. Null characters (integer value zero) are not allowed. </p> </desc> </datatype> @@ -96,7 +75,7 @@ set, a strings containing valid characters on the specific OS for environment variable names and values using <seemfa marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seemfa> - encoding. The first <c>$=</c> characters appearing in + encoding. The first <c>=</c> characters appearing in the string separates environment variable name (on the left) from environment variable value (on the right). </p> @@ -105,14 +84,11 @@ <datatype> <name name="os_command"/> <desc> - <p>All characters needs to be valid characters on the - specific OS using - <seemfa marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seemfa> - encoding. Note that specifically null characters (integer - value zero) are not allowed. However, note that not all - invalid characters not necessarily will cause - <seemfa marker="#cmd/1"><c>os:cmd/1</c></seemfa> - to fail, but may instead produce invalid results. + <p>All characters needs to be valid characters on the specific + OS using <seemfa + marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seemfa> + encoding. Null characters (integer value zero) are not + allowed. </p> </desc> </datatype> @@ -123,7 +99,7 @@ <taglist> <tag><c>max_size</c></tag> <item> - <p>The maximum size of the data returned by the <c>os:cmd</c> call. + <p>The maximum size of the data returned by the <c>os:cmd/2</c> call. See the <seemfa marker="#cmd/2"><c>os:cmd/2</c></seemfa> documentation for more details.</p> </item> @@ -141,11 +117,6 @@ <p>Executes <c><anno>Command</anno></c> in a command shell of the target OS, captures the standard output of the command, and returns this result as a string.</p> - <warning><p>Previous implementation used to allow all characters - as long as they were integer values greater than or equal to zero. - This sometimes lead to unwanted results since null characters - (integer value zero) often are interpreted as string termination. The - current implementation rejects these.</p></warning> <p><em>Examples:</em></p> <code type="none"> LsOut = os:cmd("ls"), % on unix platform @@ -264,15 +235,6 @@ DirOut = os:cmd("dir"), % on Win32 platform</code> <p>On Unix platforms, the environment is set using UTF-8 encoding if Unicode filename translation is in effect. On Windows, the environment is set using wide character interfaces.</p> - <note> - <p> - <c><anno>VarName</anno></c> is not allowed to contain - an <c>$=</c> character. Previous implementations used - to just let the <c>$=</c> character through which - silently caused erroneous results. Current implementation - will instead throw a <c>badarg</c> exception. - </p> - </note> </desc> </func> diff --git a/lib/kernel/src/erl_epmd.erl b/lib/kernel/src/erl_epmd.erl index 7cc84b2475..96806ae3e7 100644 --- a/lib/kernel/src/erl_epmd.erl +++ b/lib/kernel/src/erl_epmd.erl @@ -53,13 +53,15 @@ -import(lists, [reverse/1]). --record(state, {socket, port_no = -1, name = ""}). +-record(state, {socket, port_no = -1, name = "", family}). -type state() :: #state{}. -include("inet_int.hrl"). -include("erl_epmd.hrl"). -include_lib("kernel/include/inet.hrl"). +-define(RECONNECT_TIME, 2000). + %%%---------------------------------------------------------------------- %%% API %%%---------------------------------------------------------------------- @@ -228,7 +230,8 @@ handle_call({register, Name, PortNo, Family}, _From, State) -> {alive, Socket, Creation} -> S = State#state{socket = Socket, port_no = PortNo, - name = Name}, + name = Name, + family = Family}, {reply, {ok, Creation}, S}; Error -> case init:get_argument(erl_epmd_port) of @@ -263,7 +266,17 @@ handle_cast(_, State) -> -spec handle_info(term(), state()) -> {'noreply', state()}. handle_info({tcp_closed, Socket}, State) when State#state.socket =:= Socket -> + erlang:send_after(?RECONNECT_TIME, self(), reconnect), {noreply, State#state{socket = -1}}; +handle_info(reconnect, State) when State#state.socket =:= -1 -> + case do_register_node(State#state.name, State#state.port_no, State#state.family) of + {alive, Socket, _Creation} -> + %% ignore the received creation + {noreply, State#state{socket = Socket}}; + _Error -> + erlang:send_after(?RECONNECT_TIME, self(), reconnect), + {noreply, State} + end; handle_info(_, State) -> {noreply, State}. diff --git a/lib/kernel/src/inet_dns.erl b/lib/kernel/src/inet_dns.erl index e03f124fe6..18103771dc 100644 --- a/lib/kernel/src/inet_dns.erl +++ b/lib/kernel/src/inet_dns.erl @@ -697,7 +697,7 @@ encode_labels(Bin, Comp0, Pos, [L|Ls]=Labels) when 1 =< byte_size(L), byte_size(L) =< 63 -> case gb_trees:lookup(Labels, Comp0) of none -> - Comp = if Pos < (3 bsl 14) -> + Comp = if Pos < (1 bsl 14) -> %% Just in case - compression %% pointers cannot reach further gb_trees:insert(Labels, Pos, Comp0); diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 67faa4911c..1c7b067375 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -30,6 +30,7 @@ nodenames/1, hostnames/1, illegal_nodenames/1, hidden_node/1, dyn_node_name/1, + epmd_reconnect/1, setopts/1, table_waste/1, net_setuptime/1, inet_dist_options_options/1, @@ -54,6 +55,7 @@ tick_serv_test/2, tick_serv_test1/1, run_remote_test/1, dyn_node_name_do/2, + epmd_reconnect_do/2, setopts_do/2, keep_conn/1, time_ping/1]). @@ -64,6 +66,8 @@ -export([pinger/1]). -define(DUMMY_NODE,dummy@test01). +-define(ALT_EPMD_PORT, "12321"). +-define(ALT_EPMD_CMD, "epmd -port "++?ALT_EPMD_PORT). %%----------------------------------------------------------------- %% The distribution is mainly tested in the big old test_suite. @@ -82,6 +86,7 @@ all() -> tick, tick_change, nodenames, hostnames, illegal_nodenames, connect_node, dyn_node_name, + epmd_reconnect, hidden_node, setopts, table_waste, net_setuptime, inet_dist_options_options, {group, monitor_nodes}, @@ -117,9 +122,15 @@ init_per_testcase(TC, Config) when TC == hostnames; file:make_dir("hostnames_nodedir"), file:write_file("hostnames_nodedir/ignore_core_files",""), Config; +init_per_testcase(epmd_reconnect, Config) -> + [] = os:cmd(?ALT_EPMD_CMD++" -relaxed_command_check -daemon"), + Config; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. +end_per_testcase(epmd_reconnect, _Config) -> + os:cmd(?ALT_EPMD_CMD++" -kill"), + ok; end_per_testcase(_Func, _Config) -> ok. @@ -427,6 +438,83 @@ tick_cli_test1(Node) -> end end. +epmd_reconnect(Config) when is_list(Config) -> + NodeNames = [N1,N2,N3] = get_nodenames(3, ?FUNCTION_NAME), + Nodes = [atom_to_list(full_node_name(NN)) || NN <- NodeNames], + + DCfg = "-epmd_port "++?ALT_EPMD_PORT, + + {_N1F,Port1} = start_node_unconnected(DCfg, N1, ?MODULE, run_remote_test, + ["epmd_reconnect_do", atom_to_list(node()), "1" | Nodes]), + {_N2F,Port2} = start_node_unconnected(DCfg, N2, ?MODULE, run_remote_test, + ["epmd_reconnect_do", atom_to_list(node()), "2" | Nodes]), + {_N3F,Port3} = start_node_unconnected(DCfg, N3, ?MODULE, run_remote_test, + ["epmd_reconnect_do", atom_to_list(node()), "3" | Nodes]), + Ports = [Port1, Port2, Port3], + + ok = reap_ports(Ports), + + ok. + +reap_ports([]) -> + ok; +reap_ports(Ports) -> + case (receive M -> M end) of + {Port, Message} -> + case lists:member(Port, Ports) andalso Message of + {data,String} -> + io:format("~p: ~s\n", [Port, String]), + reap_ports(Ports); + {exit_status,0} -> + reap_ports(Ports -- [Port]) + end + end. + +epmd_reconnect_do(_Node, ["1", Node1, Node2, Node3]) -> + Names = [Name || Name <- [hd(string:tokens(Node, "@")) || Node <- [Node1, Node2, Node3]]], + %% wait until all nodes are registered + ok = wait_for_names(Names), + "Killed" ++_ = os:cmd(?ALT_EPMD_CMD++" -kill"), + open_port({spawn, ?ALT_EPMD_CMD}, []), + %% check that all nodes reregister with epmd + ok = wait_for_names(Names), + lists:foreach(fun(Node) -> + ANode = list_to_atom(Node), + pong = net_adm:ping(ANode), + {epmd_reconnect_do, ANode} ! {stop, Node1, Node} + end, [Node2, Node3]), + ok; +epmd_reconnect_do(_Node, ["2", Node1, Node2, _Node3]) -> + register(epmd_reconnect_do, self()), + receive {stop, Node1, Node2} -> + ok + after 7000 -> + exit(timeout) + end; +epmd_reconnect_do(_Node, ["3", Node1, _Node2, Node3]) -> + register(epmd_reconnect_do, self()), + receive {stop, Node1, Node3} -> + ok + after 7000 -> + exit(timeout) + end. + +wait_for_names(Names) -> + %% wait for up to 3 seconds (the current retry timer in erl_epmd is 2s) + wait_for_names(lists:sort(Names), 30, 100). + +wait_for_names(Names, N, Wait) when N > 0 -> + try + {ok, Info} = erl_epmd:names(), + Names = lists:sort([Name || {Name, _Port} <- Info]), + ok + catch + error:{badmatch, _} -> + timer:sleep(Wait), + wait_for_names(Names, N-1, Wait) + end. + + dyn_node_name(Config) when is_list(Config) -> %%run_dist_configs(fun dyn_node_name/2, Config). dyn_node_name("", Config). diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 900196d26f..9e92919bf0 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -326,12 +326,15 @@ t_accept_timeout(Config) when is_list(Config) -> %% Test that gen_tcp:connect/4 (with timeout) works. t_connect_timeout(Config) when is_list(Config) -> + ?TC_TRY(t_connect_timeout, fun() -> do_connect_timeout(Config) end). + +do_connect_timeout(Config)-> %%BadAddr = {134,138,177,16}, %%TcpPort = 80, {ok, BadAddr} = unused_ip(), TcpPort = 45638, ok = ?P("Connecting to ~p, port ~p", [BadAddr, TcpPort]), - connect_timeout({gen_tcp,connect,[BadAddr,TcpPort,?INET_BACKEND_OPTS(Config),200]}, 0.2, 5.0). + connect_timeout({gen_tcp,connect, [BadAddr,TcpPort, ?INET_BACKEND_OPTS(Config),200]}, 0.2, 5.0). %% Test that gen_tcp:connect/3 handles non-existings hosts, and other %% invalid things. @@ -376,17 +379,29 @@ t_recv_eof(Config) when is_list(Config) -> %% Test using message delimiter $X. t_recv_delim(Config) when is_list(Config) -> + ?TC_TRY(t_recv_delim, fun() -> do_recv_delim(Config) end). + +do_recv_delim(Config) -> + ?P("init"), {ok, L} = gen_tcp:listen(0, ?INET_BACKEND_OPTS(Config)), {ok, Port} = inet:port(L), Opts = ?INET_BACKEND_OPTS(Config) ++ [{active,false}, {packet,line}, {line_delimiter,$X}], {ok, Client} = gen_tcp:connect(localhost, Port, Opts), {ok, A} = gen_tcp:accept(L), + ?P("send the data"), ok = gen_tcp:send(A, "abcXefgX"), - {ok, "abcX"} = gen_tcp:recv(Client, 0, 200), - {ok, "efgX"} = gen_tcp:recv(Client, 0, 200), + %% Why do we need a timeout? + %% Sure, normally there would be no delay, + %% but this testcase has nothing to do with timeouts? + ?P("read the first chunk"), + {ok, "abcX"} = gen_tcp:recv(Client, 0), %, 200), + ?P("read the first chunk"), + {ok, "efgX"} = gen_tcp:recv(Client, 0), %, 200), + ?P("cleanup"), ok = gen_tcp:close(Client), ok = gen_tcp:close(A), + ?P("done"), ok. %%% gen_tcp:shutdown/2 @@ -414,36 +429,60 @@ t_shutdown_both(Config) when is_list(Config) -> ok. t_shutdown_error(Config) when is_list(Config) -> + ?TC_TRY(t_shutdown_error, fun() -> do_shutdown_error(Config) end). + +do_shutdown_error(Config) -> + ?P("create listen socket"), {ok, L} = gen_tcp:listen(0, ?INET_BACKEND_OPTS(Config)), + ?P("shutdown socket (with How = read_write)"), {error, enotconn} = gen_tcp:shutdown(L, read_write), + ?P("close socket"), ok = gen_tcp:close(L), + ?P("shutdown socket again (with How = read_write)"), {error, closed} = gen_tcp:shutdown(L, read_write), + ?P("done"), ok. t_shutdown_async(Config) when is_list(Config) -> + ?TC_TRY(t_shutdown_async, fun() -> do_shutdown_async(Config) end). + +do_shutdown_async(Config) -> {OS, _} = os:type(), + ?P("create listen socket"), {ok, L} = gen_tcp:listen(0, ?INET_BACKEND_OPTS(Config) ++ [{sndbuf, 4096}]), {ok, Port} = inet:port(L), + ?P("connect"), {ok, Client} = gen_tcp:connect(localhost, Port, ?INET_BACKEND_OPTS(Config) ++ [{recbuf, 4096}, {active, false}]), + ?P("accept connection"), {ok, S} = gen_tcp:accept(L), + ?P("create payload"), PayloadSize = 1024 * 1024, Payload = lists:duplicate(PayloadSize, $.), + ?P("send payload"), ok = gen_tcp:send(S, Payload), + ?P("verify queue size"), case erlang:port_info(S, queue_size) of {queue_size, N} when N > 0 -> ok; {queue_size, 0} when OS =:= win32 -> ok; {queue_size, 0} = T -> ct:fail({unexpected, T}) end, + ?P("shutdown(write) accepted socket"), ok = gen_tcp:shutdown(S, write), + ?P("recv from connected socket"), {ok, Buf} = gen_tcp:recv(Client, PayloadSize), + ?P("recv(0) from connected socket (expect closed)"), {error, closed} = gen_tcp:recv(Client, 0), + ?P("verify recv data"), case length(Buf) of - PayloadSize -> ok; - Sz -> ct:fail({payload_size, + PayloadSize -> ?P("done"), ok; + Sz -> ?P("ERROR: " + "~n extected: ~p" + "~n received: ~p", [PayloadSize, Sz]), + ct:fail({payload_size, {expected, PayloadSize}, {received, Sz}}) end. @@ -505,26 +544,32 @@ do_t_fdconnect(Config) -> Path = proplists:get_value(data_dir, Config), Lib = "gen_tcp_api_SUITE", ?P("try load util nif lib"), - case erlang:load_nif(filename:join(Path,Lib), []) of + case erlang:load_nif(filename:join(Path, Lib), []) of ok -> ok; + {error, {reload, ReasonStr}} -> + ?P("already loaded: " + "~n ~s", [ReasonStr]), + ok; {error, Reason} -> ?P("UNEXPECTED - failed loading util nif lib: " "~n ~p", [Reason]), ?SKIPT("failed loading util nif lib") end, ?P("try create listen socket"), - L = case gen_tcp:listen(0, ?INET_BACKEND_OPTS(Config) ++ [{active, false}]) of + L = case gen_tcp:listen(0, + ?INET_BACKEND_OPTS(Config) ++ [{active, false}]) of {ok, LSock} -> LSock; {error, eaddrnotavail = LReason} -> ?SKIPT(listen_failed_str(LReason)) end, {ok, Port} = inet:port(L), - ?P("try create file descriptor (fd)"), + ?P("try create file descriptor"), FD = gen_tcp_api_SUITE:getsockfd(), - ?P("try connect to using file descriptor (~w)", [FD]), - Client = case gen_tcp:connect(localhost, Port, ?INET_BACKEND_OPTS(Config) ++ + ?P("try connect using file descriptor (~w)", [FD]), + Client = case gen_tcp:connect(localhost, Port, + ?INET_BACKEND_OPTS(Config) ++ [{fd, FD}, {port, 20002}, {active, false}]) of @@ -698,46 +743,91 @@ t_local_basic(Config) -> t_local_unbound(Config) -> + ?TC_TRY(t_local_unbound, fun() -> do_local_unbound(Config) end). + +do_local_unbound(Config) -> + ?P("create local (server) filename"), SFile = local_filename(server), SAddr = {local,bin_filename(SFile)}, _ = file:delete(SFile), %% InetBackendOpts = ?INET_BACKEND_OPTS(Config), + ?P("create listen socket with ifaddr ~p", [SAddr]), L = ok(gen_tcp:listen(0, InetBackendOpts ++ [{ifaddr,SAddr},{active,false}])), + ?P("listen socket created: ~p" + "~n => try connect", [L]), C = ok(gen_tcp:connect(SAddr, 0, InetBackendOpts ++ [{active,false}])), + ?P("connected: ~p" + "~n => try accept", [C]), S = ok(gen_tcp:accept(L)), + ?P("accepted: ~p" + "~n => sockname", [S]), SAddr = ok(inet:sockname(L)), - {error,enotconn} = inet:peername(L), + ?P("sockname: ~p" + "~n => peername (expect enotconn)", [SAddr]), + {error, enotconn} = inet:peername(L), + ?P("try local handshake"), local_handshake(S, SAddr, C, {local,<<>>}), + ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("close accepted socket"), ok = gen_tcp:close(S), + ?P("close connected socket"), ok = gen_tcp:close(C), + ?P("delete (local) file"), ok = file:delete(SFile), + ?P("done"), ok. t_local_fdopen(Config) -> + ?TC_TRY(t_local_fdopen, fun() -> do_local_fdopen(Config) end). + +do_local_fdopen(Config) -> + ?P("create local (server) filename"), SFile = local_filename(server), SAddr = {local,bin_filename(SFile)}, _ = file:delete(SFile), %% InetBackendOpts = ?INET_BACKEND_OPTS(Config), + ?P("create listen socket with ifaddr ~p", [SAddr]), L = ok(gen_tcp:listen(0, InetBackendOpts ++ [{ifaddr,SAddr},{active,false}])), + ?P("listen socket created: ~p" + "~n => try connect", [L]), C0 = ok(gen_tcp:connect(SAddr, 0, InetBackendOpts ++ [{active,false}])), + ?P("connected: ~p" + "~n => get fd", [C0]), Fd = ok(prim_inet:getfd(C0)), + ?P("FD: ~p" + "~n => ignore fd", [Fd]), ok = prim_inet:ignorefd(C0, true), + ?P("ignored fd:" + "~n => try fdopen (local)"), C = ok(gen_tcp:fdopen(Fd, [local])), + ?P("fd open: ~p" + "~n => try accept", [C]), S = ok(gen_tcp:accept(L)), + ?P("accepted: ~p" + "~n => sockname", [S]), SAddr = ok(inet:sockname(L)), + ?P("sockname: ~p" + "~n => peername (expect enotconn)", [SAddr]), {error,enotconn} = inet:peername(L), + ?P("try local handshake"), local_handshake(S, SAddr, C, {local,<<>>}), + ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("close accepted socket"), ok = gen_tcp:close(S), + ?P("close connected socket (final)"), ok = gen_tcp:close(C), + ?P("close connected socket (pre)"), ok = gen_tcp:close(C0), + ?P("delete (local) file"), ok = file:delete(SFile), + ?P("done"), ok. t_local_fdopen_listen(Config) -> @@ -829,27 +919,48 @@ t_local_fdopen_connect_unbound(Config) -> ok. t_local_abstract(Config) -> + ?TC_TRY(t_local_abstract, fun() -> do_local_abstract(Config) end). + +do_local_abstract(Config) -> + ?P("only run on linux"), case os:type() of - {unix,linux} -> + {unix, linux} -> AbstAddr = {local,<<>>}, InetBackendOpts = ?INET_BACKEND_OPTS(Config), + ?P("create listen socket"), L = ok(gen_tcp:listen( 0, InetBackendOpts ++ [{ifaddr,AbstAddr},{active,false}])), - {local,_} = SAddr = ok(inet:sockname(L)), + ?P("listen socket created: ~p" + "~n => sockname", [L]), + {local, _} = SAddr = ok(inet:sockname(L)), + ?P("(listen socket) sockname verified" + "~n => try connect"), C = ok(gen_tcp:connect( SAddr, 0, InetBackendOpts ++ [{ifaddr,AbstAddr},{active,false}])), + ?P("connected: ~p" + "~n => sockname", [C]), {local,_} = CAddr = ok(inet:sockname(C)), + ?P("(connected socket) sockname verified" + "~n => try accept"), S = ok(gen_tcp:accept(L)), + ?P("accepted: ~p" + "~n => peername (expect enotconn)", [S]), {error,enotconn} = inet:peername(L), + ?P("try local handshake"), local_handshake(S, SAddr, C, CAddr), + ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("close accepted socket"), ok = gen_tcp:close(S), + ?P("close connected socket"), ok = gen_tcp:close(C), + ?P("done"), ok; _ -> + ?P("skip (unless linux)"), {skip,"AF_LOCAL Abstract Addresses only supported on Linux"} end. @@ -868,19 +979,36 @@ local_handshake(S, SAddr, C, CAddr) -> ok. t_accept_inet6_tclass(Config) when is_list(Config) -> + ?TC_TRY(t_accept_inet6_tclass, fun() -> do_accept_inet6_tclass(Config) end). + +do_accept_inet6_tclass(Config) -> TClassOpt = {tclass,8#56 bsl 2}, % Expedited forwarding Loopback = {0,0,0,0,0,0,0,1}, + ?P("create listen socket with tclass: ~p", [TClassOpt]), case gen_tcp:listen(0, ?INET_BACKEND_OPTS(Config) ++ [inet6, {ip, Loopback}, TClassOpt]) of - {ok,L} -> + {ok, L} -> + ?P("listen socket created: " + "~n ~p", [L]), LPort = ok(inet:port(L)), + ?P("try to connect to port ~p", [LPort]), Sa = ok(gen_tcp:connect(Loopback, LPort, ?INET_BACKEND_OPTS(Config))), + ?P("connected: ~p" + "~n => accept connection", [Sa]), Sb = ok(gen_tcp:accept(L)), + ?P("accepted: ~p" + "~n => getopts (tclass)", [Sb]), [TClassOpt] = ok(inet:getopts(Sb, [tclass])), + ?P("tclass verified => close accepted socket"), ok = gen_tcp:close(Sb), + ?P("close connected socket"), ok = gen_tcp:close(Sa), + ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("done"), ok; - {error,_} -> + {error, _Reason} -> + ?P("ERROR: Failed create listen socket" + "~n ~p", [_Reason]), {skip,"IPv6 TCLASS not supported"} end. @@ -940,6 +1068,8 @@ connect_timeout({M,F,A}, Lower, Upper) -> {skip, "Not tested -- got error " ++ atom_to_list(E)}; {error, enetunreach = E} -> {skip, "Not tested -- got error " ++ atom_to_list(E)}; + {error, ehostunreach = E} -> + {skip, "Not tested -- got error " ++ atom_to_list(E)}; {ok, Socket} -> % What the... Pinfo = erlang:port_info(Socket), Db = inet_db:lookup_socket(Socket), @@ -971,18 +1101,20 @@ unused_ip() -> %% This is not supported on all platforms (yet), so... try net:getifaddrs() of {ok, IfAddrs} -> - io:format("we = ~p," - "unused_ip = ~p" - " ~p" - "~n", [Hent, IP, IfAddrs]); + ?P("~n we = ~p" + "~n unused_ip = ~p" + "~n ~p", [Hent, IP, IfAddrs]); {error, _} -> - io:format("we = ~p, unused_ip = ~p~n", [Hent, IP]) + ?P("~n we: ~p" + "~n unused_ip: ~p", [Hent, IP]) catch _:_:_ -> - io:format("we = ~p, unused_ip = ~p~n", [Hent, IP]) + ?P("~n we: ~p" + "~n unused_ip: ~p", [Hent, IP]) end; true -> - io:format("we = ~p, unused_ip = ~p~n", [Hent, IP]) + ?P("~n we: ~p" + "~n unused_ip: ~p", [Hent, IP]) end, IP. diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index e0713002ed..2c2725ad30 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2020. All Rights Reserved. +%% Copyright Ericsson AB 1998-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1573,42 +1573,63 @@ econnreset_after_async_send_active_once(Config) when is_list(Config) -> do_econnreset_after_async_send_active_once(Config) -> {OS, _} = os:type(), + ?P("create listen socket with active = false"), {ok, L} = ?LISTEN(Config, 0, [{active, false}, {recbuf, 4096}]), {ok, Port} = inet:port(L), + ?P("create connect socket (~w)", [Port]), Client = case ?CONNECT(Config, localhost, Port, - [{active, false}, - {sndbuf, 4096}, - {show_econnreset, true}]) of + [{active, false}, + {sndbuf, 4096}, + {show_econnreset, true}]) of {ok, CSock} -> CSock; {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, + ?P("create accept socket"), {ok,S} = gen_tcp:accept(L), + ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("create payload"), Payload = lists:duplicate(1024 * 1024, $.), + ?P("[connect] send payload"), ok = gen_tcp:send(Client, Payload), + ?P("[connect] verify socket queue size"), case erlang:port_info(Client, queue_size) of {queue_size, N} when N > 0 -> ok; {queue_size, 0} when OS =:= win32 -> ok; {queue_size, 0} = T -> ct:fail(T) end, + ?P("[accept] send something"), ok = gen_tcp:send(S, "Whatever"), + ?P("sleep some"), ok = ct:sleep(20), + ?P("[accept] set socket option linger: {true, 0}"), ok = inet:setopts(S, [{linger, {true, 0}}]), + ?P("[accept] close socket"), ok = gen_tcp:close(S), + ?P("sleep some"), ok = ct:sleep(20), + ?P("receive 'unexpected message'"), ok = receive Msg -> {unexpected_msg, Msg} after 0 -> ok end, + ?P("[connect] set socket option active: once"), ok = inet:setopts(Client, [{active, once}]), + ?P("[connect] expect econreset"), receive {tcp_error, Client, econnreset} -> + ?P("[connect] received econreset -> expect socket close message"), receive {tcp_closed, Client} -> + ?P("[connect] received socket close message - done"), ok; Other -> + ?P("[connect] received unexpected message: " + "~n ~p", [Other]), ct:fail({unexpected1, Other}) end; Other -> + ?P("[connect] received unexpected message: " + "~n ~p", [Other]), ct:fail({unexpected2, Other}) end. @@ -1735,7 +1756,7 @@ do_linger_zero(Config) -> {ok, Port} = inet:port(L), ?P("connect (create client socket)"), Client = case ?CONNECT(Config, localhost, Port, - [{active, false}, {sndbuf, 4096}]) of + [{active, false}, {sndbuf, 4096}]) of {ok, CSock} -> CSock; {error, eaddrnotavail = Reason} -> @@ -1745,13 +1766,16 @@ do_linger_zero(Config) -> {ok, S} = gen_tcp:accept(L), ?P("close listen socket"), ok = gen_tcp:close(L), + ?P("create payload"), PayloadSize = 1024 * 1024, Payload = lists:duplicate(PayloadSize, $.), + ?P("ensure empty queue"), lz_ensure_non_empty_queue(Client, Payload, OS), ?P("linger: {true, 0}"), ok = inet:setopts(Client, [{linger, {true, 0}}]), ?P("close client socket"), ok = gen_tcp:close(Client), + ?P("sleep some"), ok = ct:sleep(1), ?P("verify client socket (port) not connected"), @@ -1766,7 +1790,7 @@ do_linger_zero(Config) -> ok. %% THIS DOES NOT WORK FOR 'SOCKET' -lz_ensure_non_empty_queue(Sock, Payload, OS) -> +lz_ensure_non_empty_queue(Sock, Payload, OS) when is_port(Sock) -> lz_ensure_non_empty_queue(Sock, Payload, OS, 1). -define(LZ_MAX_SENDS, 3). @@ -2486,21 +2510,34 @@ do_partial_recv_and_close_4(Config) -> test_prio_put_get(Config) -> Tos = 3 bsl 5, + ?P("test_prio_put_get -> create listen socket"), {ok,L1} = ?LISTEN(Config, 0, [{active,false}]), + ?P("test_prio_put_get -> set opts prio (= 3)"), ok = inet:setopts(L1,[{priority,3}]), + ?P("test_prio_put_get -> set opts tos (= ~p)", [Tos]), ok = inet:setopts(L1,[{tos,Tos}]), + ?P("test_prio_put_get -> verify opts prio and tos"), {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + ?P("test_prio_put_get -> set opts prio (= 3)"), ok = inet:setopts(L1,[{priority,3}]), % Dont destroy each other + ?P("test_prio_put_get -> verify opts prio and tos"), {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + ?P("test_prio_put_get -> set opts reuseaddr (= true)"), ok = inet:setopts(L1,[{reuseaddr,true}]), % Dont let others destroy + ?P("test_prio_put_get -> verify opts prio and tos"), {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + ?P("test_prio_put_get -> close listen socket"), gen_tcp:close(L1), + ?P("test_prio_put_get -> done"), ok. test_prio_accept(Config) -> - {ok,Sock}=?LISTEN(Config, 0,[binary,{packet,0},{active,false}, - {reuseaddr,true},{priority,4}]), - {ok,Port} = inet:port(Sock), + ?P("test_prio_accept -> create listen socket"), + {ok, Sock} = ?LISTEN(Config, 0, [binary,{packet,0},{active,false}, + {reuseaddr,true},{priority,4}]), + ?P("test_prio_accept -> get port number of listen socket"), + {ok, Port} = inet:port(Sock), + ?P("test_prio_accept -> connect to port ~p", [Port]), Sock2 = case ?CONNECT(Config, "localhost",Port,[binary,{packet,0}, {active,false}, {reuseaddr,true}, @@ -2510,22 +2547,33 @@ test_prio_accept(Config) -> {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, - {ok,Sock3}=gen_tcp:accept(Sock), - {ok,[{priority,4}]} = inet:getopts(Sock,[priority]), - {ok,[{priority,4}]} = inet:getopts(Sock2,[priority]), - {ok,[{priority,4}]} = inet:getopts(Sock3,[priority]), + ?P("test_prio_accept -> connected => accept connection"), + {ok, Sock3} = gen_tcp:accept(Sock), + ?P("test_prio_accept -> accepted => getopts prio for listen socket"), + {ok, [{priority,4}]} = inet:getopts(Sock, [priority]), + ?P("test_prio_accept -> getopts prio for connected socket"), + {ok, [{priority,4}]} = inet:getopts(Sock2, [priority]), + ?P("test_prio_accept -> getopts prio for accepted socket"), + {ok, [{priority,4}]} = inet:getopts(Sock3, [priority]), + ?P("test_prio_accept -> close listen socket"), gen_tcp:close(Sock), + ?P("test_prio_accept -> close connected socket"), gen_tcp:close(Sock2), + ?P("test_prio_accept -> close accepted socket"), gen_tcp:close(Sock3), + ?P("test_prio_accept -> done"), ok. test_prio_accept2(Config) -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, - {ok,Sock}=?LISTEN(Config, 0,[binary,{packet,0},{active,false}, - {reuseaddr,true},{priority,4}, - {tos,Tos1}]), - {ok,Port} = inet:port(Sock), + ?P("test_prio_accept2 -> create listen socket"), + {ok, Sock} = ?LISTEN(Config, 0,[binary,{packet,0},{active,false}, + {reuseaddr,true},{priority,4}, + {tos,Tos1}]), + ?P("test_prio_accept2 -> get port number of listen socket"), + {ok, Port} = inet:port(Sock), + ?P("test_prio_accept2 -> connect to port ~p", [Port]), Sock2 = case ?CONNECT(Config, "localhost",Port,[binary,{packet,0}, {active,false}, {reuseaddr,true}, @@ -2536,22 +2584,33 @@ test_prio_accept2(Config) -> {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, - {ok,Sock3}=gen_tcp:accept(Sock), + ?P("test_prio_accept2 -> connected => accept connection"), + {ok, Sock3} = gen_tcp:accept(Sock), + ?P("test_prio_accept2 -> accepted => getopts prio and tos for listen socket"), {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), + ?P("test_prio_accept2 -> getopts prio and tos for connected socket"), {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), + ?P("test_prio_accept2 -> getopts prio and tos for accepted socket"), {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), + ?P("test_prio_accept2 -> close listen socket"), gen_tcp:close(Sock), + ?P("test_prio_accept2 -> close connected socket"), gen_tcp:close(Sock2), + ?P("test_prio_accept2 -> close accepted socket"), gen_tcp:close(Sock3), + ?P("test_prio_accept2 -> done"), ok. test_prio_accept3(Config) -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, - {ok,Sock}=?LISTEN(Config, 0,[binary,{packet,0},{active,false}, - {reuseaddr,true}, - {tos,Tos1}]), + ?P("test_prio_accept3 -> create listen socket"), + {ok, Sock} = ?LISTEN(Config, 0,[binary,{packet,0},{active,false}, + {reuseaddr,true}, + {tos,Tos1}]), + ?P("test_prio_accept3 -> get port number of listen socket"), {ok,Port} = inet:port(Sock), + ?P("test_prio_accept3 -> connect to port ~p", [Port]), Sock2 = case ?CONNECT(Config, "localhost",Port,[binary,{packet,0}, {active,false}, {reuseaddr,true}, @@ -2561,19 +2620,29 @@ test_prio_accept3(Config) -> {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, - {ok,Sock3}=gen_tcp:accept(Sock), - {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), - {ok,[{priority,0},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), - {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), + ?P("test_prio_accept3 -> connected => accept connection"), + {ok, Sock3} = gen_tcp:accept(Sock), + ?P("test_prio_accept3 -> " + "accepted => getopts prio and tos for listen socket"), + {ok, [{priority,0},{tos,Tos1}]} = inet:getopts(Sock, [priority,tos]), + ?P("test_prio_accept3 -> getopts prio and tos for connected socket"), + {ok, [{priority,0},{tos,Tos2}]} = inet:getopts(Sock2, [priority,tos]), + ?P("test_prio_accept3 -> getopts prio and tos for accepted socket"), + {ok, [{priority,0},{tos,Tos1}]} = inet:getopts(Sock3, [priority,tos]), + ?P("test_prio_accept3 -> close listen socket"), gen_tcp:close(Sock), + ?P("test_prio_accept3 -> close connected socket"), gen_tcp:close(Sock2), + ?P("test_prio_accept3 -> close accepted socket"), gen_tcp:close(Sock3), + ?P("test_prio_accept3 -> done"), ok. test_prio_accept_async(Config) -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, Ref = make_ref(), + ?P("test_prio_accept_async -> create prio server"), spawn(?MODULE, priority_server, [Config, {self(),Ref}]), Port = receive {Ref,P} -> P @@ -2582,6 +2651,7 @@ test_prio_accept_async(Config) -> receive after 3000 -> ok end, + ?P("test_prio_accept_async -> connect to port ~p", [Port]), Sock2 = case ?CONNECT(Config, "localhost",Port,[binary,{packet,0}, {active,false}, {reuseaddr,true}, @@ -2592,6 +2662,8 @@ test_prio_accept_async(Config) -> {error, eaddrnotavail = Reason} -> ?SKIPT(connect_failed_str(Reason)) end, + ?P("test_prio_accept_async -> " + "connected => await prio and tos for listen socket"), receive {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> ok; @@ -2599,6 +2671,7 @@ test_prio_accept_async(Config) -> ct:fail({missmatch,Error}) after 5000 -> ct:fail({error,"helper process timeout"}) end, + ?P("test_prio_accept_async -> await prio and tos for accepted socket"), receive {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> ok; @@ -2607,8 +2680,11 @@ test_prio_accept_async(Config) -> after 5000 -> ct:fail({error,"helper process timeout"}) end, - {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), + ?P("test_prio_accept_async -> getopts prio and tos for connected socket"), + {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2, [priority,tos]), + ?P("test_prio_accept_async -> close connected socket"), catch gen_tcp:close(Sock2), + ?P("test_prio_accept_async -> done"), ok. priority_server(Config, {Parent,Ref}) -> @@ -2624,32 +2700,40 @@ priority_server(Config, {Parent,Ref}) -> ok. test_prio_fail(Config) -> + ?P("test_prio_fail -> create listen socket"), {ok,L} = ?LISTEN(Config, 0, [{active,false}]), + ?P("test_prio_fail -> try set (and fail) opts prio (= 1000)"), {error,_} = inet:setopts(L,[{priority,1000}]), + ?P("test_prio_fail -> close listen socket"), gen_tcp:close(L), + ?P("test_prio_fail -> done"), ok. test_prio_udp() -> Tos = 3 bsl 5, + ?P("test_prio_udp -> create UDP socket (open)"), {ok,S} = gen_udp:open(0,[{active,false},binary,{tos, Tos}, {priority,3}]), + ?P("test_prio_udp -> getopts prio and tos"), {ok,[{priority,3},{tos,Tos}]} = inet:getopts(S,[priority,tos]), + ?P("test_prio_fail -> close socket"), gen_udp:close(S), + ?P("test_prio_fail -> done"), ok. %% Tests the so_priority and ip_tos options on sockets when applicable. so_priority(Config) when is_list(Config) -> - try do_so_priority(Config) - catch - throw:{skip, _} = SKIP -> - SKIP - end. + ?TC_TRY(so_priority, fun() -> do_so_priority(Config) end). do_so_priority(Config) -> + ?P("create listen socket"), {ok,L} = ?LISTEN(Config, 0, [{active,false}]), + ?P("set opts on listen socket: prio to 1"), ok = inet:setopts(L,[{priority,1}]), + ?P("verify prio"), case inet:getopts(L,[priority]) of {ok,[{priority,1}]} -> + ?P("close listen socket"), gen_tcp:close(L), test_prio_put_get(Config), test_prio_accept(Config), @@ -2658,12 +2742,15 @@ do_so_priority(Config) -> test_prio_accept_async(Config), test_prio_fail(Config), test_prio_udp(), + ?P("done"), ok; - _ -> + _X -> case os:type() of {unix,linux} -> case os:version() of {X,Y,_} when (X > 2) or ((X =:= 2) and (Y >= 4)) -> + ?P("so prio should work on this version: " + "~n ~p", [_X]), ct:fail({error, "so_priority should work on this " "OS, but does not"}); @@ -2742,8 +2829,9 @@ recvtclass(Config) -> %% platforms - change {unix,_} to false? %% pktoptions is not supported for IPv4 -recvtos_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); -recvtos_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {19,4,0}); +recvtos_ok({unix,netbsd}, _OSVer) -> false; +recvtos_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); +recvtos_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); %% Using the option returns einval, so it is not implemented. recvtos_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); recvtos_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); @@ -2754,8 +2842,9 @@ recvtos_ok({unix,_}, _) -> true; recvtos_ok(_, _) -> false. %% pktoptions is not supported for IPv4 -recvttl_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); -recvttl_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {19,4,0}); +recvttl_ok({unix,netbsd}, _OSVer) -> false; +recvttl_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); +recvttl_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); %% Using the option returns einval, so it is not implemented. recvttl_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); recvttl_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); @@ -2766,8 +2855,9 @@ recvttl_ok({unix,_}, _) -> true; recvttl_ok(_, _) -> false. %% pktoptions is not supported for IPv6 -recvtclass_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); -recvtclass_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {19,4,0}); +recvtclass_ok({unix,netbsd}, _OSVer) -> false; +recvtclass_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); +recvtclass_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); recvtclass_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); %% Using the option returns einval, so it is not implemented. recvtclass_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); @@ -3004,6 +3094,8 @@ collect_accepts(N,Tmo) -> ?SKIPT(accept_failed_str(Reason)); {accepted,P,Msg} -> + ?P("received accepted from ~p: " + "~n ~p", [P, Msg]), NextN = if N =:= infinity -> N; true -> N - 1 end, [{P,Msg}] ++ collect_accepts(NextN, Tmo - (millis()-A)) @@ -3029,6 +3121,8 @@ collect_connects(Tmo) -> ?SKIPT(connect_failed_str(Reason)); {connected,P,Msg} -> + ?P("received connected from ~p: " + "~n ~p", [P, Msg]), [{P,Msg}] ++ collect_connects(Tmo-(millis() - A)) after Tmo -> @@ -3320,6 +3414,7 @@ accept_timeouts_mixed(Config) when is_list(Config) -> end. do_accept_timeouts_mixed(Config) -> + ?P("create listen socket"), LS = case ?LISTEN(Config, 0,[]) of {ok, LSocket} -> LSocket; @@ -3328,30 +3423,46 @@ do_accept_timeouts_mixed(Config) -> end, Parent = self(), {ok,PortNo}=inet:port(LS), + ?P("create acceptor process 1 (with timeout 1000)"), P1 = spawn(mktmofun(1000,Parent,LS)), + ?P("await ~p accepting", [P1]), wait_until_accepting(P1,500), + ?P("create acceptor process 2 (with timeout 2000)"), P2 = spawn(mktmofun(2000,Parent,LS)), + ?P("await ~p accepting", [P2]), wait_until_accepting(P2,500), + ?P("create acceptor process 3 (with timeout 3000)"), P3 = spawn(mktmofun(3000,Parent,LS)), + ?P("await ~p accepting", [P3]), wait_until_accepting(P3,500), + ?P("create acceptor process 4 (with timeout 4000)"), P4 = spawn(mktmofun(4000,Parent,LS)), + ?P("await ~p accepting", [P4]), wait_until_accepting(P4,500), + ?P("expect accept from 1 (~p) with timeout", [P1]), ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}}],infinity,1500), + ?P("connect"), case ?CONNECT(Config, "localhost", PortNo, []) of {ok, _} -> ok; {error, eaddrnotavail = Reason1} -> ?SKIPT(connect_failed_str(Reason1)) end, + ?P("expect accept from 2 (~p) with success", [P2]), ok = ?EXPECT_ACCEPTS([{P2,{ok,Port0}}] when is_port(Port0),infinity,100), + ?P("expect accept from 3 (~p) with timeout", [P3]), ok = ?EXPECT_ACCEPTS([{P3,{error,timeout}}],infinity,2000), + ?P("connect"), case ?CONNECT(Config, "localhost", PortNo, []) of {error, eaddrnotavail = Reason2} -> ?SKIPT(connect_failed_str(Reason2)); _ -> ok end, - ok = ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),infinity,100). + ?P("expect accept from 4 (~p) with success", [P4]), + ok = ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),infinity,100), + ?P("done"), + ok. %% Check that single acceptor behaves as expected when killed. killing_acceptor(Config) when is_list(Config) -> @@ -3362,25 +3473,35 @@ killing_acceptor(Config) when is_list(Config) -> end. do_killing_acceptor(Config) -> + ?P("create listen socket"), LS = case ?LISTEN(Config, 0,[]) of {ok, LSocket} -> LSocket; {error, eaddrnotavail = Reason} -> ?SKIPT(listen_failed_str(Reason)) end, + ?P("create acceptor process"), Pid = spawn( fun() -> erlang:display({accepted,self(),gen_tcp:accept(LS)}) end), + ?P("sleep some"), receive after 100 -> ok end, + ?P("get status for listen socket"), {ok,L1} = prim_inet:getstatus(LS), + ?P("verify listen socket accepting"), true = lists:member(accepting, L1), + ?P("kill acceptor"), exit(Pid,kill), + ?P("sleep some"), receive after 100 -> ok end, + ?P("get status for listen socket"), {ok,L2} = prim_inet:getstatus(LS), + ?P("verify listen socket *not* accepting"), false = lists:member(accepting, L2), + ?P("done"), ok. %% Check that multi acceptors behaves as expected when killed. @@ -3426,6 +3547,7 @@ killing_multi_acceptors2(Config) when is_list(Config) -> end. do_killing_multi_acceptors2(Config) -> + ?P("create listen socket"), LS = case ?LISTEN(Config, 0,[]) of {ok, LSocket} -> LSocket; @@ -3433,46 +3555,67 @@ do_killing_multi_acceptors2(Config) -> ?SKIPT(listen_failed_str(Reason)) end, Parent = self(), - {ok,PortNo}=inet:port(LS), + ?P("get port number for listen socket"), + {ok, PortNo} = inet:port(LS), F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, F2 = mktmofun(1000,Parent,LS), + ?P("create acceptor process 1"), Pid = spawn(F), + ?P("create acceptor process 2"), Pid2 = spawn(F), + ?P("wait some"), receive after 100 -> ok end, - {ok,L1} = prim_inet:getstatus(LS), + ?P("get status for listen socket"), + {ok, L1} = prim_inet:getstatus(LS), + ?P("verify listen socket *is* accepting"), true = lists:member(accepting, L1), + ?P("kill acceptor process 1"), exit(Pid,kill), + ?P("wait some"), receive after 100 -> ok end, - {ok,L2} = prim_inet:getstatus(LS), + ?P("get status for listen socket"), + {ok, L2} = prim_inet:getstatus(LS), + ?P("verify listen socket *is* accepting"), true = lists:member(accepting, L2), + ?P("kill acceptor process 1"), exit(Pid2,kill), + ?P("wait some"), receive after 100 -> ok end, - {ok,L3} = prim_inet:getstatus(LS), + ?P("get status for listen socket"), + {ok, L3} = prim_inet:getstatus(LS), + ?P("verify listen socket is *not* accepting"), false = lists:member(accepting, L3), + ?P("create acceptor process 3"), Pid3 = spawn(F2), + ?P("wait some"), receive after 100 -> ok end, + ?P("get status for listen socket"), {ok,L4} = prim_inet:getstatus(LS), + ?P("verify listen socket *is* accepting"), true = lists:member(accepting, L4), - ?CONNECT(Config, "localhost",PortNo,[]), + ?P("connect to port ~p", [PortNo]), + ?CONNECT(Config, "localhost", PortNo,[]), + ?P("accepts"), ok = ?EXPECT_ACCEPTS([{Pid3,{ok,Port}}] when is_port(Port),1,100), - {ok,L5} = prim_inet:getstatus(LS), + ?P("get status for listen socket"), + {ok, L5} = prim_inet:getstatus(LS), + ?P("verify listen socket *is* accepting"), false = lists:member(accepting, L5), + ?P("done"), ok. %% Checks that multi-accept works when more than one accept can be %% done at once (wb test of inet_driver). several_accepts_in_one_go(Config) when is_list(Config) -> - try do_several_accepts_in_one_go(Config) - catch - throw:{skip, _} = SKIP -> - SKIP - end. + ?TC_TRY(several_accepts_in_one_go, + fun() -> do_several_accepts_in_one_go(Config) end). do_several_accepts_in_one_go(Config) -> + ?P("create listen socket"), LS = case ?LISTEN(Config, 0,[]) of {ok, LSock} -> LSock; @@ -3480,15 +3623,25 @@ do_several_accepts_in_one_go(Config) -> ?SKIPT(listen_failed_str(Reason)) end, Parent = self(), - {ok,PortNo}=inet:port(LS), - F1 = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - F2 = fun() -> Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} end, + {ok, PortNo} = inet:port(LS), + F1 = fun() -> ?P("acceptor starting"), + Parent ! {accepted,self(),gen_tcp:accept(LS)} + end, + F2 = fun() -> ?P("connector starting"), + Parent ! {connected,self(),?CONNECT(Config, "localhost",PortNo,[])} + end, Ns = lists:seq(1,8), + ?P("start acceptors"), _ = [spawn(F1) || _ <- Ns], + ?P("await accept timeouts"), ok = ?EXPECT_ACCEPTS([],1,500), % wait for tmo + ?P("start connectors"), _ = [spawn(F2) || _ <- Ns], + ?P("await accepts"), ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],8,15000), + ?P("await connects"), ok = ?EXPECT_CONNECTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],1000), + ?P("done"), ok. flush(Msgs) -> @@ -4992,46 +5145,71 @@ port_failed_str(Reason) -> %% 30-second test for gen_tcp in {active, N} mode, ensuring it does not get stuck. %% Verifies that erl_check_io properly handles extra EPOLLIN signals. bidirectional_traffic(Config) when is_list(Config) -> + ?TC_TRY(bidirectional_traffic, + fun() -> do_bidirectional_traffic(Config) end). + +do_bidirectional_traffic(_Config) -> + ?P("begin"), Workers = erlang:system_info(schedulers_online) * 2, + ?P("Use ~w workers", [Workers]), Payload = crypto:strong_rand_bytes(32), + ?P("create listen socket"), {ok, LSock} = gen_tcp:listen(0, [binary, {packet, 0}, {active, false}, {reuseaddr, true}]), %% get all sockets to know failing ends {ok, Port} = inet:port(LSock), + ?P("listen socket port number ~w", [Port]), Control = self(), + ?P("create ~w receivers", [Workers]), Receivers = [spawn_link(fun () -> exchange(LSock, Port, Payload, Control) end) || _ <- lists:seq(1, Workers)], + ?P("await the result"), Result = receive {timeout, Socket, Total} -> + ?P("timeout msg for ~p: ~w", [Socket, Total]), {fail, {timeout, Socket, Total}}; {error, Socket, Reason} -> + ?P("error msg for ~p: ~p", [Socket, Reason]), {fail, {error, Socket, Reason}} after 30000 -> - %% if it does not fail in 30 seconds, it most likely works - ok + %% if it does not fail in 30 seconds, it most likely works + ?P("timeout => success?"), + ok end, + ?P("terminate receivers"), [begin unlink(Rec), exit(Rec, kill) end || Rec <- Receivers], + ?P("done"), Result. exchange(LSock, Port, Payload, Control) -> %% spin up client _ClntRcv = spawn( fun () -> - {ok, Client} = gen_tcp:connect("localhost", Port, [binary, {packet, 0}, {active, ?ACTIVE_N}]), - send_recv_loop(Client, Payload, Control) + ?P("connect"), + {ok, Client} = + gen_tcp:connect("localhost", + Port, + [binary, {packet, 0}, {active, ?ACTIVE_N}]), + ?P("connected: ~p", [Client]), + send_recv_loop(Client, Payload, Control) end), + ?P("accept"), {ok, Socket} = gen_tcp:accept(LSock), + ?P("accepted: ~p", [Socket]), %% sending process send_recv_loop(Socket, Payload, Control). send_recv_loop(Socket, Payload, Control) -> %% {active, N} must be set to active > 12 to trigger the issue %% {active, 30} seems to trigger it quite often & reliably + ?P("set (initial) active: ~p", [?ACTIVE_N]), inet:setopts(Socket, [{active, ?ACTIVE_N}]), + ?P("spawn sender"), _Snd = spawn_link( fun Sender() -> _ = gen_tcp:send(Socket, Payload), Sender() end), + ?P("begin recv"), recv(Socket, 0, Control). recv(Socket, Total, Control) -> @@ -5042,10 +5220,32 @@ recv(Socket, Total, Control) -> inet:setopts(Socket, [{active, ?ACTIVE_N}]), recv(Socket, Total, Control); {tcp_closed, Socket} -> + ?P("[recv] closed when total received: ~w", [Total]), exit(terminate); - Other-> + Other -> + ?P("[recv] received unexpected when total received: ~w" + "~n ~p" + "~n Socket: ~p" + "~n Port stat: ~p" + "~n Port status: ~p" + "~n Port Info: ~p", + [Total, Other, + Socket, + (catch inet:getstat(Socket)), + (catch prim_inet:getstatus(Socket)), + (catch erlang:port_info(Socket))]), Control ! {error, Socket, Other} after 2000 -> - %% no data received in 2 seconds, test failed - Control ! {timeout, Socket, Total} + %% no data received in 2 seconds, test failed + ?P("[recv] received nothing when total received: ~w" + "~n Socket: ~p" + "~n Port stat: ~p" + "~n Port status: ~p" + "~n Port Info: ~p", + [Total, + Socket, + (catch inet:getstat(Socket)), + (catch prim_inet:getstatus(Socket)), + (catch erlang:port_info(Socket))]), + Control ! {timeout, Socket, Total} end. diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index a71237e11f..a52f70933e 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -752,7 +752,9 @@ do_sendtclass() -> %% Using the option returns einval, so it is not implemented. recvtos_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {17,6,0}); %% Using the option returns einval, so it is not implemented. -recvtos_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); +recvtos_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); +%% Using the option returns einval, so it is not implemented. +recvtos_ok({unix,netbsd}, _OSVer) -> false; %% Using the option returns einval, so it is not implemented. recvtos_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); %% @@ -781,7 +783,8 @@ recvtclass_ok(_, _) -> false. %% Using the option returns einval, so it is not implemented. sendtos_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {19,0,0}); -sendtos_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); +sendtos_ok({unix,netbsd}, _OSVer) -> false; +sendtos_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); sendtos_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); sendtos_ok({unix,linux}, OSVer) -> not semver_lt(OSVer, {4,0,0}); sendtos_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); @@ -790,13 +793,13 @@ sendtos_ok({unix,_}, _) -> true; sendtos_ok(_, _) -> false. %% Using the option returns einval, so it is not implemented. -sendttl_ok({unix,darwin}, OSVer) -> not semver_lt(OSVer, {19,4,0}); +sendttl_ok({unix,darwin}, OSVer) -> false; % not semver_lt(OSVer, {19,6,0}); sendttl_ok({unix,linux}, OSVer) -> not semver_lt(OSVer, {4,0,0}); %% Using the option returns enoprotoopt, so it is not implemented. sendttl_ok({unix,freebsd}, OSVer) -> not semver_lt(OSVer, {12,2,0}); %% Option has no effect sendttl_ok({unix,sunos}, OSVer) -> not semver_lt(OSVer, {5,12,0}); -sendttl_ok({unix,openbsd}, OSVer) -> not semver_lt(OSVer, {6,8,0}); +sendttl_ok({unix,openbsd}, _OSVer) -> false; % not semver_lt(OSVer, {6,9,0}); %% sendttl_ok({unix,_}, _) -> true; sendttl_ok(_, _) -> false. diff --git a/lib/kernel/test/inet_res_SUITE.erl b/lib/kernel/test/inet_res_SUITE.erl index 91ff883466..54686c326a 100644 --- a/lib/kernel/test/inet_res_SUITE.erl +++ b/lib/kernel/test/inet_res_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2020. All Rights Reserved. +%% Copyright Ericsson AB 2009-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,7 +33,8 @@ ]). -export([basic/1, resolve/1, edns0/1, txt_record/1, files_monitor/1, last_ms_answer/1, intermediate_error/1, - servfail_retry_timeout_default/1, servfail_retry_timeout_1000/1 + servfail_retry_timeout_default/1, servfail_retry_timeout_1000/1, + label_compression_limit/1 ]). -export([ gethostbyaddr/0, gethostbyaddr/1, @@ -71,7 +72,9 @@ suite() -> all() -> [basic, resolve, edns0, txt_record, files_monitor, last_ms_answer, - intermediate_error, servfail_retry_timeout_default, servfail_retry_timeout_1000, + intermediate_error, + servfail_retry_timeout_default, servfail_retry_timeout_1000, + label_compression_limit, gethostbyaddr, gethostbyaddr_v6, gethostbyname, gethostbyname_v6, getaddr, getaddr_v6, ipv4_to_ipv6, host_and_addr]. @@ -974,6 +977,82 @@ servfail_retry_timeout_1000(Config) when is_list(Config) -> %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Test that label encoding compression limits at 14 bits pointer size + +label_compression_limit(Config) when is_list(Config) -> + FirstSz = 8, + Count = 512, + Sz = 20, + %% We create a DNS message with an answer list containing + %% 1+512+1 RR:s. The first label is 8 chars that with message + %% and RR overhead places the second label on offset 32. + %% All other labels are 20 chars that with RR overhead + %% places them on offsets of N * 32. + %% + %% The labels are: "ZZZZZZZZ", then; "AAAAAAAAAAAAAAAAAAAA", + %% "AAAAAAAAAAAAAAAAAAAB", incrementing, so no one is + %% equal and can not be compressed, until the last one + %% that refers to the second to last one, so it could be compressed. + %% + %% However, the second to last label lands on offset 512 * 32 = 16384 + %% which is out of reach for compression since compression uses + %% a 14 bit reference from the start of the message. + %% + %% The last label can only be compressed when we instead + %% generate a message with one less char in the first label, + %% placing the second to last label on offset 16383. + %% + %% So, MsgShort can use compression for the last RR + %% by referring to the second to last RR, but MsgLong can not. + %% + %% Disclaimer: + %% All offsets and overheads are deduced + %% through trial and observation + %% + [D | Domains] = gen_domains(Count, lists:duplicate(Sz, $A), []), + LastD = "Y." ++ D, + DomainsShort = + [lists:duplicate(FirstSz-1, $Z) | + lists:reverse(Domains, [D, LastD])], + DomainsLong = + [lists:duplicate(FirstSz, $Z) | + lists:reverse(Domains, [D, LastD])], + MsgShort = gen_msg(DomainsShort), + MsgLong = gen_msg(DomainsLong), + DataShort = inet_dns:encode(MsgShort), + DataShortSz = byte_size(DataShort), + ?P("DataShort[~w]:~n ~p~n", [DataShortSz, DataShort]), + DataLong = inet_dns:encode(MsgLong), + DataLongSz = byte_size(DataLong), + ?P("DataLong[~w]:~n ~p~n", [DataLongSz, DataLong]), + %% When we increase the first RR size by 1, the compressed + %% label that occupied a 2 bytes reference instead becomes + %% a label with 1 byte size and a final empty label size 1 + 0 = DataLongSz - (DataShortSz+1 - 2 + 1+Sz+1), + ok. + +gen_msg(Domains) -> + inet_dns:make_msg( + [{header, inet_dns:make_header()}, + {anlist, gen_rrs(Domains)}]). + +gen_rrs(Domains) -> + [inet_dns:make_rr([{class,in},{type,a},{domain,D}]) || + D <- Domains]. + +gen_domains(0, _Domain, Acc) -> + Acc; +gen_domains(N, Domain, Acc) -> + gen_domains( + N - 1, incr_domain(Domain), [lists:reverse(Domain) | Acc]). + +incr_domain([$Z | Domain]) -> + [$A | incr_domain(Domain)]; +incr_domain([Char | Domain]) -> + [Char+1 | Domain]. + + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Compatibility tests. Call the inet_SUITE tests, but with %% lookup = [file,dns] instead of [native] diff --git a/lib/kernel/test/interactive_shell_SUITE.erl b/lib/kernel/test/interactive_shell_SUITE.erl index f058b53497..3993f99ed0 100644 --- a/lib/kernel/test/interactive_shell_SUITE.erl +++ b/lib/kernel/test/interactive_shell_SUITE.erl @@ -766,7 +766,7 @@ start_runerl_node(RunErl,Erl,Tempdir,Nodename,Args) -> start_runerl_command(RunErl, Tempdir, Cmd) -> FullCmd = "\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++" \""++Cmd++"\"", - ct:pal("~s",[FullCmd]), + ct:pal("~ts",[FullCmd]), os:cmd(FullCmd). start_toerl_server(ToErl,Tempdir) -> diff --git a/lib/kernel/test/kernel_test_lib.erl b/lib/kernel/test/kernel_test_lib.erl index 440309974e..c4de916b1a 100644 --- a/lib/kernel/test/kernel_test_lib.erl +++ b/lib/kernel/test/kernel_test_lib.erl @@ -1383,7 +1383,7 @@ linux_info_lookup_collect(Key1, [Key2, Value|Rest], Values) -> linux_info_lookup_collect(_, _, Values) -> lists:reverse(Values). -maybe_skip(HostInfo) -> +maybe_skip(_HostInfo) -> %% We have some crap machines that causes random test case failures %% for no obvious reason. So, attempt to identify those without actually diff --git a/lib/kernel/test/seq_trace_SUITE.erl b/lib/kernel/test/seq_trace_SUITE.erl index f8efd1ffea..4a00b8d3d0 100644 --- a/lib/kernel/test/seq_trace_SUITE.erl +++ b/lib/kernel/test/seq_trace_SUITE.erl @@ -26,7 +26,7 @@ init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2]). -export([token_set_get/1, tracer_set_get/1, print/1, - old_heap_token/1, + old_heap_token/1,mature_heap_token/1, send/1, distributed_send/1, recv/1, distributed_recv/1, trace_exit/1, distributed_exit/1, call/1, port/1, port_clean_token/1, @@ -54,7 +54,7 @@ suite() -> all() -> [token_set_get, tracer_set_get, print, send, send_literal, distributed_send, recv, distributed_recv, trace_exit, - old_heap_token, + old_heap_token, mature_heap_token, distributed_exit, call, port, match_set_seq_token, port_clean_token, gc_seq_token, label_capability_mismatch, @@ -538,18 +538,24 @@ call(Config) when is_list(Config) -> %% The token should follow spawn, just like it follows messages. inherit_on_spawn(Config) when is_list(Config) -> - lists:foreach(fun (Test) -> - inherit_on_spawn_test(Test) - end, - [spawn, spawn_link, spawn_monitor, - spawn_opt, spawn_request]), + lists:foreach( + fun (Test) -> + lists:foreach( + fun (TraceFlags) -> + inherit_on_spawn_test(Test, TraceFlags) + end, + combinations(spawn_trace_flags())) + end, + [spawn, spawn_link, spawn_monitor, + spawn_opt, spawn_request]), ok. -inherit_on_spawn_test(Spawn) -> - io:format("Testing ~p()~n", [Spawn]), +inherit_on_spawn_test(Spawn, TraceFlags) -> + io:format("Testing ~p() with ~p trace flags~n", [Spawn, TraceFlags]), seq_trace:reset_trace(), start_tracer(), + start_spawn_tracer(TraceFlags), Ref = make_ref(), seq_trace:set_token(label,Ref), @@ -580,6 +586,7 @@ inherit_on_spawn_test(Spawn) -> receive {gurka,Ref} -> ok end, seq_trace:reset_trace(), + erlang:trace(self(),false,[procs|TraceFlags]), Sequence = lists:keysort(3, stop_tracer(6)), io:format("Sequence: ~p~n", [Sequence]), @@ -644,9 +651,35 @@ inherit_on_spawn_test(Spawn) -> GurkaMsg}, _} = RGurkaMsg, + + Links = not(spawn =:= spawn orelse Spawn =:= spawn_monitor), + SoL = lists:member(set_on_link,TraceFlags) orelse + lists:member(set_on_first_link,TraceFlags), + SoS = lists:member(set_on_spawn,TraceFlags) orelse + lists:member(set_on_first_sapwn,TraceFlags), + + NoTraceMessages = + if + SoS andalso Links -> + 4; + SoS andalso not Links -> + 2; + SoL andalso Links -> + 4; + SoL andalso not Links-> + 1; + Links andalso not SoL andalso not SoS -> + 2; + not Links andalso not SoL andalso not SoS -> + 1 + end, + + TraceMessages = stop_spawn_tracer(NoTraceMessages), + unlink(Other), exit(Other, kill), + ok. inherit_on_dist_spawn(Config) when is_list(Config) -> @@ -971,6 +1004,24 @@ old_heap_token(Config) when is_list(Config) -> {label,NewLabel} = seq_trace:get_token(label), ok. +%% Verify changing label on existing token when it resides on mature heap. +%% Bug caused faulty ref from old to new heap. +mature_heap_token(Config) when is_list(Config) -> + + seq_trace:set_token(label, 1), + erlang:garbage_collect(self(), [{type, minor}]), + %% Now token should be on mature heap + %% Set a new non-literal label which should reside on new-heap. + NewLabel = {self(), "new label"}, + seq_trace:set_token(label, NewLabel), + + %% If bug, we now have a ref from mature to new heap. If we now GC + %% twice the token will refer to deallocated memory. + erlang:garbage_collect(self(), [{type, minor}]), + erlang:garbage_collect(self(), [{type, minor}]), + {label,NewLabel} = seq_trace:get_token(label), + ok. + match_set_seq_token(doc) -> ["Tests that match spec function set_seq_token does not " @@ -1366,7 +1417,6 @@ start_tracer(Node) -> unlink(Pid), Pid end. - set_token_flags([]) -> ok; @@ -1379,6 +1429,51 @@ set_token_flags([Flag|Flags]) -> seq_trace:set_token(Flag, true), set_token_flags(Flags). +start_spawn_tracer(TraceFlags) -> + + %% Disable old trace flags + erlang:trace(self(), false, spawn_trace_flags()), + + Me = self(), + Ref = make_ref(), + Pid = spawn_link( + fun () -> + register(spawn_tracer, self()), + Me ! Ref, + (fun F(Data) -> + receive + {get, N, StopRef, Pid} when N =< length(Data) -> + Pid ! {lists:reverse(Data), StopRef}; + M when element(1,M) =:= trace -> + F([M|Data]) + end + end)([]) + end), + receive + Ref -> + erlang:trace(self(),true,[{tracer,Pid}, procs | TraceFlags]) + end. + +stop_spawn_tracer(N) -> + Ref = make_ref(), + spawn_tracer ! {get, N, Ref, self()}, + receive + {Data, Ref} -> + Data + end. + +spawn_trace_flags() -> + [set_on_spawn, set_on_link, set_on_spawn, + set_on_first_link, set_on_first_spawn]. + +combinations(Flags) -> + %% Do a bit of sofs magic to create a list of lists with + %% all the combinations of all the flags above + Set = sofs:from_term(Flags), + Product = sofs:product(list_to_tuple(lists:duplicate(length(Flags),Set))), + Combinations = [lists:usort(tuple_to_list(T)) || T <- sofs:to_external(Product)], + [[] | lists:usort(Combinations)]. + check_ts(no_timestamp, Ts) -> try no_timestamp = Ts diff --git a/lib/megaco/doc/src/notes.xml b/lib/megaco/doc/src/notes.xml index d3e1c9716a..7ea065ff72 100644 --- a/lib/megaco/doc/src/notes.xml +++ b/lib/megaco/doc/src/notes.xml @@ -37,7 +37,23 @@ section is the version number of Megaco.</p> - <section><title>Megaco 3.19.4</title> + <section><title>Megaco 3.19.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed usage of <c>AC_CONFIG_AUX_DIRS()</c> macros in + configure script sources.</p> + <p> + Own Id: OTP-17093 Aux Id: ERL-1447, PR-2948 </p> + </item> + </list> + </section> + +</section> + +<section><title>Megaco 3.19.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/megaco/test/megaco_segment_SUITE.erl b/lib/megaco/test/megaco_segment_SUITE.erl index a403c3309d..3763a20954 100644 --- a/lib/megaco/test/megaco_segment_SUITE.erl +++ b/lib/megaco/test/megaco_segment_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2020. All Rights Reserved. +%% Copyright Ericsson AB 2006-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -2827,6 +2827,42 @@ do_send_segmented_msg_ooo1([MgcNode, MgNode]) -> d("[MG] start the simulation"), {ok, MgId} = megaco_test_megaco_generator:exec(Mg, MgEvSeq), + %% Await MGC ready for segments + d("await MGC trigger event"), + MgcPid = + receive + {ready_for_segmented_msg, mgc, Pid1} -> + d("received MGC trigger event"), + Pid1 + after 5000 -> + d("timeout waiting for MGC trigger event: ~p", + [megaco_test_lib:flush()]), + ?ERROR(timeout_MGC_trigger_event) + end, + + %% Await MG ready for segments + d("await MG trigger event"), + MgPid = + receive + {ready_for_segmented_msg, mg, Pid2} -> + d("received MG trigger event"), + Pid2 + after 5000 -> + d("timeout waiting for MG trigger event: ~p", + [megaco_test_lib:flush()]), + ?ERROR(timeout_MG_trigger_event) + end, + + %% Instruct the MG to continue + d("send continue to MG"), + MgPid ! {continue_with_segmented_msg, self()}, + + sleep(500), + + %% Instruct the MGC to continue + d("send continue to MGC"), + MgcPid ! {continue_with_segmented_msg, self()}, + d("await the generator reply(s)"), await_completion([MgcId, MgId]), @@ -2853,6 +2889,8 @@ ssmo1_mgc_event_sequence(text, tcp) -> Mid = {deviceName,"mgc"}, ScrVerifyFun = ssmo1_mgc_verify_service_change_req_msg_fun(), ServiceChangeRep = ssmo1_mgc_service_change_reply_msg(Mid, 1), + AnnounceReadySegs = ssmo1_mgc_announce_ready_for_segmented_msg_fun(), + AwaitContinueSegs = ssmo1_mgc_continue_with_segmented_msg_fun(), TermId1 = #megaco_term_id{id = ["00000000","00000000","00000001"]}, CtxId1 = 1, @@ -2923,7 +2961,13 @@ ssmo1_mgc_event_sequence(text, tcp) -> {expect_accept, any}, {expect_receive, "service-change-request", {ScrVerifyFun, 5000}}, {send, "service-change-reply", ServiceChangeRep}, - {expect_nothing, 1000}, + + {trigger, "announce ready for segmented message", + AnnounceReadySegs}, + {trigger, "await continue for segmented message", + AwaitContinueSegs}, + + %% {expect_nothing, 1000}, {send, "notify request", NotifyReq}, {expect_receive, "notify reply: segment 1", {NrVerifyFun1, 1000}}, {expect_receive, "notify reply: segment 2", {NrVerifyFun2, 1000}}, @@ -3053,6 +3097,23 @@ ssmo1_mgc_verify_service_change_req(#'MegacoMessage'{mess = Mess} = M) -> {error, {invalid_serviceChangeParms, Parms}} end. +ssmo1_mgc_announce_ready_for_segmented_msg_fun() -> + TC = self(), + fun() -> + TC ! {ready_for_segmented_msg, mgc, self()} + end. + +ssmo1_mgc_continue_with_segmented_msg_fun() -> + TC = self(), + fun() -> + p("[MGC] await continue with segmented message"), + receive + {continue_with_segmented_msg, TC} -> + p("[MGC] received continue with segmented message"), + ok + end + end. + ssmo1_mgc_verify_notify_reply_segment_msg_fun(SN, Last, TransId, TermId, Cid) -> fun(Msg) -> @@ -3219,6 +3280,8 @@ ssmo1_mg_event_sequence(text, tcp) -> ConnectVerify = ssmo1_mg_verify_handle_connect_fun(), ServiceChangeReq = ssmo1_mg_service_change_request_ar(Mid, 1), ServiceChangeReplyVerify = ssmo1_mg_verify_service_change_reply_fun(), + AnnounceReadySegs = ssmo1_mg_announce_ready_for_segmented_msg_fun(), + AwaitContinueSegs = ssmo1_mg_continue_with_segmented_msg_fun(), Tid1 = #megaco_term_id{id = ["00000000","00000000","00000001"]}, Tid2 = #megaco_term_id{id = ["00000000","00000000","00000002"]}, Tid3 = #megaco_term_id{id = ["00000000","00000000","00000003"]}, @@ -3247,8 +3310,13 @@ ssmo1_mg_event_sequence(text, tcp) -> {megaco_callback, handle_trans_reply, ServiceChangeReplyVerify}, {megaco_update_conn_info, protocol_version, ?VERSION}, {megaco_update_conn_info, segment_send, 3}, - {megaco_update_conn_info, max_pdu_size, 128}, - {sleep, 1000}, + {megaco_update_conn_info, max_pdu_size, 128}, + + {trigger, "announce ready for segmented message", + AnnounceReadySegs}, + {trigger, "await continue for segmented message", + AwaitContinueSegs}, + {megaco_callback, handle_trans_request, NotifyReqVerify}, {megaco_callback, handle_trans_ack, AckVerify, 15000}, megaco_stop_user, @@ -3257,7 +3325,6 @@ ssmo1_mg_event_sequence(text, tcp) -> ], EvSeq. - ssmo1_mg_verify_handle_connect_fun() -> fun(Ev) -> ssmo1_mg_verify_handle_connect(Ev) end. @@ -3333,6 +3400,23 @@ ssmo1_mg_do_verify_scr(AR) -> {error, Reason6, ok} end. +ssmo1_mg_announce_ready_for_segmented_msg_fun() -> + TC = self(), + fun() -> + TC ! {ready_for_segmented_msg, mg, self()} + end. + +ssmo1_mg_continue_with_segmented_msg_fun() -> + TC = self(), + fun() -> + p("[MG] await continue with segmented message"), + receive + {continue_with_segmented_msg, TC} -> + p("[MG] received continue with segmented message"), + ok + end + end. + ssmo1_mg_verify_notify_request_fun(Tids) -> fun(Req) -> ssmo1_mg_verify_notify_request(Req, Tids) end. @@ -7913,6 +7997,9 @@ try_tc(TCName, Name, Verbosity, Pre, Case, Post) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +p(F) -> + p(F, []). + p(F, A) -> io:format("*** [~s] ~p ***" "~n " ++ F ++ "~n", diff --git a/lib/megaco/test/megaco_test_megaco_generator.erl b/lib/megaco/test/megaco_test_megaco_generator.erl index 4eedd8d731..f6ea57ab41 100644 --- a/lib/megaco/test/megaco_test_megaco_generator.erl +++ b/lib/megaco/test/megaco_test_megaco_generator.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2020. All Rights Reserved. +%% Copyright Ericsson AB 2007-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -313,6 +313,9 @@ handle_parse({megaco_callback, Verifiers0} = _Instruction, State) handle_parse({trigger, Trigger} = Instruction, State) when is_function(Trigger) -> {ok, Instruction, State}; +handle_parse({trigger, Desc, Trigger} = Instruction, State) + when is_list(Desc) andalso is_function(Trigger) -> + {ok, Instruction, State}; handle_parse(Instruction, _State) -> error({invalid_instruction, Instruction}). @@ -770,6 +773,10 @@ handle_exec({trigger, Trigger}, State) when is_function(Trigger) -> p("trigger"), (catch Trigger()), {ok, State}; +handle_exec({trigger, Desc, Trigger}, State) when is_function(Trigger) -> + p("trigger: ~s", [Desc]), + (catch Trigger()), + {ok, State}; handle_exec({sleep, To}, State) -> p("sleep ~p", [To]), diff --git a/lib/megaco/test/megaco_test_mgc.erl b/lib/megaco/test/megaco_test_mgc.erl index 8a9b182368..1204dbba07 100644 --- a/lib/megaco/test/megaco_test_mgc.erl +++ b/lib/megaco/test/megaco_test_mgc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2020. All Rights Reserved. +%% Copyright Ericsson AB 2003-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -411,7 +411,7 @@ loop(S) -> server_reply(Parent, update_conn_info_ack, Res), loop(evs(S, {uci, {Tag, Val}})); - {{conn_info, Tag}, Parent} when S#mgc.parent == Parent -> + {{conn_info, Tag}, Parent} when S#mgc.parent =:= Parent -> i("loop -> got conn_info request for ~w", [Tag]), Conns = megaco:user_info(S#mgc.mid, connections), Fun = fun(CH) -> @@ -450,48 +450,63 @@ loop(S) -> %% Give me statistics {{statistics, 1}, Parent} when S#mgc.parent == Parent -> - i("loop -> got request for statistics 1"), + i("loop(stats1) -> got request for statistics 1"), {ok, Gen} = megaco:get_stats(), - GetTrans = + i("loop(stats1) -> gen stats: " + "~n ~p", [Gen]), + GetTrans = fun(CH) -> + i("loop(stats1):GetTrans -> " + "get stats for connection ~p", [CH]), Reason = {statistics, CH}, Pid = megaco:conn_info(CH, control_pid), + i("loop(stats1):GetTrans -> control pid: ~p", [Pid]), SendMod = megaco:conn_info(CH, send_mod), + i("loop(stats1):GetTrans -> " + "send module: ~p", [SendMod]), SendHandle = megaco:conn_info(CH, send_handle), + i("loop(stats1):GetTrans -> " + "send handle: ~p", [SendHandle]), {ok, Stats} = case SendMod of megaco_tcp -> megaco_tcp:get_stats(SendHandle); megaco_udp -> megaco_udp:get_stats(SendHandle); SendMod -> exit(Pid, Reason) end, + i("loop(stats1):GetTrans -> stats: " + "~n ~p", [Stats]), {SendHandle, Stats} end, - Mid = S#mgc.mid, - Trans = - lists:map(GetTrans, megaco:user_info(Mid, connections)), + Mid = S#mgc.mid, + Trans = lists:map(GetTrans, megaco:user_info(Mid, connections)), Reply = {ok, [{gen, Gen}, {trans, Trans}]}, + i("loop(stats1) -> send reply"), server_reply(Parent, {statistics_reply, 1}, Reply), + i("loop(stats1) -> done"), loop(evs(S, {stats, 1})); {{statistics, 2}, Parent} when S#mgc.parent == Parent -> - i("loop -> got request for statistics 2"), + i("loop(stats2) -> got request for statistics 2"), {ok, Gen} = megaco:get_stats(), #mgc{tcp_sup = TcpSup, udp_sup = UdpSup} = S, TcpStats = get_trans_stats(TcpSup, megaco_tcp), UdpStats = get_trans_stats(UdpSup, megaco_udp), Reply = {ok, [{gen, Gen}, {trans, [TcpStats, UdpStats]}]}, + i("loop(stats2) -> send reply"), server_reply(Parent, {statistics_reply, 2}, Reply), + i("loop(stats2) -> done"), loop(evs(S, {stats, 2})); %% Megaco callback messages {request, Request, From} -> - d("loop -> received megaco request from ~p:" + d("loop(request) -> received megaco request from ~p:" "~n ~p", [From, Request]), {Reply, S1} = handle_megaco_request(Request, S), - d("loop -> send request reply: ~n~p", [Reply]), + d("loop(request) -> send reply: ~n~p", [Reply]), reply(From, Reply), + d("loop(request) -> done"), loop(evs(S1, {req, Request})); @@ -557,9 +572,14 @@ loop(S) -> evs(#mgc{evs = EVS} = S, Ev) when (length(EVS) < ?EVS_MAX) -> - S#mgc{evs = [{?FTS(), Ev}|EVS]}; + echo_evs(S#mgc{evs = [{?FTS(), Ev}|EVS]}); evs(#mgc{evs = EVS} = S, Ev) -> - S#mgc{evs = [{?FTS(), Ev}|lists:droplast(EVS)]}. + echo_evs(S#mgc{evs = [{?FTS(), Ev}|lists:droplast(EVS)]}). + +echo_evs(#mgc{evs = EVS} = S) -> + i("Events: " + "~n ~p", [EVS]), + S. done(#mgc{evs = EVS}, Reason) -> info_msg("Exiting with latest event(s): " diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index 48a7f4822c..f416a0324a 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = megaco -MEGACO_VSN = 3.19.4 +MEGACO_VSN = 3.19.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(MEGACO_VSN)$(PRE_VSN)" diff --git a/lib/mnesia/doc/src/Mnesia_chap1.xml b/lib/mnesia/doc/src/Mnesia_chap1.xml index fd78d01ab4..6e66132a52 100644 --- a/lib/mnesia/doc/src/Mnesia_chap1.xml +++ b/lib/mnesia/doc/src/Mnesia_chap1.xml @@ -32,13 +32,13 @@ <rev>C</rev> <file>Mnesia_chap1.xml</file> </header> - <p>The Mnesia application provides a heavy duty real-time + <p>The Mnesia application provides a heavy-duty real-time distributed database.</p> <section> <title>Scope</title> <p>This User's Guide describes how to - build Mnesia database applications, and how to integrate + build Mnesia-backed applications, and how to integrate and use the Mnesia database management system with OTP. Programming constructs are described, and numerous programming examples are included to illustrate the use of @@ -51,7 +51,7 @@ </item> <item><seeguide marker="Mnesia_chap2">Getting Started</seeguide> introduces Mnesia with an example database. Examples - are included how to start an Erlang session, specify a + are included on how to start an Erlang session, specify a Mnesia database directory, initialize a database schema, start Mnesia, and create tables. Initial prototyping of record definitions is also discussed. @@ -64,29 +64,29 @@ </item> <item><seeguide marker="Mnesia_chap4">Transactions and Other Access Contexts</seeguide> describes the transactions properties that make Mnesia into - a fault tolerant, real-time distributed database management + a fault-tolerant, real-time distributed database management system. This section also describes the concept of locking to ensure consistency in tables, and "dirty - operations", or short cuts, which bypass the transaction system + operations", or shortcuts, which bypass the transaction system to improve speed and reduce overheads. </item> <item><seeguide marker="Mnesia_chap5">Miscellaneous Mnesia Features</seeguide> describes features that enable the construction of more complex database applications. These features include indexing, checkpoints, distribution and fault - tolerance, disc-less nodes, replication manipulation, local + tolerance, disc-less nodes, replica manipulation, local content tables, concurrency, and object-based programming in Mnesia. </item> <item><seeguide marker="Mnesia_chap7">Mnesia System Information</seeguide> describes the files contained in the Mnesia database directory, database configuration data, - core and table dumps, as well as the important subject of - backup, fall-back, and disaster recovery principles. + core and table dumps, as well as the functions used for + backup, restore, fallback, and disaster recovery. </item> <item><seeguide marker="Mnesia_chap8">Combine Mnesia with - SNMP</seeguide> is a short section that outlines Mnesia - integrated with SNMP. + SNMP</seeguide> is a short section that outlines + the integration between Mnesia and SNMP. </item> <item><seeguide marker="Mnesia_App_A">Appendix A: Backup Callback Interface</seeguide> is a program listing of the @@ -110,4 +110,3 @@ database management systems.</p> </section> </chapter> - diff --git a/lib/mnesia/info b/lib/mnesia/info index bfd0816a62..2fc77fc444 100644 --- a/lib/mnesia/info +++ b/lib/mnesia/info @@ -1,2 +1,2 @@ group: dat Database Applications -short: A heavy duty real-time distributed database +short: A heavy-duty real-time distributed database diff --git a/lib/odbc/c_src/odbcserver.c b/lib/odbc/c_src/odbcserver.c index ee5dc9cd0a..b22a6cb7af 100644 --- a/lib/odbc/c_src/odbcserver.c +++ b/lib/odbc/c_src/odbcserver.c @@ -178,7 +178,7 @@ static void encode_column_dyn(db_column column, int column_nr, db_state *state); static void encode_data_type(SQLSMALLINT sql_type, SQLINTEGER size, SQLSMALLINT decimal_digits, db_state *state); -static Boolean decode_params(db_state *state, byte *buffer, int *index, param_array **params, +static Boolean decode_params(db_state *state, char *buffer, int *index, param_array **params, int i, int j, int num_param_values); /*------------- Erlang port communication functions ----------------------*/ @@ -222,7 +222,7 @@ static SQLLEN* alloc_strlen_indptr(int n, int val); static void init_driver(int erl_auto_commit_mode, int erl_trace_driver, db_state *state); -static void init_param_column(param_array *params, byte *buffer, int *index, +static void init_param_column(param_array *params, char *buffer, int *index, int num_param_values, db_state* state); static void init_param_statement(int cols, @@ -235,7 +235,7 @@ static void map_dec_num_2_c_column(col_type *type, int precision, static db_result_msg map_sql_2_c_column(db_column* column, db_state *state); -static param_array * bind_parameter_arrays(byte *buffer, int *index, +static param_array * bind_parameter_arrays(char *buffer, int *index, int cols, int num_param_values, db_state *state); @@ -259,10 +259,10 @@ static void str_tolower(char *str, int len); /* ----------------------------- CODE ------------------------------------*/ #if defined(WIN32) -# define DO_EXIT(code) do { ExitProcess((code)); exit((code));} while (0) -/* exit() called only to avoid a warning */ +# define DO_EXIT(code) do { ExitProcess((code)); _exit((code));} while (0) +/* _exit() called only to avoid a warning */ #else -# define DO_EXIT(code) exit((code)) +# define DO_EXIT(code) _exit((code)) #endif /* ----------------- Main functions --------------------------------------*/ @@ -278,7 +278,7 @@ int main(void) msg = receive_erlang_port_msg(); - temp = strtok(msg, ";"); + temp = strtok((char*)msg, ";"); if (temp == NULL) DO_EXIT(EXIT_STDIN_BODY); length = strlen(temp); @@ -509,7 +509,9 @@ static db_result_msg db_connect(byte *args, db_state *state) diagnos = get_diagnos(SQL_HANDLE_DBC, connection_handle(state), extended_errors(state)); strcat((char *)diagnos.error_msg, " Connection to database failed."); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError ); + msg = encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError ); if(!sql_success(SQLFreeHandle(SQL_HANDLE_DBC, connection_handle(state)))) @@ -544,7 +546,9 @@ static db_result_msg db_connect(byte *args, db_state *state) diagnos = get_diagnos(SQL_HANDLE_DBC, connection_handle(state), extended_errors(state)); strcat((char *)diagnos.error_msg, " Set autocommit mode failed."); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char*)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); if(!sql_success(SQLFreeHandle(SQL_HANDLE_DBC, connection_handle(state)))) @@ -576,7 +580,9 @@ static db_result_msg db_close_connection(db_state *state) if (!sql_success(result)) { diagnos = get_diagnos(SQL_HANDLE_DBC, connection_handle(state), extended_errors(state)); - return encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + return encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); } if(!sql_success(SQLFreeHandle(SQL_HANDLE_DBC, @@ -603,7 +609,9 @@ static db_result_msg db_end_tran(byte compleationtype, db_state *state) if (!sql_success(result)) { diagnos = get_diagnos(SQL_HANDLE_DBC, connection_handle(state), extended_errors(state)); - return encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + return encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); } else { return encode_atom_message("ok"); } @@ -645,7 +653,7 @@ static db_result_msg db_query(byte *sql, db_state *state) it as we want a nice and clean Erlang API */ if((strcmp((char *)is_error, "error") == 0)) { - msg = encode_error_message((char *)diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char *)diagnos.error_msg,extended_error(state, diagnos.sqlState), diagnos.nativeError); clean_state(state); return msg; } @@ -686,7 +694,7 @@ static db_result_msg db_query(byte *sql, db_state *state) ei_x_free(&(dynamic_buffer(state))); return msg; } else { - msg.buffer = dynamic_buffer(state).buff; + msg.buffer = (byte*)dynamic_buffer(state).buff; msg.length = dynamic_buffer(state).index; msg.dyn_alloc = TRUE; return msg; @@ -722,7 +730,9 @@ static db_result_msg db_select_count(byte *sql, db_state *state) if(!sql_success(SQLExecDirect(statement_handle(state), (SQLCHAR *)sql, SQL_NTS))) { diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); clean_state(state); - return encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + return encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); } if(!sql_success(SQLNumResultCols(statement_handle(state), @@ -810,7 +820,7 @@ static db_result_msg db_select(byte *args, db_state *state) ei_x_free(&(dynamic_buffer(state))); return msg; } else { - msg.buffer = dynamic_buffer(state).buff; + msg.buffer = (byte*)dynamic_buffer(state).buff; msg.length = dynamic_buffer(state).index; msg.dyn_alloc = TRUE; return msg; @@ -819,9 +829,10 @@ static db_result_msg db_select(byte *args, db_state *state) /* Description: Handles parameterized queries ex: INSERT INTO FOO VALUES(?, ?) */ -static db_result_msg db_param_query(byte *buffer, db_state *state) +static db_result_msg db_param_query(byte *byte_buffer, db_state *state) { - byte *sql; + char *sql; + char *buffer = (char *)byte_buffer; db_result_msg msg; SQLLEN num_param_values; int i, ver = 0, @@ -847,7 +858,7 @@ static db_result_msg db_param_query(byte *buffer, db_state *state) ei_get_type(buffer, &index, &erl_type, &size); - sql = (byte*)safe_malloc((sizeof(byte) * (size + 1))); + sql = safe_malloc((sizeof(byte) * (size + 1))); ei_decode_string(buffer, &index, sql); ei_decode_long(buffer, &index, &long_num_param_values); @@ -871,7 +882,9 @@ static db_result_msg db_param_query(byte *buffer, db_state *state) updates/deletes that affect no rows */ if(!sql_success(result) && !(result == SQL_NO_DATA && !strcmp((char *)diagnos.sqlState, INFO))) { - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char*)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); } else { for (i = 0; i < param_status.params_processed; i++) { switch (param_status.param_status_array[i]) { @@ -886,7 +899,9 @@ static db_result_msg db_param_query(byte *buffer, db_state *state) default: diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char*)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); i = param_status.params_processed; break; } @@ -899,7 +914,7 @@ static db_result_msg db_param_query(byte *buffer, db_state *state) msg = encode_result(state); } if(msg.length == 0) { - msg.buffer = dynamic_buffer(state).buff; + msg.buffer = (byte *)dynamic_buffer(state).buff; msg.length = dynamic_buffer(state).index; msg.dyn_alloc = TRUE; } else { /* Error occurred */ @@ -956,7 +971,9 @@ static db_result_msg db_describe_table(byte *sql, db_state *state) if (!sql_success(SQLPrepare(statement_handle(state), (SQLCHAR *)sql, SQL_NTS))){ diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); clean_state(state); return msg; } @@ -964,7 +981,9 @@ static db_result_msg db_describe_table(byte *sql, db_state *state) if(!sql_success(SQLNumResultCols(statement_handle(state), &num_of_columns))) { diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); clean_state(state); return msg; } @@ -992,7 +1011,7 @@ static db_result_msg db_describe_table(byte *sql, db_state *state) ei_x_encode_empty_list(&dynamic_buffer(state)); clean_state(state); - msg.buffer = dynamic_buffer(state).buff; + msg.buffer = (byte *)dynamic_buffer(state).buff; msg.length = dynamic_buffer(state).index; msg.dyn_alloc = TRUE; return msg; @@ -1088,7 +1107,9 @@ static db_result_msg encode_result(db_state *state) if(!sql_success(SQLNumResultCols(statement_handle(state), &num_of_columns))) { diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); clean_state(state); return msg; } @@ -1105,7 +1126,9 @@ static db_result_msg encode_result(db_state *state) if(!sql_success(SQLRowCount(statement_handle(state), &RowCountPtr))) { diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char *)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); clean_state(state); return msg; } @@ -1652,7 +1675,7 @@ static void encode_data_type(SQLSMALLINT sql_type, SQLINTEGER size, } } -static Boolean decode_params(db_state *state, byte *buffer, int *index, param_array **params, +static Boolean decode_params(db_state *state, char *buffer, int *index, param_array **params, int i, int j, int num_param_values) { int erl_type, size; @@ -1688,7 +1711,7 @@ static Boolean decode_params(db_state *state, byte *buffer, int *index, param_ar if(erl_type != ERL_STRING_EXT) { return FALSE; } - ei_decode_string(buffer, index, &(param->values.string[param->offset])); + ei_decode_string(buffer, index, (char*)&(param->values.string[param->offset])); param->offset += param->type.len; } break; @@ -2197,7 +2220,7 @@ static void init_driver(int erl_auto_commit_mode, int erl_trace_driver, DO_EXIT(EXIT_CONNECTION); } -static void init_param_column(param_array *params, byte *buffer, int *index, +static void init_param_column(param_array *params, char *buffer, int *index, int num_param_values, db_state* state) { long user_type, precision, scale, length; @@ -2509,7 +2532,7 @@ static db_result_msg map_sql_2_c_column(db_column* column, db_state *state) return msg; } -static param_array * bind_parameter_arrays(byte *buffer, int *index, +static param_array * bind_parameter_arrays(char *buffer, int *index, int cols, int num_param_values, db_state *state) { @@ -2669,7 +2692,7 @@ static db_result_msg retrive_scrollable_cursor_support_info(db_state *state) ei_x_encode_atom(&dynamic_buffer(state), "false"); ei_x_encode_atom(&dynamic_buffer(state), "false"); } - msg.buffer = dynamic_buffer(state).buff; + msg.buffer = (byte *)dynamic_buffer(state).buff; msg.length = dynamic_buffer(state).index; msg.dyn_alloc = TRUE; return msg; @@ -2700,7 +2723,9 @@ static db_result_msg more_result_sets(db_state *state) diagnos = get_diagnos(SQL_HANDLE_STMT, statement_handle(state), extended_errors(state)); strcat((char *)diagnos.error_msg, "Failed to create on of the result sets"); - msg = encode_error_message(diagnos.error_msg, extended_error(state, diagnos.sqlState), diagnos.nativeError); + msg = encode_error_message((char*)diagnos.error_msg, + extended_error(state, diagnos.sqlState), + diagnos.nativeError); return msg; } } diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index 891c6a9c31..3533310e51 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,23 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.13.1</title> + <section><title>ODBC 2.13.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed usage of <c>AC_CONFIG_AUX_DIRS()</c> macros in + configure script sources.</p> + <p> + Own Id: OTP-17093 Aux Id: ERL-1447, PR-2948 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.13.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/odbc/test/README b/lib/odbc/test/README index 0a8495afbb..5ae6073d9a 100644 --- a/lib/odbc/test/README +++ b/lib/odbc/test/README @@ -47,7 +47,7 @@ something like this: --- Start example of .odbc.ini ---- -[Postgres] +[PostgresLinux64Ubuntu] Driver=/usr/lib/psqlodbc.so Description=Postgres driver ServerName=myhost diff --git a/lib/odbc/test/postgres.erl b/lib/odbc/test/postgres.erl index 1955358206..e055be9544 100644 --- a/lib/odbc/test/postgres.erl +++ b/lib/odbc/test/postgres.erl @@ -207,7 +207,7 @@ bit_true_selected() -> %------------------------------------------------------------------------- float_min() -> - 1.79e-307. + 5.0e-324. float_max() -> 1.79e+308. @@ -215,7 +215,7 @@ create_float_table() -> " (FIELD float)". float_underflow() -> - "1.80e-308". + "2.4e-324". float_overflow() -> "1.80e+308". @@ -288,7 +288,7 @@ describe_string() -> {"str4",{sql_varchar,10}}]}. describe_floating() -> - {ok,[{"f",sql_real},{"r",sql_real},{"d",{sql_float,15}}]}. + {ok,[{"f",sql_real},{"r",sql_real},{"d",{sql_float,17}}]}. describe_dec_num() -> {ok,[{"mydec",{sql_numeric,9,3}},{"mynum",{sql_numeric,9,2}}]}. diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index a8e78b5a32..c08f40d233 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.13.1 +ODBC_VSN = 2.13.2 diff --git a/lib/public_key/src/pubkey_cert.erl b/lib/public_key/src/pubkey_cert.erl index 5f8a5bd530..e84edffd53 100644 --- a/lib/public_key/src/pubkey_cert.erl +++ b/lib/public_key/src/pubkey_cert.erl @@ -1218,13 +1218,28 @@ validity(Opts) -> DefFrom0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())-1), DefTo0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())+7), {DefFrom, DefTo} = proplists:get_value(validity, Opts, {DefFrom0, DefTo0}), - Format = + + GenFormat = fun({Y,M,D}) -> lists:flatten( io_lib:format("~4..0w~2..0w~2..0w130000Z",[Y,M,D])) end, - #'Validity'{notBefore={generalTime, Format(DefFrom)}, - notAfter ={generalTime, Format(DefTo)}}. + + UTCFormat = + fun({Y,M,D}) -> + [_, _, Y3, Y4] = integer_to_list(Y), + lists:flatten( + io_lib:format("~s~2..0w~2..0w130000Z",[[Y3, Y4],M,D])) + end, + + #'Validity'{notBefore = validity_format(DefFrom, GenFormat, UTCFormat), + notAfter = validity_format(DefTo, GenFormat, UTCFormat)}. + +validity_format({Year, _, _} = Validity, GenFormat, _UTCFormat) when Year >= 2049 -> + {generalTime, GenFormat(Validity)}; +validity_format(Validity, _GenFormat, UTCFormat) -> + {utcTime, UTCFormat(Validity)}. + sign_algorithm(#'RSAPrivateKey'{} = Key , Opts) -> case proplists:get_value(rsa_padding, Opts, rsa_pkcs1_pss_padding) of diff --git a/lib/runtime_tools/doc/src/scheduler.xml b/lib/runtime_tools/doc/src/scheduler.xml index 713d70548b..d539ccb1c6 100644 --- a/lib/runtime_tools/doc/src/scheduler.xml +++ b/lib/runtime_tools/doc/src/scheduler.xml @@ -63,7 +63,10 @@ <taglist> <tag><c>{normal, SchedulerId, Util, Percent}</c></tag> <item>Scheduler utilization of a normal scheduler with number - <c>SchedulerId</c>.</item> + <c>SchedulerId</c>. Schedulers that are not online will also be + included. + <seeerl marker="erts:erlang#system_info_schedulers_online">Online + schedulers</seeerl> have the lowest <c>SchedulerId</c>.</item> <tag><c>{cpu, SchedulerId, Util, Percent}</c></tag> <item>Scheduler utilization of a dirty-cpu scheduler with number <c>SchedulerId</c>.</item> @@ -117,6 +120,29 @@ <p>Calculate scheduler utilizations for the time interval from when <c><anno>Sample</anno></c> was taken and "now". The same as calling <c>scheduler:utilization(Sample, scheduler:sample_all())</c>.</p> + <note> + <p> + Scheduler utilization is measured as an average value over a time + interval, calculated as the difference between two samples. To get + good useful utilization values at least a couple of seconds should + have passed between the two samples. For this reason, you should not + do + </p> +<pre> +scheduler:utilization(scheduler:sample()). % DO NOT DO THIS! +</pre> + <p> + The above example takes two samples in rapid succession and calculates + the scheduler utilization between them. The resulting values will + probably be more misleading than informative. + </p> + <p> + Instead use <seemfa marker="#utilization/1"> + <c>scheduler:utilization(Seconds)</c></seemfa> or let some time pass + between <c>Sample=scheduler:sample()</c> and + <c>scheduler:utilization(Sample)</c>. + </p> + </note> </desc> </func> diff --git a/lib/runtime_tools/test/erts_alloc_config_SUITE.erl b/lib/runtime_tools/test/erts_alloc_config_SUITE.erl index 6ae51d9a26..9ab61b89d2 100644 --- a/lib/runtime_tools/test/erts_alloc_config_SUITE.erl +++ b/lib/runtime_tools/test/erts_alloc_config_SUITE.erl @@ -25,7 +25,9 @@ -include_lib("common_test/include/ct.hrl"). %-compile(export_all). --export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). +-export([all/0, suite/0, + init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2]). %% Testcases -export([basic/1]). @@ -40,6 +42,18 @@ suite() -> all() -> [basic]. +init_per_suite(Config) -> + case test_server:is_asan() of + true -> + %% No point testing own allocators under address sanitizer. + {skip, "Address sanitizer"}; + false -> + Config + end. + +end_per_suite(_Config) -> + ok. + init_per_testcase(Case, Config) when is_list(Config) -> [{testcase, Case}, {erl_flags_env, save_env()} | Config]. diff --git a/lib/sasl/src/systools_relup.erl b/lib/sasl/src/systools_relup.erl index 63e18d394b..19d25fef6e 100644 --- a/lib/sasl/src/systools_relup.erl +++ b/lib/sasl/src/systools_relup.erl @@ -290,9 +290,9 @@ foreach_baserel_up(TopRel, TopApps, [BaseRelDc|BaseRelDcs], Path, Opts, %% {RUs1, Ws1} = collect_appup_scripts(up, TopApps, BaseRel, Ws0++Ws, []), - {RUs2, Ws2} = create_add_app_scripts(BaseRel, TopRel, RUs1, Ws1), + {RUs2, Ws2} = prepend_add_app_scripts(BaseRel, TopRel, RUs1, Ws1), - {RUs3, Ws3} = create_remove_app_scripts(BaseRel, TopRel, RUs2, Ws2), + {RUs3, Ws3} = append_remove_app_scripts(BaseRel, TopRel, RUs2, Ws2), {RUs4, Ws4} = check_for_emulator_restart(TopRel, BaseRel, RUs3, Ws3, Opts), @@ -342,9 +342,9 @@ foreach_baserel_dn(TopRel, TopApps, [BaseRelDc|BaseRelDcs], Path, Opts, %% {RUs1, Ws1} = collect_appup_scripts(dn, TopApps, BaseRel, Ws0++Ws, []), - {RUs2, Ws2} = create_add_app_scripts(TopRel, BaseRel, RUs1, Ws1), + {RUs2, Ws2} = prepend_add_app_scripts(TopRel, BaseRel, RUs1, Ws1), - {RUs3, Ws3} = create_remove_app_scripts(TopRel, BaseRel, RUs2, Ws2), + {RUs3, Ws3} = append_remove_app_scripts(TopRel, BaseRel, RUs2, Ws2), {RUs4, Ws4} = check_for_emulator_restart(TopRel, BaseRel, RUs3, Ws3, Opts), @@ -439,7 +439,7 @@ collect_appup_scripts(_, [], _, Ws, RUs) -> {RUs, Ws}. %% FromRel = ToRel = #release %% ToApps = [#application] %% -create_add_app_scripts(FromRel, ToRel, RU0s, W0s) -> +prepend_add_app_scripts(FromRel, ToRel, RU0s, W0s) -> AddedNs = [{N, T} || {N, _V, T} <- ToRel#release.applications, not lists:keymember(N, 1, FromRel#release.applications)], %% io:format("Added apps: ~p~n", [AddedNs]), @@ -454,12 +454,12 @@ create_add_app_scripts(FromRel, ToRel, RU0s, W0s) -> %% %% XXX ToApps not used. %% -create_remove_app_scripts(FromRel, ToRel, RU0s, W0s) -> +append_remove_app_scripts(FromRel, ToRel, RU0s, W0s) -> RemovedNs = [N || {N, _V, _T} <- FromRel#release.applications, not lists:keymember(N, 1, ToRel#release.applications)], %% io:format("Removed apps: ~p~n", [RemovedNs]), RUs = [[{remove_application, N}] || N <- RemovedNs], - {RUs ++ RU0s, W0s}. + { RU0s ++ RUs, W0s}. %% get_script_from_appup(Mode, TopApp, BaseVsn, Ws, RUs) -> {NRUs, NWs} %% Mode = up | dn diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 3c60c0fa21..4b67d406a6 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -72,7 +72,8 @@ groups() -> otp_9507_path_ebin, additional_files_tar, erts_tar]}, {relup, [], [normal_relup, restart_relup, abnormal_relup, no_sasl_relup, - no_appup_relup, bad_appup_relup, app_start_type_relup, regexp_relup + no_appup_relup, bad_appup_relup, app_start_type_relup, regexp_relup, + replace_app_relup ]}, {hybrid, [], [normal_hybrid,hybrid_no_old_sasl,hybrid_no_new_sasl]}, {options, [], [otp_6226_outdir,app_file_defaults]}]. @@ -1942,6 +1943,59 @@ regexp_relup(Config) -> ok. +%% make_relup: Replace an application dependency with another +%% The key part here is that the new application should be +%% started before the old one is stopped. +replace_app_relup(Config) when is_list(Config) -> + {ok, OldDir} = file:get_cwd(), + + {LatestDir,LatestName} = create_script(replace_app0,Config), + {_LatestDir1,LatestName1} = create_script(replace_app1,Config), + + DataDir = filename:absname(?copydir), + LibDir = [fname([DataDir, d_replace_app, lib])], + P = [fname([LibDir, '*', ebin]), + fname([DataDir, lib, kernel, ebin]), + fname([DataDir, lib, stdlib, ebin]), + fname([DataDir, lib, sasl, ebin])], + + ok = file:set_cwd(LatestDir), + + ok = systools:make_relup(LatestName, [LatestName1], [LatestName1], + [{path, P}]), + + check_start_stop_order([{start,gh},{stop,fe}], [{start,fe},{stop,gh}]), + + ok = file:set_cwd(OldDir), + ok. + + +check_start_stop_order(UpOrder, DownOrder) -> + + {ok, [{_V0, [{_V1, [], Up}], + [{_V1, [], Down}] + }]} = file:consult(relup), + + GetAppStartStop = fun(Instr) -> + [{Action,App} || {apply,{application,Action,[App|_]}} <- Instr, + lists:member(Action,[start,stop])] + end, + + case GetAppStartStop(Up) of + UpOrder -> ok; + ActualUpOrder -> + ct:fail("Incorrect upgrade order.~nExpected: ~p~nGot:~p", + [UpOrder,ActualUpOrder]) + end, + + case GetAppStartStop(Down) of + DownOrder -> ok; + ActualDownOrder -> + ct:fail("Incorrect down order.~nExpected: ~p~nGot:~p", + [DownOrder,ActualDownOrder]) + end, + + ok. %% make_hybrid_boot: Normal case. %% For upgrade of erts - create a boot file which is a hybrid between @@ -2497,7 +2551,13 @@ create_script({unicode,RelVsn},Config) -> do_create_script(unicode,RelVsn,Config,current,Apps); create_script(duplicate_modules,Config) -> Apps = core_apps(current) ++ [{app1,"1.0"},{app2,"1.0"}], - do_create_script(duplicate_modules,Config,current,Apps). + do_create_script(duplicate_modules,Config,current,Apps); +create_script(replace_app0,Config) -> + Apps = core_apps(current) ++ [{db,"1.1"},{gh,"1.0"}], + do_create_script(repace_app0,Config,current,Apps); +create_script(replace_app1,Config) -> + Apps = core_apps(current) ++ [{db,"1.0"},{fe,"2.1"}], + do_create_script(repace_app1,Config,current,Apps). do_create_script(Id,Config,ErtsVsn,AppVsns) -> diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/ebin/db.app b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/ebin/db.app new file mode 100644 index 0000000000..d12fcfaf7d --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/ebin/db.app @@ -0,0 +1,8 @@ +{application, db, + [{description, "ERICSSON NR FOR DB"}, + {vsn, "1.0"}, + {modules, [db1, db2]}, + {registered, []}, + {applications, [fe]}, + {env, []}, + {start, {db1, start, []}}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db1.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db1.erl new file mode 100644 index 0000000000..a17640316e --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db1.erl @@ -0,0 +1,2 @@ +-module(db2). +-vsn("1.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db2.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db2.erl new file mode 100644 index 0000000000..a17640316e --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.0/src/db2.erl @@ -0,0 +1,2 @@ +-module(db2). +-vsn("1.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.app b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.app new file mode 100644 index 0000000000..517a0810f9 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.app @@ -0,0 +1,8 @@ +{application, db, + [{description, "ERICSSON NR FOR DB"}, + {vsn, "1.1"}, + {modules, [db1, db2]}, + {registered, []}, + {applications, [gh]}, + {env, []}, + {start, {db1, start, []}}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.appup b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.appup new file mode 100644 index 0000000000..12d7ad4c9c --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/ebin/db.appup @@ -0,0 +1,14 @@ +{ + "1.1", +%%% Upgrade from: + [ + {"1.0", [{update, db1, soft, soft_purge, soft_purge, []}, + {update, db2, soft, soft_purge, soft_purge, [db1]}]} + ], + +%%% Downgrade to: + [ + {"1.0", [{update, db1, soft, soft_purge, soft_purge, []}, + {update, db2, soft, soft_purge, soft_purge, [db1]}]} + ] +}. diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db1.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db1.erl new file mode 100644 index 0000000000..ee7497f5c1 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db1.erl @@ -0,0 +1,2 @@ +-module(db2). +-vsn("1.1"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db2.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db2.erl new file mode 100644 index 0000000000..ee7497f5c1 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/db-1.1/src/db2.erl @@ -0,0 +1,2 @@ +-module(db2). +-vsn("1.1"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/ebin/fe.app b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/ebin/fe.app new file mode 100644 index 0000000000..717d30cf45 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/ebin/fe.app @@ -0,0 +1,8 @@ +{application, fe, + [{description, "ERICSSON NR FOR FE"}, + {vsn, "2.1"}, + {modules, [fe1, fe2, fe3]}, + {registered, []}, + {applications, []}, + {env, []}, + {start, {fe2, start, []}}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe1.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe1.erl new file mode 100644 index 0000000000..aa5bfa8098 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe1.erl @@ -0,0 +1,2 @@ +-module(fe1). +-vsn("1.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe2.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe2.erl new file mode 100644 index 0000000000..869f3b93c8 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe2.erl @@ -0,0 +1,2 @@ +-module(fe2). +-vsn("1.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe3.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe3.erl new file mode 100644 index 0000000000..6473342f52 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/fe-2.1/src/fe3.erl @@ -0,0 +1,2 @@ +-module(fe3). +-vsn("2.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/ebin/gh.app b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/ebin/gh.app new file mode 100644 index 0000000000..2823a7e592 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/ebin/gh.app @@ -0,0 +1,8 @@ +{application, gh, + [{description, "ERICSSON NR FOR GH"}, + {vsn, "1.0"}, + {modules, [gh1]}, + {registered, []}, + {applications, []}, + {env, []}, + {start, {gh1, start, []}}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/src/gh1.erl b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/src/gh1.erl new file mode 100644 index 0000000000..acd0f43d6a --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_replace_app/lib/gh-1.0/src/gh1.erl @@ -0,0 +1,2 @@ +-module(gh1). +-vsn("1.0"). diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 2646005f4b..792090f3bf 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -34,7 +34,58 @@ </header> - <section><title>SNMP 5.7</title> + <section><title>SNMP 5.7.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + [manager] In a function handling snmp errors, an unused + result (_Error) could result in matching issues and + therefor case clause runtime errors (crash). Note that + this would only happen in *very* unusual error cases.</p> + <p> + Own Id: OTP-17161</p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.7.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + [manager] Misspelled priv protocol (atom) made it + impossible to update usm user 'priv_key' configuration + for usmAesCfb128Protocol via function calls.</p> + <p> + Own Id: OTP-17110 Aux Id: ERIERL-586 </p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.7.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed usage of <c>AC_CONFIG_AUX_DIRS()</c> macros in + configure script sources.</p> + <p> + Own Id: OTP-17093 Aux Id: ERL-1447, PR-2948 </p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.7</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/snmp/doc/src/snmpa.xml b/lib/snmp/doc/src/snmpa.xml index 178f25ccb0..c3025dc945 100644 --- a/lib/snmp/doc/src/snmpa.xml +++ b/lib/snmp/doc/src/snmpa.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2004</year><year>2020</year> + <year>2004</year><year>2021</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -503,6 +503,24 @@ notification_delivery_info() = #snmpa_notification_delivery_info{} <desc> <p>Retrieve all tables known to the agent.</p> + <marker id="which_transports"></marker> + </desc> + </func> + + <func> + <name since="">which_transports() -> Result</name> + <fsummary>Get all configured transports</fsummary> + <type> + <v>Result = [{TDomain, TAddress} | {TDomain, TAddress, Kind}]</v> + <v>TDomain = transportDomainUdpIpv4 | transportDomainUdpIpv6</v> + <v>TAddress = {IpAddr, IpPort}</v> + <v>IpAddr = inet:ip_address()</v> + <v>IpPort = pos_integer()</v> + <v>Kind = req_responder | trap_sender</v> + </type> + <desc> + <p>Retrieve all configured transports.</p> + <marker id="which_variables"></marker> </desc> </func> diff --git a/lib/snmp/src/agent/snmpa.erl b/lib/snmp/src/agent/snmpa.erl index 9e428466fa..729789d487 100644 --- a/lib/snmp/src/agent/snmpa.erl +++ b/lib/snmp/src/agent/snmpa.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -63,6 +63,8 @@ register_subagent/3, unregister_subagent/2, + which_transports/0, + send_notification2/3, send_notification/3, send_notification/4, send_notification/5, send_notification/6, send_notification/7, @@ -832,6 +834,18 @@ sys_up_time() -> %%%----------------------------------------------------------------- +which_transports() -> + {value, Transports} = snmp_framework_mib:intAgentTransports(get), + [case Kind of + all -> + {Domain, Address}; + _ -> + {Domain, Address, Kind} + end || {Domain, Address, Kind, _} <- Transports]. + + +%%%----------------------------------------------------------------- + restart_worker() -> restart_worker(snmp_master_agent). diff --git a/lib/snmp/src/agent/snmpa_net_if.erl b/lib/snmp/src/agent/snmpa_net_if.erl index 835b8d4375..b6b115bd75 100644 --- a/lib/snmp/src/agent/snmpa_net_if.erl +++ b/lib/snmp/src/agent/snmpa_net_if.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2020. All Rights Reserved. +%% Copyright Ericsson AB 2004-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -130,6 +130,7 @@ {socket, kind = all :: all | transport_kind(), domain = snmpUDPDomain, + address :: inet:ip_address(), port_no :: pos_integer(), port_info :: port_info(), %% <EPHEMERAL-FOR-FUTUR-USE> @@ -273,8 +274,8 @@ do_init(Prio, NoteStore, MasterAgent, Parent, Opts) -> %% will be taken from the "global" socket options (which serve as %% default values). %% Also, note that Ephm are not actually used at this time. - {Ephm, PortInfo, SocketOpts} = socket_opts(Domain, Address, - RawSocketOpts, Opts), + {Ephm, IpAddr, PortInfo, SocketOpts} = socket_opts(Domain, Address, + RawSocketOpts, Opts), ?vtrace("socket opts processed:" "~n Ephm: ~p" "~n Port Info: ~p" @@ -296,6 +297,7 @@ do_init(Prio, NoteStore, MasterAgent, Parent, Opts) -> %% or a range), so it could have been "generated". %% Also, shall we push this into the transport (handled by the %% FRAMEWORK MIB)? Would not work for ephemeral sockets. + address = IpAddr, port_no = IpPort, port_info = PortInfo, ephm = Ephm, @@ -2034,7 +2036,7 @@ socket_opts(Domain, {IpAddr, PortInfo}, SocketOpts, DefaultOpts) -> %% Ephm = get_ephemeral(SocketOpts), %% {Ephm, PortInfo, Opts}. %% </EPHEMERAL-FOR-FUTUR-USE> - {none, PortInfo, Opts}. + {none, IpAddr, PortInfo, Opts}. %% ---------------------------------------------------------------- @@ -2150,10 +2152,21 @@ get_info(#state{transports = Transports, reqs = Reqs}) -> [{reqs, Reqs}, {counters, Counters}, {process_memory, ProcSize}, - {transport_info, [{PortNo, Kind, get_port_info(Socket)} || - #transport{socket = Socket, - port_no = PortNo, - kind = Kind} <- Transports]}]. + {transport_info, [#{tdomain => Domain, + taddress => {Address, PortNo}, + transport_kind => Kind, + port_info => PortInfo, + opts => Opts, + socket_info => get_port_info(Socket), + num_reqs => length(Refs)} || + #transport{socket = Socket, + domain = Domain, + address = Address, + port_no = PortNo, + port_info = PortInfo, + opts = Opts, + kind = Kind, + req_refs = Refs} <- Transports]}]. proc_mem(P) when is_pid(P) -> case (catch erlang:process_info(P, memory)) of diff --git a/lib/snmp/src/manager/snmpm_config.erl b/lib/snmp/src/manager/snmpm_config.erl index 10a39986a0..356ba44b08 100644 --- a/lib/snmp/src/manager/snmpm_config.erl +++ b/lib/snmp/src/manager/snmpm_config.erl @@ -3118,7 +3118,7 @@ do_update_usm_user_info(Key, {error, {unsupported_crypto, des_cbc}} end; do_update_usm_user_info(Key, - #usm_user{priv = usmAesCfb128Protocoll} = User, + #usm_user{priv = usmAesCfb128Protocol} = User, priv_key, Val) when length(Val) =:= 16 -> case is_crypto_supported(aes_cfb128) of diff --git a/lib/snmp/src/manager/snmpm_server.erl b/lib/snmp/src/manager/snmpm_server.erl index fedb1d2b32..ca18637b8d 100644 --- a/lib/snmp/src/manager/snmpm_server.erl +++ b/lib/snmp/src/manager/snmpm_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2020. All Rights Reserved. +%% Copyright Ericsson AB 2004-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1612,16 +1612,17 @@ handle_snmp_error(#pdu{request_id = ReqId} = Pdu, Reason, State) -> handle_snmp_error(CrapError, Reason, _State) -> error_msg("received crap (snmp) error =>" - "~n~p~n~p", [CrapError, Reason]), + "~n ~p" + "~n ~p", [CrapError, Reason]), ok. handle_snmp_error(Domain, Addr, ReqId, Reason, State) -> - ?vtrace("handle_snmp_error -> entry with~n" - " Domain: ~p~n" - " Addr: ~p~n" - " ReqId: ~p~n" - " Reason: ~p", [Domain, Addr, ReqId, Reason]), + ?vtrace("handle_snmp_error -> entry with" + "~n Domain: ~p" + "~n Addr: ~p" + "~n ReqId: ~p" + "~n Reason: ~p", [Domain, Addr, ReqId, Reason]), case snmpm_config:get_agent_user_id(Domain, Addr) of {ok, UserId} -> @@ -1629,24 +1630,24 @@ handle_snmp_error(Domain, Addr, ReqId, Reason, State) -> {ok, UserMod, UserData} -> handle_error(UserId, UserMod, Reason, ReqId, UserData, State); - _Error -> + _Error1 -> case snmpm_config:user_info() of {ok, DefUserId, DefMod, DefData} -> handle_error(DefUserId, DefMod, Reason, ReqId, DefData, State); - _Error -> + _Error2 -> error_msg("failed retreiving the default user " "info handling snmp error " "<~p,~p>: ~n~w~n~w", [Domain, Addr, ReqId, Reason]) end end; - _Error -> + _Error3 -> case snmpm_config:user_info() of {ok, DefUserId, DefMod, DefData} -> handle_error(DefUserId, DefMod, Reason, ReqId, DefData, State); - _Error -> + _Error4 -> error_msg("failed retreiving the default user " "info handling snmp error " "<~p,~p>: ~n~w~n~w", @@ -1679,10 +1680,10 @@ do_handle_error(Mod, ReqId, Reason, Data) -> handle_snmp_pdu(#pdu{type = 'get-response', request_id = ReqId} = Pdu, Domain, Addr, State) -> - ?vtrace("handle_snmp_pdu(get-response) -> entry with~n" - " Domain: ~p~n" - " Addr: ~p~n" - " Pdu: ~p", [Domain, Addr, Pdu]), + ?vtrace("handle_snmp_pdu(get-response) -> entry with" + "~n Domain: ~p" + "~n Addr: ~p" + "~n Pdu: ~p", [Domain, Addr, Pdu]), case ets:lookup(snmpm_request_table, ReqId) of @@ -1843,6 +1844,7 @@ handle_snmp_pdu(#pdu{type = 'get-response', request_id = ReqId} = Pdu, end end; + handle_snmp_pdu(CrapPdu, Domain, Addr, _State) -> error_msg("received crap (snmp) Pdu from ~w:~w =>" "~p", [Domain, Addr, CrapPdu]), diff --git a/lib/snmp/test/snmp_agent_SUITE.erl b/lib/snmp/test/snmp_agent_SUITE.erl index 3d8170cada..83bc05ff05 100644 --- a/lib/snmp/test/snmp_agent_SUITE.erl +++ b/lib/snmp/test/snmp_agent_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2020. All Rights Reserved. +%% Copyright Ericsson AB 2003-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -5005,16 +5005,74 @@ command_handler([]) -> ok; command_handler([{_No, _Desc, Cmd}|Rest]) -> ?IPRINT("command_handler -> command ~w: ~n ~s", [_No, _Desc]), - case (catch Cmd()) of - ok -> - ?IPRINT("command_handler -> ~w: ok", [_No]), - command_handler(Rest); - {error, Reason} -> - ?EPRINT("command_handler -> ~w error: ~n~p", [_No, Reason]), - ?line ?FAIL(Reason); - Error -> - ?EPRINT("command_handler -> ~w unexpected: ~n~p", [_No, Error]), - ?line ?FAIL({unexpected_command_result, Error}) + %% case (catch Cmd()) of + %% ok -> + %% ?IPRINT("command_handler -> ~w: ok", [_No]), + %% command_handler(Rest); + %% {error, Reason} -> + %% ?EPRINT("command_handler -> ~w error: ~n~p", [_No, Reason]), + %% ?line ?FAIL(Reason); + %% Error -> + %% ?EPRINT("command_handler -> ~w unexpected: ~n~p", [_No, Error]), + %% ?line ?FAIL({unexpected_command_result, Error}) + %% end. + try Cmd() of + ok -> + ?IPRINT("command_handler -> ~w: ok", [_No]), + command_handler(Rest); + {error, Reason} -> + ?IPRINT("command_handler -> command ~w error", [_No]), + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("command_handler -> ~w error: ~n~p", [_No, Reason]), + ?line ?FAIL(Reason); + true -> + ?WPRINT("command_handler -> " + "failed when we got system events: " + "~n Reason: ~p" + "~n Sys Events: ~p" + "~n", [Reason, SysEvs]), + ?SKIP([{reason, Reason}, {system_events, SysEvs}]) + end; + Error -> + ?IPRINT("command_handler -> command ~w unexpected", [_No]), + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("command_handler -> " + "~w unexpected: ~n~p", [_No, Error]), + ?line ?FAIL({unexpected_command_result, Error}); + true -> + ?WPRINT("command_handler -> " + "unexpected when we got system events: " + "~n Unexpected: ~p" + "~n Sys Events: ~p" + "~n", [Error, SysEvs]), + ?SKIP([{unexpected, Error}, {system_events, SysEvs}]) + end + catch + C:E:S -> + ?IPRINT("command_handler -> command ~w catched", [_No]), + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("command_handler -> ~w catched: " + "~n Class: ~p" + "~n Error: ~p" + "~n Stack: ~p", [_No, C, E, S]), + ?line ?FAIL({catched_command_result, {C, E, S}}); + true -> + ?WPRINT("command_handler -> " + "catched when we got system events: " + "~n Catched: " + "~n Class: ~p" + "~n Error: ~p" + "~n Stack: ~p" + "~n Sys Events: ~p" + "~n", [C, E, S, SysEvs]), + ?SKIP([{catched, {C, E, S}}, {system_events, SysEvs}]) + end end. @@ -5488,6 +5546,16 @@ snmp_framework_mib_3(Config) when is_list(Config) -> %% Therefor we must take that into account when we check if the %% Engine Time diff (between the two checks) is acceptably. snmp_framework_mib_test() -> + + ?IPRINT("transports: " + "~n ~p" + "~ninfo: " + "~n ~p", + [ + rpc:call(get(master_node), snmpa, which_transports, []), + rpc:call(get(master_node), snmpa, info, []) + ]), + Sleep = 5, ?line ["agentEngine"] = get_req(1, [[snmpEngineID,0]]), T1 = snmp_misc:now(ms), @@ -6182,81 +6250,129 @@ loop_mib_3(Config) when is_list(Config) -> %% Req. As many mibs all possible loop_mib_1_test() -> - ?DBG("loop_mib_1_test -> entry",[]), + ?IPRINT("loop_mib_1_test -> entry"), N = loop_it_1([1,1], 0), - io:format(user, "found ~w varibles\n", [N]), + ?IPRINT("found ~w varibles", [N]), ?line N = if N < 100 -> 100; true -> N end. loop_it_1(Oid, N) -> - ?DBG("loop_it_1_test -> entry with~n" - "\tOid: ~p~n" - "\tN: ~p",[Oid,N]), + ?IPRINT("loop_it_1_test -> entry with" + "~n Oid: ~p" + "~n N: ~p", [Oid, N]), case get_next_req([Oid]) of #pdu{type = 'get-response', error_status = noError, error_index = 0, varbinds = [#varbind{oid = NOid, value = _Value}]} when NOid > Oid -> - ?DBG("loop_it_1_test -> " - "~n NOid: ~p" - "~n Value: ~p", [NOid, _Value]), + ?IPRINT("loop_it_1_test -> " + "expected intermediate (get-next) result: " + "~n NOid: ~p" + "~n Value: ~p", [NOid, _Value]), ?line [_Value2] = get_req(1, [NOid]), % must not be same - ?DBG("loop_it_1_test -> " - "~n Value2: ~p", [_Value2]), + ?IPRINT("loop_it_1_test -> expected intermediate (get) result: " + "~n Value2: ~p", [_Value2]), loop_it_1(NOid, N+1); #pdu{type = 'get-response', error_status = noError, error_index = 0, varbinds = Vbs} -> - exit({unexpected_vbs, ?LINE, Vbs}); + ?EPRINT("loop_it_1_test -> unexpected (get-response) vbs: " + "~n Vbs: ~p", [Vbs]), + ?line ?FAIL({unexpected_vbs, + [{get_next_oid, Oid}, + {counter, N}, + {varbinds, Vbs}]}); #pdu{type = 'get-response', error_status = noSuchName, error_index = 1, varbinds = [_]} -> - ?DBG("loop_it_1_test -> done: ~p",[N]), + ?IPRINT("loop_it_1_test -> done: ~p", [N]), N; #pdu{type = 'get-response', error_status = Err, error_index = Idx, varbinds = Vbs} -> - exit({unexpected_pdu, ?LINE, Err, Idx, Vbs}); + ?EPRINT("loop_it_1_test -> unexpected (get-response) pdu: " + "~n Err: ~p" + "~n Idx: ~p" + "~n Vbs: ~p", [Err, Idx, Vbs]), + ?line ?FAIL({unexpected_pdu, + [{get_next_oid, Oid}, + {counter, N}, + {error_status, Err}, + {error_index, Idx}, + {varbinds, Vbs}]}); #pdu{type = Type, error_status = Err, error_index = Idx, varbinds = Vbs} -> - exit({unexpected_pdu, ?LINE, Type, Err, Idx, Vbs}); + ?EPRINT("loop_it_1_test -> unexpected pdu: " + "~n Type: ~p" + "~n Err: ~p" + "~n Idx: ~p" + "~n Vbs: ~p", [Type, Err, Idx, Vbs]), + ?line ?FAIL({unexpected_pdu, + [{get_next_oid, Oid}, + {counter, N}, + {type, Type}, + {error_status, Err}, + {error_index, Idx}, + {varbinds, Vbs}]}); {error, Reason} -> - exit({error, Reason, ?LINE}) + %% Regardless of the error here (its usually timeout), + %% if we have had system events we skip since the results + %% in those cases are simply not reliable. + %% There is just no point in trying to analyze the reason. + ?IPRINT("loop_it_1_test -> receive error: " + "~n ~p", [Reason]), + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("loop_it_1_test -> error: " + "~n ~p", [Reason]), + ?line ?FAIL([{get_next_oid, Oid}, + {counter, N}, + {reason, Reason}]); + + true -> + ?WPRINT("loop_it_1_test -> " + "error when we got system events: " + "~n Reason: ~p" + "~n Sys Events: ~p" + "~n", [Reason, SysEvs]), + ?SKIP([{reason, Reason}, {system_events, SysEvs}]) + end end. %% Req. As many mibs all possible loop_mib_2_test() -> - ?DBG("loop_mib_2_test -> entry",[]), + ?IPRINT("loop_mib_2_test -> entry"), N = loop_it_2([1,1], 0), - io:format(user, "found ~w varibles\n", [N]), + ?IPRINT("found ~w varibles", [N]), ?line N = if N < 100 -> 100; true -> N end. loop_it_2(Oid, N) -> - ?DBG("loop_it_2 -> entry with" - "~n Oid: ~p" - "~n N: ~p",[Oid, N]), + ?IPRINT("loop_it_2 -> entry with" + "~n Oid: ~p" + "~n N: ~p", [Oid, N]), case get_next_req([Oid]) of #pdu{type = 'get-response', error_status = noError, error_index = 0, varbinds = [#varbind{oid = _NOid, value = endOfMibView}]} -> - ?DBG("loop_it_2 -> " - "~n NOid: ~p", [_NOid]), + ?IPRINT("loop_it_2 -> done: " + "~n NOid: ~p", [_NOid]), N; #pdu{type = 'get-response', @@ -6264,52 +6380,82 @@ loop_it_2(Oid, N) -> error_index = 0, varbinds = [#varbind{oid = NOid, value = _Value}]} when NOid > Oid -> - ?DBG("loop_it_2 -> " - "~n NOid: ~p" - "~n Value: ~p", [NOid, _Value]), + ?IPRINT("loop_it_2 -> " + "expected intermediate (get-next) result: " + "~n NOid: ~p" + "~n Value: ~p", [NOid, _Value]), ?line [_Value2] = get_req(1, [NOid]), % must not be same - ?DBG("loop_it_2 -> " - "~n Value2: ~p", [_Value2]), + ?IPRINT("loop_it_2 -> expected intermediate (get) result: " + "~n Value2: ~p", [_Value2]), loop_it_2(NOid, N+1); #pdu{type = 'get-response', error_status = noError, error_index = 0, varbinds = Vbs} -> - exit({unexpected_pdu, ?LINE, - [{varbinds, Vbs}, - {get_next_oid, Oid}, - {counter, N}]}); + ?EPRINT("loop_it_2 -> unexpected (get-response) vbs: " + "~n Vbs: ~p", [Vbs]), + ?line ?FAIL({unexpected_vbs, + [{get_next_oid, Oid}, + {counter, N}, + {varbinds, Vbs}]}); #pdu{type = 'get-response', error_status = ES, error_index = EI, varbinds = Vbs} -> - exit({unexpected_pdu, ?LINE, - [{error_status, ES}, - {error_index, EI}, - {varbinds, Vbs}, - {get_next_oid, Oid}, - {counter, N}]}); + ?EPRINT("loop_it_2 -> unexpected (get-response) pdu: " + "~n ES: ~p" + "~n EI: ~p" + "~n Vbs: ~p", [ES, EI, Vbs]), + ?line ?FAIL({unexpected_pdu, + [{get_next_oid, Oid}, + {counter, N}, + {error_status, ES}, + {error_index, EI}, + {varbinds, Vbs}]}); #pdu{type = Type, error_status = ES, error_index = EI, varbinds = Vbs} -> - exit({unexpected_pdu, ?LINE, - [{type, Type}, - {error_status, ES}, - {error_index, EI}, - {varbinds, Vbs}, - {get_next_oid, Oid}, - {counter, N}]}); + ?EPRINT("loop_it_2 -> unexpected pdu: " + "~n Type: ~p" + "~n ES: ~p" + "~n EI: ~p" + "~n Vbs: ~p", [Type, ES, EI, Vbs]), + ?line ?FAIL({unexpected_pdu, + [{get_next_oid, Oid}, + {counter, N}, + {type, Type}, + {error_status, ES}, + {error_index, EI}, + {varbinds, Vbs}]}); {error, Reason} -> - exit({unexpected_result, ?LINE, - [{reason, Reason}, - {get_next_oid, Oid}, - {counter, N}]}) - + %% Regardless of the error here (its usually timeout), + %% if we have had system events we skip since the results + %% in those cases are simply not reliable. + %% There is just no point in trying to analyze the reason. + ?IPRINT("loop_it_2 -> receive error: " + "~n ~p", [Reason]), + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("loop_it_2 -> error: " + "~n ~p", [Reason]), + ?line ?FAIL([{get_next_oid, Oid}, + {counter, N}, + {reason, Reason}]); + + true -> + ?WPRINT("loop_it_2 -> " + "error when we got system events: " + "~n Reason: ~p" + "~n Sys Events: ~p" + "~n", [Reason, SysEvs]), + ?SKIP([{reason, Reason}, {system_events, SysEvs}]) + end end. loop_mib_3_test() -> @@ -7930,22 +8076,29 @@ otp16649_validate_transports([], []) -> ok; otp16649_validate_transports([AgentRawTransport|AgentRawTransports], [TI|TIs]) -> + ?IPRINT("validate transport:" + "~n AgentRawTransport: ~p" + "~n TI: ~p", [AgentRawTransport, TI]), otp16649_validate_transport(AgentRawTransport, TI), otp16649_validate_transports(AgentRawTransports, TIs). -otp16649_validate_transport({PortInfo, Kind}, {PortNo, Kind, _}) -> +otp16649_validate_transport({PortInfo, Kind}, #{taddress := {_, PortNo}, + transport_kind := Kind}) -> ?IPRINT("validate ~w transport:" "~n PortNo: ~w" "~n PortInfo: ~p", [Kind, PortNo, PortInfo]), otp16649_validate_port(PortInfo, PortNo); -otp16649_validate_transport({_, ConfKind}, {PortNo, ActualKind, _}) -> +otp16649_validate_transport({_, ConfKind}, #{taddress := {_, PortNo}, + transport_kind := ActualKind}) -> exit({invalid_transport_kind, {PortNo, ConfKind, ActualKind}}); -otp16649_validate_transport({PortInfo, Kind, _}, {PortNo, Kind, _}) -> +otp16649_validate_transport({PortInfo, Kind, _}, #{taddress := {_, PortNo}, + transport_kind := Kind}) -> ?IPRINT("validate ~w transport:" "~n PortNo: ~w" "~n PortInfo: ~p", [Kind, PortNo, PortInfo]), otp16649_validate_port(PortInfo, PortNo); -otp16649_validate_transport({_, ConfKind, _}, {PortNo, ActualKind, _}) -> +otp16649_validate_transport({_, ConfKind, _}, #{taddress := {_, PortNo}, + transport_kind := ActualKind}) -> exit({invalid_transport_kind, {PortNo, ConfKind, ActualKind}}). otp16649_validate_port(PortNo, PortNo) when is_integer(PortNo) -> @@ -8007,7 +8160,8 @@ otp16649_which_trap_port_no(TIs) -> otp16649_which_port_no([], Kind) -> exit({no_transport_port_no, Kind}); -otp16649_which_port_no([{PortNo, Kind, _}|_], Kind) -> +otp16649_which_port_no([#{taddress := {_, PortNo}, + transport_kind := Kind}|_], Kind) -> PortNo; otp16649_which_port_no([_|TIs], Kind) -> otp16649_which_port_no(TIs, Kind). @@ -8675,5 +8829,3 @@ rcall(Node, Mod, Func, Args) -> Else -> Else end. - - diff --git a/lib/snmp/test/snmp_agent_test_lib.erl b/lib/snmp/test/snmp_agent_test_lib.erl index da2762c3fb..96cc6add81 100644 --- a/lib/snmp/test/snmp_agent_test_lib.erl +++ b/lib/snmp/test/snmp_agent_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2020. All Rights Reserved. +%% Copyright Ericsson AB 2005-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -395,7 +395,21 @@ await_tc_runner_done(Runner, OldFlag) -> unlink_and_flush_exit(Runner), case Ret of {error, Reason} -> - exit(Reason); + %% Any failures while we have system events are skipped + SysEvs = snmp_test_global_sys_monitor:events(), + if + (SysEvs =:= []) -> + ?EPRINT("TC failure: " + "~n ~p" + "~n", [Reason]), + exit(Reason); + true -> + ?WPRINT("TC failure when we got system events: " + "~n Reason: ~p" + "~n Sys Events: ~p" + "~n", [Reason, SysEvs]), + skip([{reason, Reason}, {system_events, SysEvs}]) + end; {skip, Reason} -> skip(Reason); OK -> @@ -498,34 +512,22 @@ tc_run(Mod, Func, Args, Opts) -> {dir, Dir}, {mibs, mibs(StdM, M)}]) of {ok, _Pid} -> - case (catch apply(Mod, Func, Args)) of - {'EXIT', {skip, Reason}} -> - ?WPRINT("apply skip detected: " - "~n ~p", [Reason]), - (catch snmp_test_mgr:stop()), - ?SKIP(Reason); - {'EXIT', Reason} -> - %% We have hosts (mostly *very* slooow VMs) that - %% can timeout anything. Since we are basically - %% testing communication, we therefor must check - %% for system events at every failure. Grrr! - SysEvs = snmp_test_global_sys_monitor:events(), - (catch snmp_test_mgr:stop()), - if - (SysEvs =:= []) -> - ?EPRINT("TC runner failed: " - "~n ~p~n", [Reason]), - ?FAIL({apply_failed, {Mod, Func, Args}, Reason}); - true -> - ?WPRINT("apply exit catched when we got system events: " - "~n Reason: ~p" - "~n Sys Events: ~p" - "~n", [Reason, SysEvs]), - ?SKIP([{reason, Reason}, {system_events, SysEvs}]) - end; - Res -> + try apply(Mod, Func, Args) of + Res -> (catch snmp_test_mgr:stop()), Res + catch + C:{skip, Reason} -> + ?WPRINT("apply (~w-) skip detected: " + "~n ~p", [C, Reason]), + (catch snmp_test_mgr:stop()), + ?SKIP(Reason); + + throw:{error, Reason} -> + tc_run_skip_sheck(Mod, Func, Args, Reason, throw); + + exit:Reason -> + tc_run_skip_sheck(Mod, Func, Args, Reason, exit) end; {error, Reason} -> @@ -541,6 +543,28 @@ tc_run(Mod, Func, Args, Opts) -> ?line ?FAIL({mgr_start_failure, Err}) end. +%% We have hosts (mostly *very* slooow VMs) that +%% can timeout anything. Since we are basically +%% testing communication, we therefor must check +%% for system events at every failure. Grrr! +tc_run_skip_sheck(Mod, Func, Args, Reason, Cat) -> + SysEvs = snmp_test_global_sys_monitor:events(), + (catch snmp_test_mgr:stop()), + if + (SysEvs =:= []) -> + ?EPRINT("TC runner (~w-) failed: " + "~n ~p~n", [Cat, Reason]), + ?FAIL({apply_failed, {Mod, Func, Args}, Reason}); + true -> + ?WPRINT("apply (~w) catched " + "when we got system events: " + "~n Reason: ~p" + "~n Sys Events: ~p" + "~n", [Cat, Reason, SysEvs]), + ?SKIP([{category, Cat}, + {reason, Reason}, {system_events, SysEvs}]) + end. + %% --------------------------------------------------------------- %% --- --- diff --git a/lib/snmp/test/snmp_manager_config_SUITE.erl b/lib/snmp/test/snmp_manager_config_SUITE.erl index 9a7b485a60..f7f7fd6928 100644 --- a/lib/snmp/test/snmp_manager_config_SUITE.erl +++ b/lib/snmp/test/snmp_manager_config_SUITE.erl @@ -2249,6 +2249,7 @@ register_usm_user_using_function(Conf) when is_list(Conf) -> {auth, usmHMACMD5AuthProtocol}, {auth_key, [1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6]}, {priv, usmNoPrivProtocol}], + ?line ok = snmpm_config:register_usm_user(EngineID, UserName1, UsmConfig1), ?IPRINT("try register user 1 again (error)"), ?line {error, {already_registered, EngineID, UserName1}} = @@ -2270,7 +2271,17 @@ register_usm_user_using_function(Conf) when is_list(Conf) -> {auth_key, [1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6]}, {priv, usmNoPrivProtocol}], ?line ok = snmpm_config:register_usm_user(EngineID, UserName3, UsmConfig3), - + + ?IPRINT("register user 4 (ok)"), + UserName4 = "samu4", + SecName4 = "samu_auth4", + UsmConfig4 = [{sec_name, SecName4}, + {auth, usmHMACMD5AuthProtocol}, + {auth_key, [1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6]}, + {priv, usmAesCfb128Protocol}, + {priv_key, [190,54,66,227,33,171,152,0,133,223,204,155,109,111,77,44]}], + ?line ok = snmpm_config:register_usm_user(EngineID, UserName4, UsmConfig4), + ?IPRINT("lookup 1 (ok)"), ?line {ok, #usm_user{name = UserName1} = User1} = snmpm_config:get_usm_user_from_sec_name(EngineID, SecName1), @@ -2286,9 +2297,14 @@ register_usm_user_using_function(Conf) when is_list(Conf) -> snmpm_config:get_usm_user_from_sec_name(EngineID, SecName3), ?IPRINT("User: ~p", [User3]), - ?IPRINT("lookup 4 (error)"), + ?IPRINT("lookup 4 (ok)"), + ?line {ok, #usm_user{name = UserName4} = User4} = + snmpm_config:get_usm_user_from_sec_name(EngineID, SecName4), + ?IPRINT("User: ~p", [User4]), + + ?IPRINT("lookup 5 (error)"), ?line {error, not_found} = - snmpm_config:get_usm_user_from_sec_name(EngineID, SecName3 ++ "_1"), + snmpm_config:get_usm_user_from_sec_name(EngineID, SecName4 ++ "_1"), %% -- ?IPRINT("stop config process"), @@ -2341,6 +2357,7 @@ update_usm_user_info(Conf) when is_list(Conf) -> ?IPRINT("start"), process_flag(trap_exit, true), + ?IPRINT("Start crypto and ensure support"), case ?CRYPTO_START() of ok -> case ?CRYPTO_SUPPORT() of @@ -2353,9 +2370,62 @@ update_usm_user_info(Conf) when is_list(Conf) -> ?SKIP({failed_starting_crypto, Reason}) end, - _ConfDir = ?config(manager_conf_dir, Conf), - _DbDir = ?config(manager_db_dir, Conf), - ?SKIP(not_yet_implemented). + ConfDir = ?config(manager_conf_dir, Conf), + DbDir = ?config(manager_db_dir, Conf), + + ?IPRINT("write manager config"), + write_manager_conf(ConfDir), + + Opts = [{versions, [v3]}, + {config, [{verbosity, trace}, {dir, ConfDir}, {db_dir, DbDir}]}], + + ?IPRINT("Start config server"), + ?line {ok, _Pid} = snmpm_config:start_link(Opts), + + ?IPRINT("Register usm user"), + EngineID = "engine", + UsmUser = "UsmUser", + SecName = UsmUser, + AuthProto = usmHMACMD5AuthProtocol, + AuthKey = [1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6], + PrivProto1 = usmNoPrivProtocol, + UsmConfig = [{sec_name, SecName}, + {auth, AuthProto}, + {auth_key, AuthKey}, + {priv, PrivProto1}], + ok = snmpm_config:register_usm_user(EngineID, UsmUser, UsmConfig), + + ?IPRINT("verify user user config"), + ?line {ok, AuthProto} = snmpm_config:usm_user_info(EngineID, UsmUser, auth), + ?line {ok, AuthKey} = snmpm_config:usm_user_info(EngineID, UsmUser, auth_key), + ?line {ok, PrivProto1} = snmpm_config:usm_user_info(EngineID, UsmUser, priv), + + ?IPRINT("usm user update 1"), + PrivProto2 = usmAesCfb128Protocol, + PrivKey2 = [190,54,66,227,33,171,152,0,133,223,204,155,109,111,77,44], + ok = snmpm_config:update_usm_user_info(EngineID, UsmUser, priv, PrivProto2), + ok = snmpm_config:update_usm_user_info(EngineID, UsmUser, priv_key, PrivKey2), + + ?IPRINT("verify updated user user config after update 1"), + ?line {ok, AuthProto} = snmpm_config:usm_user_info(EngineID, UsmUser, auth), + ?line {ok, AuthKey} = snmpm_config:usm_user_info(EngineID, UsmUser, auth_key), + ?line {ok, PrivProto2} = snmpm_config:usm_user_info(EngineID, UsmUser, priv), + ?line {ok, PrivKey2} = snmpm_config:usm_user_info(EngineID, UsmUser, priv_key), + + ?IPRINT("usm user update 2"), + PrivProto3 = PrivProto1, + ok = snmpm_config:update_usm_user_info(EngineID, UsmUser, priv, PrivProto3), + + ?IPRINT("verify updated user user config after update 2"), + ?line {ok, AuthProto} = snmpm_config:usm_user_info(EngineID, UsmUser, auth), + ?line {ok, AuthKey} = snmpm_config:usm_user_info(EngineID, UsmUser, auth_key), + ?line {ok, PrivProto3} = snmpm_config:usm_user_info(EngineID, UsmUser, priv), + + ?IPRINT("Stop config server"), + ?line ok = snmpm_config:stop(), + + ?IPRINT("done"), + ok. %% diff --git a/lib/snmp/test/snmp_test_global_sys_monitor.erl b/lib/snmp/test/snmp_test_global_sys_monitor.erl index 54cc7d588e..c3f2e24096 100644 --- a/lib/snmp/test/snmp_test_global_sys_monitor.erl +++ b/lib/snmp/test/snmp_test_global_sys_monitor.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% Copyright Ericsson AB 2019-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,7 +28,8 @@ -include("snmp_test_lib.hrl"). --define(NAME, ?MODULE). +-define(NAME, ?MODULE). +-define(TIMEOUT, timer:seconds(6)). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -43,10 +44,10 @@ stop() -> %% This does not reset the global counter but the "collector" %% See events for more info. reset_events() -> - call(reset_events). + call(reset_events, ?TIMEOUT). events() -> - call(events). + call(events, ?TIMEOUT). log(Event) -> cast({node(), Event}). @@ -198,23 +199,66 @@ cast(Msg) -> {error, {catched, C, E}} end. -call(Req) -> - call(Req, infinity). - -call(Req, Timeout) -> - Ref = make_ref(), - try global:send(?NAME, {?MODULE, Ref, self(), Req}) of - Pid when is_pid(Pid) -> - receive - {?MODULE, Ref, Rep} -> - Rep - after Timeout -> - {error, timeout} - end - catch - C:E:_ -> - {error, {catched, C, E}} +%% call(Req) -> +%% call(Req, infinity). + +%% call(Req, Timeout) -> +%% Ref = make_ref(), +%% try global:send(?NAME, {?MODULE, Ref, self(), Req}) of +%% Pid when is_pid(Pid) -> +%% receive +%% {?MODULE, Ref, Rep} -> +%% Rep +%% after Timeout -> +%% {error, timeout} +%% end +%% catch +%% C:E:_ -> +%% {error, {catched, C, E}} +%% end. + +call(Req, Timeout) when (Timeout =:= infinity) -> + call(Req, Timeout, Timeout); +call(Req, Timeout) when is_integer(Timeout) andalso (Timeout > 2000) -> + call(Req, Timeout, Timeout - 1000); +call(Req, Timeout) when is_integer(Timeout) andalso (Timeout > 1000) -> + call(Req, Timeout, Timeout - 500); +call(Req, Timeout) when is_integer(Timeout) -> + call(Req, Timeout, Timeout div 2). + +%% This peace of wierdness is because on some machines this call has +%% hung (in a call during end_per_testcase, which had a 1 min timeout, +%% or if that was the total time for the test case). +%% But because it hung there, we don't really know what where it git stuck. +%% So, by making the call in a tmp process, that we supervise, we can +%% keep control. Also, we change the default timeout from infinity to an +%% actual time (16 seconds). +call(Req, Timeout1, Timeout2) -> + F = fun() -> + Ref = make_ref(), + try global:send(?NAME, {?MODULE, Ref, self(), Req}) of + NamePid when is_pid(NamePid) -> + receive + {?MODULE, Ref, Rep} -> + Rep + after Timeout2 -> + {error, timeout} + end + catch + C:E:_ -> + {error, {catched, C, E}} + end + end, + {Pid, Mon} = spawn_monitor(F), + receive + {'DOWN', Mon, process, Pid, Result} -> + Result + after Timeout1 -> + PInfo = process_info(Pid), + exit(Pid, kill), + {error, {timeout, PInfo}} end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/snmp/test/snmp_test_mgr.erl b/lib/snmp/test/snmp_test_mgr.erl index d7db3a2b0d..fe2852c573 100644 --- a/lib/snmp/test/snmp_test_mgr.erl +++ b/lib/snmp/test/snmp_test_mgr.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2020. All Rights Reserved. +%% Copyright Ericsson AB 1996-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -168,6 +168,7 @@ get_timeout() -> get_timeout(_) -> 10000. % Trying to improve test results % 3500. + %%---------------------------------------------------------------------- %% Receives a trap from the agent. %% Returns: TrapPdu|{error, Reason} diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 7ccaf67aa6..e1e018878a 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.7 +SNMP_VSN = 5.7.3 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index ca4c9a1145..57a1d5f9c7 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,22 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.10.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The SSH daemon erroneously replaced LF with CRLF also + when there was no pty requested from the server.</p> + <p> + Own Id: OTP-17108 Aux Id: ERL-1442 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.10.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 10b1d4aec7..33f6833830 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -504,7 +504,8 @@ recv_ext_info }). --record(ssh_pty, {term = "", % e.g. "xterm" +-record(ssh_pty, {c_version = "", % client version string, e.g "SSH-2.0-Erlang/4.10.5" + term = "", % e.g. "xterm" width = 80, height = 25, pixel_width = 1024, diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 653c65d949..13a44beea3 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -480,23 +480,29 @@ get_tty_command(left, N, _TerminalType) -> %% convert input characters to buffer and to writeout %% Note that the buf is reversed but the buftail is not %% (this is handy; the head is always next to the cursor) -conv_buf([], AccBuf, AccBufTail, AccWrite, Col) -> +conv_buf([], AccBuf, AccBufTail, AccWrite, Col, _Tty) -> {AccBuf, AccBufTail, lists:reverse(AccWrite), Col}; -conv_buf([13, 10 | Rest], _AccBuf, AccBufTail, AccWrite, _Col) -> - conv_buf(Rest, [], tl2(AccBufTail), [10, 13 | AccWrite], 0); -conv_buf([13 | Rest], _AccBuf, AccBufTail, AccWrite, _Col) -> - conv_buf(Rest, [], tl1(AccBufTail), [13 | AccWrite], 0); -conv_buf([10 | Rest], _AccBuf, AccBufTail, AccWrite, _Col) -> - conv_buf(Rest, [], tl1(AccBufTail), [10, 13 | AccWrite], 0); -conv_buf([C | Rest], AccBuf, AccBufTail, AccWrite, Col) -> - conv_buf(Rest, [C | AccBuf], tl1(AccBufTail), [C | AccWrite], Col + 1). +conv_buf([13, 10 | Rest], _AccBuf, AccBufTail, AccWrite, _Col, Tty) -> + conv_buf(Rest, [], tl2(AccBufTail), [10, 13 | AccWrite], 0, Tty); +conv_buf([13 | Rest], _AccBuf, AccBufTail, AccWrite, _Col, Tty) -> + conv_buf(Rest, [], tl1(AccBufTail), [13 | AccWrite], 0, Tty); +conv_buf([10 | Rest], _AccBuf, AccBufTail, AccWrite0, _Col, Tty) -> + AccWrite = + case pty_opt(onlcr,Tty) of + 0 -> [10 | AccWrite0]; + 1 -> [10,13 | AccWrite0]; + undefined -> [10 | AccWrite0] + end, + conv_buf(Rest, [], tl1(AccBufTail), AccWrite, 0, Tty); +conv_buf([C | Rest], AccBuf, AccBufTail, AccWrite, Col, Tty) -> + conv_buf(Rest, [C | AccBuf], tl1(AccBufTail), [C | AccWrite], Col + 1, Tty). %%% put characters at current position (possibly overwriting %%% characters after current position in buffer) -put_chars(Chars, {Buf, BufTail, Col}, _Tty) -> +put_chars(Chars, {Buf, BufTail, Col}, Tty) -> {NewBuf, NewBufTail, WriteBuf, NewCol} = - conv_buf(Chars, Buf, BufTail, [], Col), + conv_buf(Chars, Buf, BufTail, [], Col, Tty), {WriteBuf, {NewBuf, NewBufTail, NewCol}}. %%% insert character at current position @@ -504,7 +510,7 @@ insert_chars([], {Buf, BufTail, Col}, _Tty) -> {[], {Buf, BufTail, Col}}; insert_chars(Chars, {Buf, BufTail, Col}, Tty) -> {NewBuf, _NewBufTail, WriteBuf, NewCol} = - conv_buf(Chars, Buf, [], [], Col), + conv_buf(Chars, Buf, [], [], Col, Tty), M = move_cursor(special_at_width(NewCol+length(BufTail), Tty), NewCol, Tty), {[WriteBuf, BufTail | M], {NewBuf, BufTail, NewCol}}. @@ -776,14 +782,11 @@ t2str(T) -> try io_lib:format("~s",[T]) %%-------------------------------------------------------------------- % Pty can be undefined if the client never sets any pty options before % starting the shell. -get_echo(undefined) -> - true; -get_echo(#ssh_pty{modes = Modes}) -> - case proplists:get_value(echo, Modes, 1) of - 0 -> - false; - _ -> - true +get_echo(Tty) -> + case pty_opt(echo,Tty) of + 0 -> false; + 1 -> true; + undefined -> true end. % Group is undefined if the pty options are sent between open and @@ -799,6 +802,14 @@ not_zero(0, B) -> not_zero(A, _) -> A. +%%%---------------------------------------------------------------- +pty_opt(Name, Tty) -> + try + proplists:get_value(Name, Tty#ssh_pty.modes, undefined) + catch + _:_ -> undefined + end. + %%%################################################################ %%%# %%%# Tracing diff --git a/lib/ssh/src/ssh_connect.hrl b/lib/ssh/src/ssh_connect.hrl index 00d1320a78..e00e78d6b7 100644 --- a/lib/ssh/src/ssh_connect.hrl +++ b/lib/ssh/src/ssh_connect.hrl @@ -206,6 +206,7 @@ -define(IXANY,39). %% Any char will restart after stop. -define(IXOFF,40). %% Enable input flow control. -define(IMAXBEL,41). %% Ring bell on input queue full. +-define(IUTF8,42). %% Terminal input and output is assumed to be encoded in UTF-8. -define(ISIG,50). %% Enable signals INTR, QUIT, [D]SUSP. -define(ICANON,51). %% Canonicalize input lines. -define(XCASE,52). %% Enable input and output of uppercase characters by diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index c069ab237e..a966f7bbf1 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -42,7 +42,7 @@ %% Internal SSH application API -export([channel_data/5, - handle_msg/3, + handle_msg/4, handle_stop/1, open_channel/4, @@ -472,7 +472,7 @@ handle_msg(#ssh_msg_channel_open_confirmation{recipient_channel = ChannelId, sender_channel = RemoteId, initial_window_size = WindowSz, maximum_packet_size = PacketSz}, - #connection{channel_cache = Cache} = Connection0, _) -> + #connection{channel_cache = Cache} = Connection0, _, _SSH) -> #channel{remote_id = undefined} = Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), @@ -490,22 +490,22 @@ handle_msg(#ssh_msg_channel_open_failure{recipient_channel = ChannelId, reason = Reason, description = Descr, lang = Lang}, - #connection{channel_cache = Cache} = Connection0, _) -> + #connection{channel_cache = Cache} = Connection0, _, _SSH) -> Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), ssh_client_channel:cache_delete(Cache, ChannelId), reply_msg(Channel, Connection0, {open_error, Reason, Descr, Lang}); -handle_msg(#ssh_msg_channel_success{recipient_channel = ChannelId}, Connection, _) -> +handle_msg(#ssh_msg_channel_success{recipient_channel = ChannelId}, Connection, _, _SSH) -> reply_msg(ChannelId, Connection, success); -handle_msg(#ssh_msg_channel_failure{recipient_channel = ChannelId}, Connection, _) -> +handle_msg(#ssh_msg_channel_failure{recipient_channel = ChannelId}, Connection, _, _SSH) -> reply_msg(ChannelId, Connection, failure); -handle_msg(#ssh_msg_channel_eof{recipient_channel = ChannelId}, Connection, _) -> +handle_msg(#ssh_msg_channel_eof{recipient_channel = ChannelId}, Connection, _, _SSH) -> reply_msg(ChannelId, Connection, {eof, ChannelId}); handle_msg(#ssh_msg_channel_close{recipient_channel = ChannelId}, - #connection{channel_cache = Cache} = Connection0, _) -> + #connection{channel_cache = Cache} = Connection0, _, _SSH) -> case ssh_client_channel:cache_lookup(Cache, ChannelId) of #channel{sent_close = Closed, remote_id = RemoteId, @@ -538,18 +538,18 @@ handle_msg(#ssh_msg_channel_close{recipient_channel = ChannelId}, handle_msg(#ssh_msg_channel_data{recipient_channel = ChannelId, data = Data}, - Connection, _) -> + Connection, _, _SSH) -> channel_data_reply_msg(ChannelId, Connection, 0, Data); handle_msg(#ssh_msg_channel_extended_data{recipient_channel = ChannelId, data_type_code = DataType, data = Data}, - Connection, _) -> + Connection, _, _SSH) -> channel_data_reply_msg(ChannelId, Connection, DataType, Data); handle_msg(#ssh_msg_channel_window_adjust{recipient_channel = ChannelId, bytes_to_add = Add}, - #connection{channel_cache = Cache} = Connection, _) -> + #connection{channel_cache = Cache} = Connection, _, _SSH) -> #channel{send_window_size = Size, remote_id = RemoteId} = Channel0 = ssh_client_channel:cache_lookup(Cache, ChannelId), @@ -568,7 +568,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session" = Type, initial_window_size = WindowSz, maximum_packet_size = PacketSz}, #connection{options = SSHopts} = Connection0, - server) -> + server, _SSH) -> MinAcceptedPackSz = ?GET_OPT(minimal_remote_max_packet_size, SSHopts), @@ -606,7 +606,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "forwarded-tcpip", options = Options, sub_system_supervisor = SubSysSup } = C, - client) -> + client, _SSH) -> {ReplyMsg, NextChId} = case ssh_connection_handler:retrieve(C, {tcpip_forward,ConnectedHost,ConnectedPort}) of {ok, {ConnectToHost,ConnectToPort}} -> @@ -664,7 +664,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "direct-tcpip", options = Options, sub_system_supervisor = SubSysSup } = C, - server) -> + server, _SSH) -> {ReplyMsg, NextChId} = case ?GET_OPT(tcpip_tunnel_in, Options) of %% May add more to the option, like allowed ip/port pairs to connect to @@ -714,7 +714,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "direct-tcpip", handle_msg(#ssh_msg_channel_open{channel_type = "session", sender_channel = RemoteId}, Connection, - client) -> + client, _SSH) -> %% Client implementations SHOULD reject any session channel open %% requests to make it more difficult for a corrupt server to attack the %% client. See See RFC 4254 6.1. @@ -723,7 +723,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session", "Connection refused", "en"), {[{connection_reply, FailMsg}], Connection}; -handle_msg(#ssh_msg_channel_open{sender_channel = RemoteId}, Connection, _) -> +handle_msg(#ssh_msg_channel_open{sender_channel = RemoteId}, Connection, _, _SSH) -> FailMsg = channel_open_failure_msg(RemoteId, ?SSH_OPEN_ADMINISTRATIVELY_PROHIBITED, "Not allowed", "en"), @@ -732,7 +732,7 @@ handle_msg(#ssh_msg_channel_open{sender_channel = RemoteId}, Connection, _) -> handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exit-status", data = Data}, - Connection, _) -> + Connection, _, _SSH) -> <<?UINT32(Status)>> = Data, reply_msg(ChannelId, Connection, {exit_status, ChannelId, Status}); @@ -740,7 +740,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exit-signal", want_reply = false, data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> + #connection{channel_cache = Cache} = Connection0, _, _SSH) -> <<?DEC_BIN(SigName, _SigLen), ?BOOLEAN(_Core), ?DEC_BIN(Err, _ErrLen), @@ -759,7 +759,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "xon-xoff", want_reply = false, data = Data}, - Connection, _) -> + Connection, _, _SSH) -> <<?BOOLEAN(CDo)>> = Data, reply_msg(ChannelId, Connection, {xon_xoff, ChannelId, CDo=/= 0}); @@ -767,7 +767,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "window-change", want_reply = false, data = Data}, - Connection0, _) -> + Connection0, _, _SSH) -> <<?UINT32(Width),?UINT32(Height), ?UINT32(PixWidth), ?UINT32(PixHeight)>> = Data, reply_msg(ChannelId, Connection0, {window_change, ChannelId, @@ -777,7 +777,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "signal", data = Data}, - Connection0, _) -> + Connection0, _, _SSH) -> <<?DEC_BIN(SigName, _SigLen)>> = Data, reply_msg(ChannelId, Connection0, {signal, ChannelId, binary_to_list(SigName)}); @@ -786,7 +786,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "subsystem", want_reply = WantReply, data = Data}, - #connection{channel_cache = Cache} = Connection, server) -> + #connection{channel_cache = Cache} = Connection, server, _SSH) -> <<?DEC_BIN(SsName,_SsLen)>> = Data, #channel{remote_id=RemoteId} = Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), @@ -803,7 +803,7 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, {[{connection_reply,Reply}], Connection}; handle_msg(#ssh_msg_channel_request{request_type = "subsystem"}, - Connection, client) -> + Connection, client, _SSH) -> %% The client SHOULD ignore subsystem requests. See RFC 4254 6.5. {[], Connection}; @@ -811,31 +811,48 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "pty-req", want_reply = WantReply, data = Data}, - Connection, server) -> + Connection, server, SSH) -> <<?DEC_BIN(BTermName,_TermLen), ?UINT32(Width),?UINT32(Height), ?UINT32(PixWidth), ?UINT32(PixHeight), Modes/binary>> = Data, TermName = binary_to_list(BTermName), + PtyOpts0 = decode_pty_opts(Modes), + PtyOpts = case SSH#ssh.c_version of + "SSH-2.0-PuTTY"++_ -> + %% If - peer client is PuTTY + %% - it asked for pty + %% - did not tell if LF->CRLF expansion is wanted + %% then + %% - do LF->CRLF expansion + case proplists:get_value(onlcr, PtyOpts0, undefined) of + undefined -> + [{onlcr,1} | PtyOpts0]; + _ -> + PtyOpts0 + end; + _ -> + PtyOpts0 + end, PtyRequest = {TermName, Width, Height, - PixWidth, PixHeight, decode_pty_opts(Modes)}, + PixWidth, PixHeight, PtyOpts}, handle_cli_msg(Connection, ChannelId, {pty, ChannelId, WantReply, PtyRequest}); handle_msg(#ssh_msg_channel_request{request_type = "pty-req"}, - Connection, client) -> + Connection, client, _SSH) -> %% The client SHOULD ignore pty requests. See RFC 4254 6.2. {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "shell", want_reply = WantReply}, - Connection, server) -> + Connection, server, _SSH) -> handle_cli_msg(Connection, ChannelId, {shell, ChannelId, WantReply}); handle_msg(#ssh_msg_channel_request{request_type = "shell"}, - Connection, client) -> + Connection, client, _SSH) -> %% The client SHOULD ignore shell requests. See RFC 4254 6.5. {[], Connection}; @@ -843,13 +860,13 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exec", want_reply = WantReply, data = Data}, - Connection, server) -> + Connection, server, _SSH) -> <<?DEC_BIN(Command, _Len)>> = Data, handle_cli_msg(Connection, ChannelId, {exec, ChannelId, WantReply, binary_to_list(Command)}); handle_msg(#ssh_msg_channel_request{request_type = "exec"}, - Connection, client) -> + Connection, client, _SSH) -> %% The client SHOULD ignore exec requests. See RFC 4254 6.5. {[], Connection}; @@ -857,39 +874,44 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "env", want_reply = WantReply, data = Data}, - Connection, server) -> + Connection, server, _SSH) -> <<?DEC_BIN(Var,_VarLen), ?DEC_BIN(Value,_ValLen)>> = Data, handle_cli_msg(Connection, ChannelId, {env, ChannelId, WantReply, Var, Value}); handle_msg(#ssh_msg_channel_request{request_type = "env"}, - Connection, client) -> + Connection, client, _SSH) -> %% The client SHOULD ignore env requests. {[], Connection}; -handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId}, - #connection{channel_cache = Cache} = Connection, _) -> +handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, + want_reply = WantReply}, + #connection{channel_cache = Cache} = Connection, _, _SSH) -> %% Not a valid request_type. All valid types are handling the %% parameter checking in their own clauses above. %% %% The special ReqType faulty_msg signals that something went %% wrong found during decoding. %% - %% RFC4254 says: - %% "If the request is not recognized or is not - %% supported for the channel, SSH_MSG_CHANNEL_FAILURE is returned." + %% RFC4254 5.4 says: + %% "If 'want reply' is FALSE, no response will be sent to the request. + %% Otherwise, the recipient responds with either + %% SSH_MSG_CHANNEL_SUCCESS, SSH_MSG_CHANNEL_FAILURE, or request-specific + %% continuation messages. If the request is not recognized or is not + %% supported for the channel, SSH_MSG_CHANNEL_FAILURE is returned." + %% case ssh_client_channel:cache_lookup(Cache, ChannelId) of - #channel{remote_id = RemoteId} -> + #channel{remote_id = RemoteId} when WantReply==true -> FailMsg = channel_failure_msg(RemoteId), {[{connection_reply, FailMsg}], Connection}; - undefined -> %% Chanel has been closed + _ -> %% Channel has been closed or no reply is wanted {[], Connection} end; handle_msg(#ssh_msg_global_request{name = <<"tcpip-forward">>, want_reply = WantReply, data = <<?DEC_BIN(ListenAddrStr,_Len),?UINT32(ListenPort)>>}, - #connection{options = Opts} = Connection, server) -> + #connection{options = Opts} = Connection, server, _SSH) -> case ?GET_OPT(tcpip_tunnel_out, Opts) of false -> %% This daemon instance has not enabled tcpip_forwarding @@ -924,7 +946,7 @@ handle_msg(#ssh_msg_global_request{name = <<"tcpip-forward">>, handle_msg(#ssh_msg_global_request{name = _Type, want_reply = WantReply, - data = _Data}, Connection, _Role) -> + data = _Data}, Connection, _Role, _SSH) -> if WantReply == true -> FailMsg = request_failure_msg(), {[{connection_reply, FailMsg}], Connection}; @@ -933,29 +955,29 @@ handle_msg(#ssh_msg_global_request{name = _Type, end; handle_msg(#ssh_msg_request_failure{}, - #connection{requests = [{_, From} | Rest]} = Connection, _) -> + #connection{requests = [{_, From} | Rest]} = Connection, _, _SSH) -> {[{channel_request_reply, From, {failure, <<>>}}], Connection#connection{requests = Rest}}; handle_msg(#ssh_msg_request_failure{}, - #connection{requests = [{_, From,_} | Rest]} = Connection, _) -> + #connection{requests = [{_, From,_} | Rest]} = Connection, _, _SSH) -> {[{channel_request_reply, From, {failure, <<>>}}], Connection#connection{requests = Rest}}; handle_msg(#ssh_msg_request_success{data = Data}, - #connection{requests = [{_, From} | Rest]} = Connection, _) -> + #connection{requests = [{_, From} | Rest]} = Connection, _, _SSH) -> {[{channel_request_reply, From, {success, Data}}], Connection#connection{requests = Rest}}; handle_msg(#ssh_msg_request_success{data = Data}, - #connection{requests = [{_, From, Fun} | Rest]} = Connection0, _) -> + #connection{requests = [{_, From, Fun} | Rest]} = Connection0, _, _SSH) -> Connection = Fun({success,Data}, Connection0), {[{channel_request_reply, From, {success, Data}}], Connection#connection{requests = Rest}}; handle_msg(#ssh_msg_disconnect{code = Code, description = Description}, - Connection, _) -> + Connection, _, _SSH) -> {disconnect, {Code, Description}, handle_stop(Connection)}. @@ -1326,6 +1348,8 @@ encode_pty_opts2([{ixoff,Value} | Opts]) -> [?IXOFF, ?uint32(Value) | encode_pty_opts2(Opts)]; encode_pty_opts2([{imaxbel,Value} | Opts]) -> [?IMAXBEL, ?uint32(Value) | encode_pty_opts2(Opts)]; +encode_pty_opts2([{iutf8,Value} | Opts]) -> + [?IUTF8, ?uint32(Value) | encode_pty_opts2(Opts)]; encode_pty_opts2([{isig,Value} | Opts]) -> [?ISIG, ?uint32(Value) | encode_pty_opts2(Opts)]; encode_pty_opts2([{icanon,Value} | Opts]) -> @@ -1420,6 +1444,7 @@ decode_pty_opts2(<<Code, ?UINT32(Value), Tail/binary>>) -> ?IXANY -> ixany; ?IXOFF -> ixoff; ?IMAXBEL -> imaxbel; + ?IUTF8 -> iutf8; % RFC 8160 ?ISIG -> isig; ?ICANON -> icanon; ?XCASE -> xcase; diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 4e271d737d..a198a95937 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1109,7 +1109,7 @@ handle_event(_, {#ssh_msg_kexinit{},_}, {connected,Role}, D0) -> handle_event(_, #ssh_msg_disconnect{description=Desc} = Msg, StateName, D0) -> {disconnect, _, RepliesCon} = - ssh_connection:handle_msg(Msg, D0#data.connection_state, role(StateName)), + ssh_connection:handle_msg(Msg, D0#data.connection_state, role(StateName), D0#data.ssh_params), {Actions,D} = send_replies(RepliesCon, D0), disconnect_fun("Received disconnect: "++Desc, D), {stop_and_reply, {shutdown,Desc}, Actions, D}; @@ -1129,7 +1129,7 @@ handle_event(internal, {conn_msg,Msg}, StateName, #data{starter = User, event_queue = Qev0} = D0) -> Role = role(StateName), Rengotation = renegotiation(StateName), - try ssh_connection:handle_msg(Msg, Connection0, Role) of + try ssh_connection:handle_msg(Msg, Connection0, Role, D0#data.ssh_params) of {disconnect, Reason0, RepliesConn} -> {Repls, D} = send_replies(RepliesConn, D0), case {Reason0,Role} of diff --git a/lib/ssh/src/ssh_controller.erl b/lib/ssh/src/ssh_controller.erl index 4b8d10c3c1..4b8f5a7e8e 100644 --- a/lib/ssh/src/ssh_controller.erl +++ b/lib/ssh/src/ssh_controller.erl @@ -60,11 +60,13 @@ start_link(Role, RegName) -> %% Internal application API %%==================================================================== +-define(TIMEOUT, 30000). + start_system_subsystem(Controller, Sup, Host, Port, Profile, Options, ChildSpec) -> - gen_server:call(Controller, {start_system_subsystem, Sup, Host, Port, Profile, Options, ChildSpec}). + gen_server:call(Controller, {start_system_subsystem, Sup, Host, Port, Profile, Options, ChildSpec}, ?TIMEOUT). stop_system(Controller, SysSup) -> - gen_server:call(Controller, {stop_system,SysSup}). + gen_server:call(Controller, {stop_system,SysSup}, ?TIMEOUT). %%==================================================================== %% Internal process state diff --git a/lib/ssh/src/ssh_sftpd.erl b/lib/ssh/src/ssh_sftpd.erl index 158ef3d5ae..1c53690ed0 100644 --- a/lib/ssh/src/ssh_sftpd.erl +++ b/lib/ssh/src/ssh_sftpd.erl @@ -32,6 +32,7 @@ -include("ssh_xfer.hrl"). -include("ssh_connect.hrl"). %% For ?DEFAULT_PACKET_SIZE and ?DEFAULT_WINDOW_SIZE + %%-------------------------------------------------------------------- %% External exports -export([subsystem_spec/1]). @@ -453,19 +454,19 @@ get_handle(Handles, BinHandle) -> %%% read_dir/5: read directory, send names, and return new state read_dir(State0 = #state{file_handler = FileMod, max_files = MaxLength, file_state = FS0}, - XF, ReqId, Handle, RelPath, {cache, Files}) -> + XF = #ssh_xfer{cm = _CM, channel = _Channel, vsn = Vsn}, ReqId, Handle, RelPath, {cache, Files}) -> AbsPath = relate_file_name(RelPath, State0), if length(Files) > MaxLength -> {ToSend, NewCache} = lists:split(MaxLength, Files), - {NamesAndAttrs, FS1} = get_attrs(AbsPath, ToSend, FileMod, FS0), + {NamesAndAttrs, FS1} = get_attrs(AbsPath, ToSend, FileMod, FS0, Vsn), ssh_xfer:xf_send_names(XF, ReqId, NamesAndAttrs), Handles = lists:keyreplace(Handle, 1, State0#state.handles, {Handle, directory, {RelPath,{cache, NewCache}}}), State0#state{handles = Handles, file_state = FS1}; true -> - {NamesAndAttrs, FS1} = get_attrs(AbsPath, Files, FileMod, FS0), + {NamesAndAttrs, FS1} = get_attrs(AbsPath, Files, FileMod, FS0, Vsn), ssh_xfer:xf_send_names(XF, ReqId, NamesAndAttrs), Handles = lists:keyreplace(Handle, 1, State0#state.handles, @@ -473,12 +474,12 @@ read_dir(State0 = #state{file_handler = FileMod, max_files = MaxLength, file_sta State0#state{handles = Handles, file_state = FS1} end; read_dir(State0 = #state{file_handler = FileMod, max_files = MaxLength, file_state = FS0}, - XF, ReqId, Handle, RelPath, _Status) -> + XF = #ssh_xfer{cm = _CM, channel = _Channel, vsn = Vsn}, ReqId, Handle, RelPath, _Status) -> AbsPath = relate_file_name(RelPath, State0), {Res, FS1} = FileMod:list_dir(AbsPath, FS0), case Res of {ok, Files} when MaxLength == 0 orelse MaxLength > length(Files) -> - {NamesAndAttrs, FS2} = get_attrs(AbsPath, Files, FileMod, FS1), + {NamesAndAttrs, FS2} = get_attrs(AbsPath, Files, FileMod, FS1, Vsn), ssh_xfer:xf_send_names(XF, ReqId, NamesAndAttrs), Handles = lists:keyreplace(Handle, 1, State0#state.handles, @@ -486,7 +487,7 @@ read_dir(State0 = #state{file_handler = FileMod, max_files = MaxLength, file_sta State0#state{handles = Handles, file_state = FS2}; {ok, Files} -> {ToSend, Cache} = lists:split(MaxLength, Files), - {NamesAndAttrs, FS2} = get_attrs(AbsPath, ToSend, FileMod, FS1), + {NamesAndAttrs, FS2} = get_attrs(AbsPath, ToSend, FileMod, FS1, Vsn), ssh_xfer:xf_send_names(XF, ReqId, NamesAndAttrs), Handles = lists:keyreplace(Handle, 1, State0#state.handles, @@ -497,21 +498,74 @@ read_dir(State0 = #state{file_handler = FileMod, max_files = MaxLength, file_sta send_status({error, Error}, ReqId, State1) end. +type_to_string(regular) -> "-"; +type_to_string(directory) -> "d"; +type_to_string(symlink) -> "s"; +type_to_string(device) -> "?"; +type_to_string(undefined) -> "?"; +type_to_string(other) -> "?". + +%% Converts a numeric mode to its human-readable representation +mode_to_string(Mode) -> + mode_to_string(Mode, "xwrxwrxwr", []). +mode_to_string(Mode, [C|T], Acc) when Mode band 1 =:= 1 -> + mode_to_string(Mode bsr 1, T, [C|Acc]); +mode_to_string(Mode, [_|T], Acc) -> + mode_to_string(Mode bsr 1, T, [$-|Acc]); +mode_to_string(_, [], Acc) -> + Acc. + +%% Converts a POSIX time to a readable string +time_to_string({{Y, Mon, Day}, {H, Min, _}}) -> + io_lib:format("~s ~2w ~s:~s ~w", [month(Mon), Day, two_d(H), two_d(Min), Y]). + +two_d(N) -> + tl(integer_to_list(N + 100)). + +month(1) -> "Jan"; +month(2) -> "Feb"; +month(3) -> "Mar"; +month(4) -> "Apr"; +month(5) -> "May"; +month(6) -> "Jun"; +month(7) -> "Jul"; +month(8) -> "Aug"; +month(9) -> "Sep"; +month(10) -> "Oct"; +month(11) -> "Nov"; +month(12) -> "Dec". + +longame({Name, Type, Size, Mtime, Mode, Uid, Gid}) -> + io_lib:format("~s~s ~4w/~-4w ~7w ~s ~s\n", + [type_to_string(Type), mode_to_string(Mode), + Uid, Gid, Size, time_to_string(Mtime), Name]). + +%%% get_long_name: get file longname (version 3) +%%% format output : -rwxr-xr-x 1 uid/gid 348911 Mar 25 14:29 t-filexfer +get_long_name(FileName, I) when is_record(I, file_info) -> + longame({FileName, I#file_info.type, I#file_info.size, I#file_info.mtime, + I#file_info.mode, I#file_info.uid, I#file_info.gid}). %%% get_attrs: get stat of each file and return -get_attrs(RelPath, Files, FileMod, FS) -> - get_attrs(RelPath, Files, FileMod, FS, []). +get_attrs(RelPath, Files, FileMod, FS, Vsn) -> + get_attrs(RelPath, Files, FileMod, FS, Vsn, []). -get_attrs(_RelPath, [], _FileMod, FS, Acc) -> +get_attrs(_RelPath, [], _FileMod, FS, _Vsn, Acc) -> {lists:reverse(Acc), FS}; -get_attrs(RelPath, [F | Rest], FileMod, FS0, Acc) -> +get_attrs(RelPath, [F | Rest], FileMod, FS0, Vsn, Acc) -> Path = filename:absname(F, RelPath), case FileMod:read_link_info(Path, FS0) of {{ok, Info}, FS1} -> + Name = if Vsn =< 3 -> + LongName = get_long_name(F, Info), + {F, LongName}; + true -> + F + end, Attrs = ssh_sftp:info_to_attr(Info), - get_attrs(RelPath, Rest, FileMod, FS1, [{F, Attrs} | Acc]); + get_attrs(RelPath, Rest, FileMod, FS1, Vsn, [{Name, Attrs} | Acc]); {{error, enoent}, FS1} -> - get_attrs(RelPath, Rest, FileMod, FS1, Acc); + get_attrs(RelPath, Rest, FileMod, FS1, Vsn, Acc); {Error, FS1} -> {Error, FS1} end. diff --git a/lib/ssh/src/ssh_xfer.erl b/lib/ssh/src/ssh_xfer.erl index e4d97b7393..d4fa0ccd9f 100644 --- a/lib/ssh/src/ssh_xfer.erl +++ b/lib/ssh/src/ssh_xfer.erl @@ -809,6 +809,15 @@ decode_names(Vsn, I, <<?UINT32(Len), FileName:Len/binary, encode_names(Vsn, NamesAndAttrs) -> lists:mapfoldl(fun(N, L) -> encode_name(Vsn, N, L) end, 0, NamesAndAttrs). +encode_name(Vsn, {{NameUC,LongNameUC},Attr}, Len) when Vsn =< 3 -> + Name = binary_to_list(unicode:characters_to_binary(NameUC)), + NLen = length(Name), + LongName = binary_to_list(unicode:characters_to_binary(LongNameUC)), + LNLen = length(LongName), + EncAttr = encode_ATTR(Vsn, Attr), + ALen = size(EncAttr), + NewLen = Len + NLen + LNLen + 4 + 4 + ALen, + {[<<?UINT32(NLen)>>, Name, <<?UINT32(LNLen)>>, LongName, EncAttr], NewLen}; encode_name(Vsn, {NameUC,Attr}, Len) when Vsn =< 3 -> Name = binary_to_list(unicode:characters_to_binary(NameUC)), NLen = length(Name), diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index 66a79c8a17..4bfc23d5ff 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -287,7 +287,7 @@ client_loop() -> client_loop() end. -do(Pid, Fun) -> do(Pid, Fun, 30?sec). +do(Pid, Fun) -> do(Pid, Fun, 60?sec). do(Pid, Fun, Timeout) when is_function(Fun,0) -> Pid ! {please_do,Fun,Ref=make_ref(),self()}, @@ -418,7 +418,7 @@ ssh_send(C=#chan{conn_ref=ConnectionRef, ref=ChannelRef, client_pid=Pid}, Type, ok -> receive {ssh_cm,ConnectionRef,{data,ChannelRef,Type,Answer}} -> Answer - after 15?sec -> + after 30?sec -> %% receive %% Other -> {error,{unexpected,Other}} %% after 0 -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 5c6798bbcb..2bd64a9720 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -631,27 +631,7 @@ cli_exit_normal(Config) when is_list(Config) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh_connection:shell(ConnectionRef, ChannelId), - - receive - {ssh_cm, ConnectionRef,{eof, ChannelId}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end, - - receive - {ssh_cm, ConnectionRef,{exit_status,ChannelId,0}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end, - - receive - {ssh_cm, ConnectionRef,{closed, ChannelId}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end. + ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId, _ExpectedExitStatus = 0). %%--------------------------------------------------------- %%% Test that SSH client receives user provided exit-status @@ -659,10 +639,13 @@ cli_exit_status(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), UserDir = proplists:get_value(priv_dir, Config), + NonZeroExitStatus = 7, {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir},{user_dir, UserDir}, {password, "morot"}, - {ssh_cli, {ssh_cli, [fun (_) -> spawn(fun () -> exit({exit_status, 7}) end) end]}}, + {ssh_cli, {ssh_cli, [fun (_) -> + spawn(fun () -> exit({exit_status, NonZeroExitStatus}) end) + end]}}, {subsystems, []}, {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), @@ -675,27 +658,7 @@ cli_exit_status(Config) when is_list(Config) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh_connection:shell(ConnectionRef, ChannelId), - - receive - {ssh_cm, ConnectionRef,{eof, ChannelId}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end, - - receive - {ssh_cm, ConnectionRef,{exit_status,ChannelId,7}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end, - - receive - {ssh_cm, ConnectionRef,{closed, ChannelId}} -> - ok - after - 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end. + ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId, NonZeroExitStatus). %%-------------------------------------------------------------------- %%% Test that get correct error message if you try to start a daemon diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index e4cbe8045f..49168d38bb 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -73,6 +73,7 @@ start_exec_direct_fun1_read_write/1, start_exec_direct_fun1_read_write_advanced/1, start_shell/1, + start_shell_pty/1, start_shell_exec/1, start_shell_exec_direct_fun/1, start_shell_exec_direct_fun1_error/1, @@ -86,7 +87,8 @@ start_shell_sock_daemon_exec_multi/1, start_shell_sock_exec_fun/1, start_subsystem_on_closed_channel/1, - stop_listener/1 + stop_listener/1, + ssh_exec_echo/2 % called as an MFA ]). -define(SSH_DEFAULT_PORT, 22). @@ -112,6 +114,7 @@ all() -> exec_disabled, exec_shell_disabled, start_shell, + start_shell_pty, start_shell_exec, start_shell_exec_fun, start_shell_exec_fun2, @@ -578,7 +581,28 @@ start_shell(Config) when is_list(Config) -> {password, "morot"}, {user_interaction, true}, {user_dir, UserDir}]), - test_shell_is_enabled(ConnectionRef, <<"Enter command\r\n">>), + test_shell_is_enabled(ConnectionRef, <<"Enter command">>), % No pty alloc by erl client + test_exec_is_disabled(ConnectionRef), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%------------------------------------------------------------------- +start_shell_pty(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(U, H) -> start_our_shell(U, H) end} ]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + test_shell_is_enabled(ConnectionRef, <<"Enter command\r\n">>, [{pty_opts,[{onlcr,1}]}]), % alloc pty test_exec_is_disabled(ConnectionRef), ssh:close(ConnectionRef), ssh:stop_daemon(Pid). @@ -692,42 +716,54 @@ exec_shell_disabled(Config) when is_list(Config) -> %%-------------------------------------------------------------------- start_shell_exec_fun(Config) -> - do_start_shell_exec_fun(fun ssh_exec_echo/1, - "testing", <<"echo testing\r\n">>, 0, + do_start_shell_exec_fun(fun(Cmd) -> + spawn(fun() -> + io:format("echo ~s\n", [Cmd]) + end) + end, + "testing", <<"echo testing\n">>, 0, Config). start_shell_exec_fun2(Config) -> - do_start_shell_exec_fun(fun ssh_exec_echo/2, - "testing", <<"echo foo testing\r\n">>, 0, + do_start_shell_exec_fun(fun(Cmd, User) -> + spawn(fun() -> + io:format("echo ~s ~s\n",[User,Cmd]) + end) + end, + "testing", <<"echo foo testing\n">>, 0, Config). start_shell_exec_fun3(Config) -> - do_start_shell_exec_fun(fun ssh_exec_echo/3, - "testing", <<"echo foo testing\r\n">>, 0, + do_start_shell_exec_fun(fun(Cmd, User, _PeerAddr) -> + spawn(fun() -> + io:format("echo ~s ~s\n",[User,Cmd]) + end) + end, + "testing", <<"echo foo testing\n">>, 0, Config). start_shell_exec_direct_fun(Config) -> - do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/1}, + do_start_shell_exec_fun({direct, fun(Cmd) -> {ok, io_lib:format("echo ~s~n",[Cmd])} end}, "testing", <<"echo testing\n">>, 0, Config). start_shell_exec_direct_fun2(Config) -> - do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/2}, + do_start_shell_exec_fun({direct, fun(Cmd,User) -> {ok, io_lib:format("echo ~s ~s",[User,Cmd])} end}, "testing", <<"echo foo testing">>, 0, Config). start_shell_exec_direct_fun3(Config) -> - do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/3}, + do_start_shell_exec_fun({direct, fun(Cmd,User,_PeerAddr) -> {ok, io_lib:format("echo ~s ~s",[User,Cmd])} end}, "testing", <<"echo foo testing">>, 0, Config). start_shell_exec_direct_fun1_error(Config) -> - do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo_error_return/1}, + do_start_shell_exec_fun({direct, fun(_Cmd) -> {error, {bad}} end}, "testing", <<"**Error** {bad}">>, 1, Config). start_shell_exec_direct_fun1_error_type(Config) -> - do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo_error_return_type/1}, + do_start_shell_exec_fun({direct, fun(_Cmd) -> very_bad end}, "testing", <<"**Error** Bad exec fun in server. Invalid return value: very_bad">>, 1, Config). @@ -750,11 +786,11 @@ start_exec_direct_fun1_read_write(Config) -> {ok, Ch} = ssh_connection:session_channel(C, infinity), success = ssh_connection:exec(C, Ch, "> ", infinity), - ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"Tiny read/write test\r\n">>}}), + ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"Tiny read/write test\n">>}}), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"1> ">>}}), ok = ssh_connection:send(C, Ch, "hej.\n", 5000), - ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,hej}\r\n">>}}), + ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,hej}\n">>}}), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"2> ">>}}), ok = ssh_connection:send(C, Ch, "quit.\n", 5000), @@ -798,18 +834,18 @@ start_exec_direct_fun1_read_write_advanced(Config) -> {ok, Ch} = ssh_connection:session_channel(C, infinity), success = ssh_connection:exec(C, Ch, "> ", infinity), - ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"Tiny read/write test\r\n">>}}), + ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"Tiny read/write test\n">>}}), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"1> ">>}}), ok = ssh_connection:send(C, Ch, "hej.\n", 5000), - ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,hej}\r\n">>}}), + ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,hej}\n">>}}), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"2> ">>}}), ok = ssh_connection:send(C, Ch, "'Hi ", 5000), ok = ssh_connection:send(C, Ch, "there", 5000), ok = ssh_connection:send(C, Ch, "'", 5000), ok = ssh_connection:send(C, Ch, ".\n", 5000), - ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,'Hi there'}\r\n">>}}), + ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"{simple_eval,'Hi there'}\n">>}}), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,0,<<"3> ">>}}), ok = ssh_connection:send(C, Ch, "bad_input.\n", 5000), ssh_test_lib:receive_exec_result_or_fail({ssh_cm,C,{data,Ch,1,<<"**Error** {bad_input,3}">>}}), @@ -880,7 +916,8 @@ do_start_shell_exec_fun(Fun, Command, Expect, ExpectType, Config) -> after 5000 -> receive Other -> - ct:log("Received other:~n~p",[Other]), + ct:log("Received other:~n~p~nExpected: ~p~n", + [Other, {ssh_cm, ConnectionRef, {data, '_ChannelId', ExpectType, Expect}} ]), ct:fail("Unexpected response") after 0 -> ct:fail("Exec Timeout") @@ -915,7 +952,7 @@ start_shell_sock_exec_fun(Config) when is_list(Config) -> "testing", infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -959,7 +996,7 @@ start_shell_sock_daemon_exec(Config) -> "testing", infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -1018,7 +1055,7 @@ start_shell_sock_daemon_exec_multi(Config) -> success = ssh_connection:exec(ConnectionRef, ChannelId0, "testing", infinity), ct:log("~p:~p: exec on connection ~p", [?MODULE,?LINE,ConnectionRef]), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\n">>}} -> Parent ! {answer_received,self()}, ct:log("~p:~p: recevied result on connection ~p", [?MODULE,?LINE,ConnectionRef]) after 5000 -> @@ -1157,7 +1194,7 @@ stop_listener(Config) when is_list(Config) -> success = ssh_connection:exec(ConnectionRef0, ChannelId0, "testing", infinity), receive - {ssh_cm, ConnectionRef0, {data, ChannelId0, 0, <<"echo testing\r\n">>}} -> + {ssh_cm, ConnectionRef0, {data, ChannelId0, 0, <<"echo testing\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -1368,7 +1405,8 @@ test_shell_is_disabled(ConnectionRef, Expect, NotExpect) -> ct:fail("Could start disabled shell!"); R -> - ct:log("~p:~p Got unexpected ~p",[?MODULE,?LINE,R]), + ct:log("~p:~p Got unexpected ~p~nExpect: ~p~n", + [?MODULE,?LINE,R, {ssh_cm, ConnectionRef, {data, ChannelId, '0|1', Expect}} ]), ct:fail("Strange shell response") after 5000 -> @@ -1383,7 +1421,8 @@ test_exec_is_disabled(ConnectionRef) -> {ssh_cm, ConnectionRef, {data,ChannelId,1,<<"Prohibited.">>}} -> flush_msgs(); R -> - ct:log("~p:~p Got unexpected ~p",[?MODULE,?LINE,R]), + ct:log("~p:~p Got unexpected ~p~nExpect: ~p~n", + [?MODULE,?LINE,R, {ssh_cm, ConnectionRef, {data,ChannelId,1,<<"Prohibited.">>}} ]), ct:fail("Could exec erlang term although non-erlang shell") after 5000 -> ct:fail("Exec Timeout") @@ -1394,15 +1433,26 @@ test_shell_is_enabled(ConnectionRef) -> test_shell_is_enabled(ConnectionRef, <<"Eshell V">>). test_shell_is_enabled(ConnectionRef, Expect) -> + test_shell_is_enabled(ConnectionRef, Expect, []). + +test_shell_is_enabled(ConnectionRef, Expect, PtyOpts) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + case PtyOpts of + [] -> + no_alloc; + _ -> + success = ssh_connection:ptty_alloc(ConnectionRef, ChannelId, PtyOpts) + end, ok = ssh_connection:shell(ConnectionRef,ChannelId), + ExpSz = size(Expect), receive {ssh_cm,ConnectionRef, {data, ChannelId, 0, <<Expect:ExpSz/binary, _/binary>>}} -> flush_msgs(); R -> - ct:log("~p:~p Got unexpected ~p",[?MODULE,?LINE,R]), + ct:log("~p:~p Got unexpected ~p~nExpect: ~p~n", + [?MODULE,?LINE,R, {ssh_cm, ConnectionRef, {data, ChannelId, 0, Expect}} ]), ct:fail("Strange shell response") after 5000 -> @@ -1421,7 +1471,8 @@ test_exec_is_enabled(ConnectionRef, Exec, Expect) -> {ssh_cm, ConnectionRef, {data, ChannelId, 0, <<Expect:ExpSz/binary, _/binary>>}} = R -> ct:log("~p:~p Got expected ~p",[?MODULE,?LINE,R]); Other -> - ct:log("~p:~p Got unexpected ~p",[?MODULE,?LINE,Other]) + ct:log("~p:~p Got unexpected ~p~nExpect: ~p~n", + [?MODULE,?LINE, Other, {ssh_cm, ConnectionRef, {data, ChannelId, 0, Expect}} ]) after 5000 -> {fail,"Exec Timeout"} end. @@ -1502,12 +1553,3 @@ ssh_exec_echo(Cmd, User) -> spawn(fun() -> io:format("echo ~s ~s\n",[User,Cmd]) end). -ssh_exec_echo(Cmd, User, _PeerAddr) -> - ssh_exec_echo(Cmd,User). - -ssh_exec_direct_echo(Cmd) -> {ok, io_lib:format("echo ~s~n",[Cmd])}. -ssh_exec_direct_echo(Cmd, User) -> {ok, io_lib:format("echo ~s ~s",[User,Cmd])}. -ssh_exec_direct_echo(Cmd, User, _PeerAddr) -> ssh_exec_direct_echo(Cmd,User). - -ssh_exec_direct_echo_error_return(_Cmd) -> {error, {bad}}. -ssh_exec_direct_echo_error_return_type(_Cmd) -> very_bad. diff --git a/lib/ssh/test/ssh_echo_server.erl b/lib/ssh/test/ssh_echo_server.erl index e039439f87..0e2519fc84 100644 --- a/lib/ssh/test/ssh_echo_server.erl +++ b/lib/ssh/test/ssh_echo_server.erl @@ -57,6 +57,7 @@ handle_ssh_msg({ssh_cm, CM, {data, ChannelId, 0, Data}}, #state{n = N} = State) case M > 0 of true -> ?DBG(State, "ssh_cm data Cid=~p size(Data)=~p M=~p",[ChannelId,size(Data),M]), + ssh_connection:adjust_window(CM, ChannelId, size(Data)), ssh_connection:send(CM, ChannelId, Data), {ok, State#state{n = M}}; false -> diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 1dbfb0249b..09b1c7ccbb 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -1446,14 +1446,14 @@ try_to_connect(Connect, Host, Port, Pid, Tref, N) -> %%-------------------------------------------------------------------- max_sessions_drops_tcp_connects() -> - [{timetrap,{minutes,5}}]. + [{timetrap,{minutes,20}}]. max_sessions_drops_tcp_connects(Config) -> MaxSessions = 20, UseSessions = 2, % Must be =< MaxSessions FloodSessions = 1000, ParallelLogin = true, - NegTimeOut = 10*1000, + NegTimeOut = 8*1000, HelloTimeOut = 1*1000, %% Start a test daemon @@ -1470,8 +1470,8 @@ max_sessions_drops_tcp_connects(Config) -> {max_sessions, MaxSessions} ]), Host = ssh_test_lib:mangle_connect_address(Host0), - ct:log("~p Listen ~p:~p for max ~p sessions. Mangled Host = ~p", - [Pid,Host0,Port,MaxSessions,Host]), + ct:log("~p:~p ~p Listen ~p:~p for max ~p sessions. Mangled Host = ~p", + [?MODULE,?LINE,Pid,Host0,Port,MaxSessions,Host]), %% Log in UseSessions connections SSHconnect = fun(N) -> @@ -1482,7 +1482,7 @@ max_sessions_drops_tcp_connects(Config) -> {user, "carni"}, {password, "meat"} ]), - ct:log("~p: ssh:connect -> ~p", [N,R]), + ct:log("~p:~p ~p: ssh:connect -> ~p", [?MODULE,?LINE,N,R]), R end, @@ -1491,18 +1491,18 @@ max_sessions_drops_tcp_connects(Config) -> UseSessions -> %% As expected %% Try gen_tcp:connect - [ct:log("~p: gen_tcp:connect -> ~p", - [N, gen_tcp:connect(Host, Port, [])]) + [ct:log("~p:~p ~p: gen_tcp:connect -> ~p", + [?MODULE,?LINE, N, gen_tcp:connect(Host, Port, [])]) || N <- lists:seq(UseSessions+1, MaxSessions) ], - ct:log("Now try ~p gen_tcp:connect to be rejected", [FloodSessions]), - [ct:log("~p: gen_tcp:connect -> ~p", - [N, gen_tcp:connect(Host, Port, [])]) + ct:log("~p:~p Now try ~p gen_tcp:connect to be rejected", [?MODULE,?LINE,FloodSessions]), + [ct:log("~p:~p ~p: gen_tcp:connect -> ~p", + [?MODULE,?LINE, N, gen_tcp:connect(Host, Port, [])]) || N <- lists:seq(MaxSessions+1, MaxSessions+1+FloodSessions) ], - ct:log("try ~p ssh:connect", [MaxSessions - UseSessions]), + ct:log("~p:~p try ~p ssh:connect", [?MODULE,?LINE, MaxSessions - UseSessions]), try_ssh_connect(MaxSessions - UseSessions, NegTimeOut, SSHconnect); Len1 -> diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index 957dbafda6..8ad405b1d7 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -305,6 +305,7 @@ shell_channel_tree(Config) -> {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), ok = ssh_connection:shell(ConnectionRef,ChannelId0), + success = ssh_connection:ptty_alloc(ConnectionRef, ChannelId0, [{pty_opts,[{onlcr,1}]}]), ?wait_match([{_, GroupPid,worker,[ssh_server_channel]}], supervisor:which_children(ChannelSup), diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index a9591547dd..ae8f7abb70 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -55,6 +55,7 @@ rcv_lingering/1, receive_exec_result/1, receive_exec_result_or_fail/1, receive_exec_end/2, +receive_exec_end/3, receive_exec_result/3, failfun/2, hostname/0, @@ -443,10 +444,10 @@ receive_exec_result(Msgs) when is_list(Msgs) -> false -> case Msg of {ssh_cm,_,{data,_,1, Data}} -> - ct:log("~p:~p StdErr: ~p~n", [?MODULE,?FUNCTION_NAME,Data]), + ct:log("~p:~p unexpected StdErr: ~p~n~p~n", [?MODULE,?FUNCTION_NAME,Data,Msg]), receive_exec_result(Msgs); Other -> - ct:log("~p:~p Other ~p", [?MODULE,?FUNCTION_NAME,Other]), + ct:log("~p:~p unexpected Other ~p", [?MODULE,?FUNCTION_NAME,Other]), {unexpected_msg, Other} end end @@ -474,9 +475,12 @@ receive_exec_result_or_fail(Msg) -> end. receive_exec_end(ConnectionRef, ChannelId) -> + receive_exec_end(ConnectionRef, ChannelId, 0). + +receive_exec_end(ConnectionRef, ChannelId, ExitStatus) -> receive_exec_result( [{ssh_cm, ConnectionRef, {eof, ChannelId}}, - {optional, {ssh_cm, ConnectionRef, {exit_status, ChannelId, 0}}}, + {optional, {ssh_cm, ConnectionRef, {exit_status, ChannelId, ExitStatus}}}, {ssh_cm, ConnectionRef, {closed, ChannelId}} ]). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 06c611a79c..8dadccc559 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.10.6 +SSH_VSN = 4.10.7 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index e86458f52d..ee5b40dea8 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,101 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 10.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Avoid race when the first two upgrade server handshakes + (that is servers that use a gen_tcp socket as input to + ssl:handshake/2,3) start close to each other. Could lead + to that one of the handshakes would fail.</p> + <p> + Own Id: OTP-17190 Aux Id: ERIERL-606 </p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 10.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Avoid that upgrade (from TCP to TLS) servers starts + multiple session cache handlers for the same server. This + applies to Erlang distribution over TLS servers.</p> + <p> + Own Id: OTP-17139 Aux Id: ERL-1458, OTP-16239 </p> + </item> + <item> + <p> + Legacy cipher suites defined before TLS-1.2 (but still + supported) should be possible to use in TLS-1.2. They + where accidentally excluded for available cipher suites + for TLS-1.2 in OTP-23.2.2.</p> + <p> + Own Id: OTP-17174 Aux Id: ERIERL-597 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Enable Erlang distribution over TLS to run TLS-1.3, + although TLS-1.2 will still be default.</p> + <p> + Own Id: OTP-16239 Aux Id: ERL-1458, OTP-17139 </p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 10.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix CVE-2020-35733 this only affects ssl-10.2 (OTP-23.2). + This vulnerability could enable a man in the middle + attack using a fake chain to a known trusted ROOT. Also + limits alternative chain handling, for handling of + possibly extraneous certs, to improve memory management.</p> + <p> + Own Id: OTP-17098</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add support for AES CCM based cipher suites defined in + RFC 7251</p> + <p> + Also Correct cipher suite name conversion to OpenSSL + names. A few names where corrected earlier in OTP-16267 + For backwards compatible reasons we support usage of + openSSL names for cipher suites. Mostly anonymous suites + names where incorrect, but also some legacy suites.</p> + <p> + Own Id: OTP-17100</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 10.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 0bb43f4937..0b502d61a1 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -451,7 +451,7 @@ in a valid certification path. So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly; if 1 the path can be PEER, CA, ROOT-CA; if 2 the path can be PEER, CA, CA, - ROOT-CA, and so on. The default value is 1.</p> + ROOT-CA, and so on. The default value is 10.</p> </desc> </datatype> @@ -1043,6 +1043,21 @@ fun(srp, Username :: binary(), UserState :: term()) -> </desc> </datatype> + <datatype> + <name name="client_early_data"/> + <desc> + <p>Configures the early data to be sent by the client.</p> + <p>In order to be able to verify + that the server has the intention to process the early data, the following 3-tuple is + sent to the user process:</p> + <p><c>{ssl, SslSocket, {early_data, Result}}</c></p> + <p>where <c>Result</c> is either <c>accepted</c> or <c>rejected</c>.</p> + <warning> + <p>It is the responsibility of the user to handle a rejected Early Data and + to resend when it is appropriate.</p></warning> + </desc> + </datatype> + <!-- <datatype> --> <!-- <name name="ocsp_stapling"/> --> <!-- <desc><p>If true, OCSP stapling will be enabled, an extension of type "status_request" will be --> @@ -1335,6 +1350,17 @@ fun(srp, Username :: binary(), UserState :: term()) -> </datatype> <datatype> + <name name="server_early_data"/> + <desc> + <p>Configures if the server accepts (<c>enabled</c>) or rejects (<c>rejects</c>) early + data sent by a client. The default value is <c>disabled</c>. + </p> + <warning><p>This option is a placeholder, early data is not yet implemented on the server side. + </p></warning> + </desc> + </datatype> + + <datatype> <name name="connection_info"/> </datatype> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index f5905ecbab..1426206508 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -177,6 +177,16 @@ </p> </item> + <tag><c><![CDATA[server_session_ticket_max_early_data = integer() <optional>]]></c></tag> + <item> + <p> + Sets the maximum size of the early data that the server accepts and also configures + its NewSessionTicket messages to include this same size limit in their + early_data_indication extension. + Defaults to 16384. Size limit is enforced by both client and server. + </p> + </item> + <tag><c><![CDATA[client_session_ticket_lifetime = integer() <optional>]]></c></tag> <item> <p> diff --git a/lib/ssl/doc/src/standards_compliance.xml b/lib/ssl/doc/src/standards_compliance.xml index 6cf25d726f..b2e7d17c45 100644 --- a/lib/ssl/doc/src/standards_compliance.xml +++ b/lib/ssl/doc/src/standards_compliance.xml @@ -140,7 +140,7 @@ <list type="bulleted"> <item>PSK and session resumption is supported (stateful and stateless tickets)</item> <item>Anti-replay protection using Bloom-filters with stateless tickets</item> - <item>Early data and 0-RTT not supported</item> + <item>Early data and 0-RTT is supported</item> <item>Key and Initialization Vector Update is supported</item> </list> <p>For more detailed information see the @@ -251,8 +251,8 @@ </url> </cell> <cell align="left" valign="middle"></cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> @@ -387,8 +387,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -526,8 +526,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1135,14 +1135,14 @@ </url> </cell> <cell align="left" valign="middle"><em>Client</em></cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle"><em>Server</em></cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> @@ -1274,8 +1274,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1341,8 +1341,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1667,14 +1667,14 @@ </url> </cell> <cell align="left" valign="middle"><em>Client</em></cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle"><em>Server</em></cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> @@ -1684,27 +1684,27 @@ </url> </cell> <cell align="left" valign="middle"><em>Client</em></cell> - <cell align="left" valign="middle"><em>PC</em></cell> - <cell align="left" valign="middle"><em>22.2</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle"><em>Server</em></cell> - <cell align="left" valign="middle"><em>PC</em></cell> - <cell align="left" valign="middle"><em>22.2</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">early_data (RFC8446)</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>23.3</em></cell> </row> <row> diff --git a/lib/ssl/doc/src/using_ssl.xml b/lib/ssl/doc/src/using_ssl.xml index 4a66bf9d90..7f45b72db9 100644 --- a/lib/ssl/doc/src/using_ssl.xml +++ b/lib/ssl/doc/src/using_ssl.xml @@ -559,6 +559,120 @@ ok </section> <section> + <title>Early Data in TLS 1.3</title> + <p>TLS 1.3 allows clients to send data on the first flight if the endpoints have + a shared crypographic secret (pre-shared key). This means that clients can send + early data if they have a valid session ticket received in a previous + successful handshake. For more information about session resumption see + <seeguide marker="ssl:using_ssl#session-tickets-and-session-resumption-in-tls-1.3"> + Session Tickets and Session Resumption in TLS 1.3</seeguide>. + </p> + <p>The security properties of Early Data are weaker than other kinds of TLS data. + This data is not forward secret, and it is vulnerable to replay attacks. For available + mitigation strategies see + <seeguide marker="ssl:using_ssl#anti-replay-protection-in-tls-1.3"> + Anti-Replay Protection in TLS 1.3</seeguide>.</p> + <p>In normal operation, clients will not know which, if any, of the available mitigation + strategies servers actually implement, and hence must only send early data which + they deem safe to be replayed. For example, idempotent HTTP operations, such as HEAD and + GET, can usually be regarded as safe but even they can be exploited by a large number of + replays causing resource limit exhaustion and other similar problems.</p> + <p>An example of sending early data with automatic and manual session ticket handling:</p> + <warning> + <p>The Early Data feature is experimental in this version of OTP. + </p> + </warning> + + <p><em>Server (with NSS key logging)</em></p> + <code type="none"> + early_data_server() -> + application:load(ssl), + {ok, _} = application:ensure_all_started(ssl), + Port = 11029, + LOpts = [{certfile, ?SERVER_CERT}, + {keyfile, ?SERVER_KEY}, + {reuseaddr, true}, + {versions, ['tlsv1.2','tlsv1.3']}, + {session_tickets, stateless}, + {early_data, enabled}, + {keep_secrets, true} %% Enable NSS key log (debug option) + ], + {ok, LSock} = ssl:listen(Port, LOpts), + %% Accept first connection + {ok, CSock0} = ssl:transport_accept(LSock), + {ok, _} = ssl:handshake(CSock0), + %% Accept second connection + {ok, CSock1} = ssl:transport_accept(LSock), + {ok, Sock} = ssl:handshake(CSock1), + Sock. + </code> + <p><em>Exporting the secrets (optional)</em></p> + <code type="none"> + {ok, [{keylog, KeylogItems}]} = ssl:connection_information(Sock, [keylog]). + file:write_file("key.log", [[KeylogItem,$\n] || KeylogItem <- KeylogItems]). + </code> + <p><em>Client (automatic ticket handling):</em></p> + <code type="erl"> + early_data_auto() -> + %% First handshake 1-RTT - get session tickets + application:load(ssl), + {ok, _} = application:ensure_all_started(ssl), + Port = 11029, + Data = <<"HEAD / HTTP/1.1\r\nHost: \r\nConnection: close\r\n">>, + COpts0 = [{cacertfile, ?CA_CERT}, + {versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, auto}], + {ok, Sock0} = ssl:connect("localhost", Port, COpts0), + + %% Wait for session tickets + timer:sleep(500), + %% Close socket if server cannot handle multiple connections e.g. openssl s_server + ssl:close(Sock0), + + %% Second handshake 0-RTT + COpts1 = [{cacertfile, ?CA_CERT}, + {versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, auto}, + {early_data, Data}], + {ok, Sock} = ssl:connect("localhost", Port, COpts1), + Sock. + </code> + <p><em>Client (manual ticket handling):</em></p> + <code type="erl"> + early_data_manual() -> + %% First handshake 1-RTT - get session tickets + application:load(ssl), + {ok, _} = application:ensure_all_started(ssl), + Port = 11029, + Data = <<"HEAD / HTTP/1.1\r\nHost: \r\nConnection: close\r\n">>, + COpts0 = [{cacertfile, ?CA_CERT}, + {versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}], + {ok, Sock0} = ssl:connect("localhost", Port, COpts0), + + %% Wait for session tickets + Ticket = + receive + {ssl, session_ticket, Ticket0} -> + Ticket0 + end, + + %% Close socket if server cannot handle multiple connections + %% e.g. openssl s_server + ssl:close(Sock0), + + %% Second handshake 0-RTT + COpts1 = [{cacertfile, ?CA_CERT}, + {versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}, + {use_ticket, [Ticket]}, + {early_data, Data}], + {ok, Sock} = ssl:connect("localhost", Port, COpts1), + Sock. + </code> + </section> + + <section> <title>Anti-Replay Protection in TLS 1.3</title> <p>The TLS 1.3 protocol does not provide inherent protection for replay of 0-RTT data but diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile index 8abd7820f7..1a55ee7b83 100644 --- a/lib/ssl/src/Makefile +++ b/lib/ssl/src/Makefile @@ -85,6 +85,7 @@ MODULES= \ ssl_server_session_cache \ ssl_server_session_cache_db \ ssl_server_session_cache_sup \ + ssl_upgrade_server_session_cache_sup \ ssl_session \ ssl_session_cache \ ssl_srp_primes \ @@ -100,6 +101,8 @@ MODULES= \ tls_record \ tls_record_1_3 \ tls_client_ticket_store \ + tls_dist_sup \ + tls_dist_server_sup \ tls_sender \ tls_server_session_ticket\ tls_server_session_ticket_sup\ diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index 01355f664e..f9660ea1b6 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -104,7 +104,7 @@ getstat(PacketSocket, Opts) -> init([Port0, TransportInfo, EmOpts, DTLSOptions, Socket]) -> InternalActiveN = get_internal_active_n(), - {ok, SessionIdHandle} = session_id_tracker(DTLSOptions), + {ok, SessionIdHandle} = session_id_tracker(Socket, DTLSOptions), {ok, #state{active_n = InternalActiveN, port = Port0, first = true, @@ -355,9 +355,8 @@ emulated_opts_list(Opts, [active | Rest], Acc) -> %% Regardless of the option reuse_sessions we need the session_id_tracker %% to generate session ids, but no sessions will be stored unless %% reuse_sessions = true. -session_id_tracker(_) -> - dtls_server_session_cache_sup:start_child( - ssl_server_session_cache_sup:session_opts()). +session_id_tracker(Listner,_) -> + dtls_server_session_cache_sup:start_child(Listner). get_internal_active_n() -> case application:get_env(ssl, internal_active_n) of diff --git a/lib/ssl/src/dtls_server_session_cache_sup.erl b/lib/ssl/src/dtls_server_session_cache_sup.erl index 42cbda27ec..457eb90167 100644 --- a/lib/ssl/src/dtls_server_session_cache_sup.erl +++ b/lib/ssl/src/dtls_server_session_cache_sup.erl @@ -41,8 +41,8 @@ %%%========================================================================= start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(Args) -> - supervisor:start_child(?MODULE, [self() | Args]). +start_child(Listener) -> + supervisor:start_child(?MODULE, [Listener | ssl_config:pre_1_3_session_opts()]). %%%========================================================================= %%% Supervisor callback diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index 18119d9040..eaa481f119 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -766,8 +766,8 @@ nodelay() -> get_ssl_options(Type) -> try ets:lookup(ssl_dist_opts, Type) of - [{Type, Opts}] -> - [{erl_dist, true}, {versions, ['tlsv1.2']} | Opts]; + [{Type, Opts0}] -> + [{erl_dist, true} | dist_defaults(Opts0)]; _ -> get_ssl_dist_arguments(Type) catch @@ -778,11 +778,18 @@ get_ssl_options(Type) -> get_ssl_dist_arguments(Type) -> case init:get_argument(ssl_dist_opt) of {ok, Args} -> - [{erl_dist, true}, {versions, ['tlsv1.2']} | ssl_options(Type, lists:append(Args))]; + [{erl_dist, true} | dist_defaults(ssl_options(Type, lists:append(Args)))]; _ -> - [{erl_dist, true}, {versions, ['tlsv1.2']}] + [{erl_dist, true}] end. +dist_defaults(Opts) -> + case proplists:get_value(versions, Opts, undefined) of + undefined -> + [{versions, ['tlsv1.2']} | Opts]; + _ -> + Opts + end. ssl_options(_Type, []) -> []; diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 914b797f99..f3cc463cff 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -54,12 +54,15 @@ ssl_dist_sup, ssl_dist_connection_sup, ssl_dist_admin_sup, + tls_dist_sup, + tls_dist_server_sup, %% SSL/TLS session and cert handling ssl_session, ssl_session_cache, ssl_server_session_cache, ssl_server_session_cache_db, ssl_server_session_cache_sup, + ssl_upgrade_server_session_cache_sup, ssl_manager, ssl_pem_cache, ssl_pkix_db, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 26f131227c..da98c38e62 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -384,6 +384,8 @@ bloom_filter_bits()}. %% m - number of bits in bit vector -type use_ticket() :: [binary()]. -type middlebox_comp_mode() :: boolean(). +-type client_early_data() :: binary(). +-type server_early_data() :: disabled | enabled. %% ------------------------------------------------------------------------------------------------------- @@ -402,7 +404,8 @@ {signature_algs, client_signature_algs()} | {fallback, fallback()} | {session_tickets, client_session_tickets()} | - {use_ticket, use_ticket()}. %% | + {use_ticket, use_ticket()} | + {early_data, client_early_data()}. %% {ocsp_stapling, ocsp_stapling()} | %% {ocsp_responder_certs, ocsp_responder_certs()} | %% {ocsp_nonce, ocsp_nonce()}. @@ -453,7 +456,8 @@ {signature_algs, server_signature_algs()} | {session_tickets, server_session_tickets()} | {anti_replay, anti_replay()} | - {cookie, cookie()}. + {cookie, cookie()} | + {early_data, server_early_data()}. -type server_cacerts() :: [public_key:der_encoded()]. -type server_cafile() :: file:filename(). @@ -554,7 +558,7 @@ stop() -> TCPSocket :: socket(), TLSOptions :: [tls_client_option()]. -connect(Socket, SslOptions) when is_port(Socket) -> +connect(Socket, SslOptions) -> connect(Socket, SslOptions, infinity). -spec connect(TCPSocket, TLSOptions, Timeout) -> @@ -571,24 +575,23 @@ connect(Socket, SslOptions) when is_port(Socket) -> Port :: inet:port_number(), TLSOptions :: [tls_client_option()]. -connect(Socket, SslOptions0, Timeout) when is_port(Socket), +connect(Socket, SslOptions0, Timeout) when is_list(SslOptions0) andalso (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> + CbInfo = handle_option_cb_info(SslOptions0, tls), - Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), try handle_options(SslOptions0 ++ SocketValues, client) of - {ok, Config} -> - tls_socket:upgrade(Socket, Config, Timeout) + {ok, Config} -> + tls_socket:upgrade(Socket, Config, Timeout) catch - _:{error, Reason} -> + _:{error, Reason} -> {error, Reason} - end; + end; connect(Host, Port, Options) -> connect(Host, Port, Options, infinity). - -spec connect(Host, Port, TLSOptions, Timeout) -> {ok, sslsocket()} | {ok, sslsocket(),Ext :: protocol_extensions()} | @@ -743,9 +746,8 @@ handshake(#sslsocket{} = Socket, Timeout) when (is_integer(Timeout) andalso Tim %% %% If Socket is an sslsocket(): provides extra SSL/TLS/DTLS options to those %% specified in ssl:listen/2 and then performs the SSL/TLS/DTLS handshake. -handshake(ListenSocket, SslOptions) when is_port(ListenSocket) -> +handshake(ListenSocket, SslOptions) -> handshake(ListenSocket, SslOptions, infinity). - -spec handshake(Socket, Options, Timeout) -> {ok, SslSocket} | {ok, SslSocket, Ext} | @@ -779,28 +781,25 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) catch Error = {error, _Reason} -> Error end; -handshake(Socket, SslOptions, Timeout) when is_port(Socket), - (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> +handshake(Socket, SslOptions, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> CbInfo = handle_option_cb_info(SslOptions, tls), - Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), ConnetionCb = connection_cb(SslOptions), try handle_options(SslOptions ++ SocketValues, server) of - {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} -> - ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), - {ok, Port} = tls_socket:port(Transport, Socket), - {ok, SessionIdHandle} = tls_socket:session_id_tracker(SslOpts), - ssl_gen_statem:handshake(ConnetionCb, Port, Socket, + {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} -> + ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), + {ok, Port} = tls_socket:port(Transport, Socket), + {ok, SessionIdHandle} = tls_socket:session_id_tracker(ssl_unknown_listener, SslOpts), + ssl_gen_statem:handshake(ConnetionCb, Port, Socket, {SslOpts, tls_socket:emulated_socket_options(EmOpts, #socket_options{}), [{session_id_tracker, SessionIdHandle}]}, self(), CbInfo, Timeout) catch - Error = {error, _Reason} -> Error - end. - + Error = {error, _Reason} -> Error + end. %%-------------------------------------------------------------------- -spec handshake_continue(HsSocket, Options) -> @@ -1708,6 +1707,32 @@ handle_option(client_renegotiation = Option, Value0, ['tlsv1','tlsv1.1','tlsv1.2']), Value = validate_option(Option, Value0), OptionsMap#{Option => Value}; +handle_option(early_data = Option, unbound, OptionsMap, #{rules := Rules}) -> + Value = validate_option(Option, default_value(Option, Rules)), + OptionsMap#{Option => Value}; +handle_option(early_data = Option, Value0, #{session_tickets := SessionTickets, + versions := Versions} = OptionsMap, + #{role := server = Role}) -> + assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), + assert_option_dependency(Option, session_tickets, [SessionTickets], + [stateful, stateless]), + Value = validate_option(Option, Value0, Role), + OptionsMap#{Option => Value}; +handle_option(early_data = Option, Value0, #{session_tickets := SessionTickets, + use_ticket := UseTicket, + versions := Versions} = OptionsMap, + #{role := client = Role}) -> + assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), + assert_option_dependency(Option, session_tickets, [SessionTickets], + [manual, auto]), + case UseTicket of + undefined when SessionTickets =/= auto -> + throw({error, {options, dependency, {Option, use_ticket}}}); + _ -> + ok + end, + Value = validate_option(Option, Value0, Role), + OptionsMap#{Option => Value}; handle_option(eccs = Option, unbound, #{versions := [HighestVersion|_]} = OptionsMap, #{rules := _Rules}) -> Value = handle_eccs_option(eccs(), HighestVersion), OptionsMap#{Option => Value}; @@ -1829,13 +1854,13 @@ handle_option(server_name_indication = Option, unbound, OptionsMap, #{host := Ho handle_option(server_name_indication = Option, Value0, OptionsMap, _Env) -> Value = validate_option(Option, Value0), OptionsMap#{Option => Value}; -handle_option(session_tickets = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), +handle_option(session_tickets = Option, unbound, OptionsMap, #{role := Role, + rules := Rules}) -> + Value = validate_option(Option, default_value(Option, Rules), Role), OptionsMap#{Option => Value}; handle_option(session_tickets = Option, Value0, #{versions := Versions} = OptionsMap, #{role := Role}) -> assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - assert_role_value(Role, Option, Value0, [disabled, stateful, stateless], [disabled, manual, auto]), - Value = validate_option(Option, Value0), + Value = validate_option(Option, Value0, Role), OptionsMap#{Option => Value}; handle_option(signature_algs = Option, unbound, #{versions := [HighestVersion|_]} = OptionsMap, #{role := Role}) -> Value = @@ -2047,24 +2072,6 @@ assert_role(server_only, _, _, undefined) -> assert_role(Type, _, Key, _) -> throw({error, {option, Type, Key}}). - -assert_role_value(client, Option, Value, _, ClientValues) -> - case lists:member(Value, ClientValues) of - true -> - ok; - false -> - %% throw({error, {option, client, Option, Value, ClientValues}}) - throw({error, {options, role, {Option, {Value, {client, ClientValues}}}}}) - end; -assert_role_value(server, Option, Value, ServerValues, _) -> - case lists:member(Value, ServerValues) of - true -> - ok; - false -> - %% throw({error, {option, server, Option, Value, ServerValues}}) - throw({error, {options, role, {Option, {Value, {server, ServerValues}}}}}) - end. - assert_option_dependency(Option, OptionDep, Values0, AllowedValues) -> case is_dtls_configured(Values0) of true -> @@ -2099,304 +2106,384 @@ is_dtls_configured(Versions) -> end, lists:any(Fun, Versions). -validate_option(versions, Versions) -> - validate_versions(Versions, Versions); -validate_option(verify, Value) - when Value == verify_none; Value == verify_peer -> +validate_option(Option, Value) -> + validate_option(Option, Value, undefined). +%% +validate_option(Opt, Value, _) + when Opt =:= alpn_advertised_protocols orelse + Opt =:= alpn_preferred_protocols, + is_list(Value) -> + validate_binary_list(Opt, Value), Value; -validate_option(verify_fun, undefined) -> +validate_option(Opt, Value, _) + when Opt =:= alpn_advertised_protocols orelse + Opt =:= alpn_preferred_protocols, + Value =:= undefined -> undefined; -%% Backwards compatibility -validate_option(verify_fun, Fun) when is_function(Fun) -> - {fun(_,{bad_cert, _} = Reason, OldFun) -> - case OldFun([Reason]) of - true -> - {valid, OldFun}; - false -> - {fail, Reason} - end; - (_,{extension, _}, UserState) -> - {unknown, UserState}; - (_, valid, UserState) -> - {valid, UserState}; - (_, valid_peer, UserState) -> - {valid, UserState} - end, Fun}; -validate_option(verify_fun, {Fun, _} = Value) when is_function(Fun) -> - Value; -validate_option(partial_chain, Value) when is_function(Value) -> +validate_option(anti_replay, '10k', _) -> + %% n = 10000 + %% p = 0.030003564 (1 in 33) + %% m = 72985 (8.91KiB) + %% k = 5 + {10, 5, 72985}; +validate_option(anti_replay, '100k', _) -> + %% n = 100000 + %% p = 0.03000428 (1 in 33) + %% m = 729845 (89.09KiB) + %% k = 5 + {10, 5, 729845}; +validate_option(anti_replay, Value, _) + when (is_tuple(Value) andalso + tuple_size(Value) =:= 3) -> Value; -validate_option(fail_if_no_peer_cert, Value) when is_boolean(Value) -> +validate_option(beast_mitigation, Value, _) + when Value == one_n_minus_one orelse + Value == zero_n orelse + Value == disabled -> + Value; +%% certfile must be present in some cases otherwhise it can be set +%% to the empty string. +validate_option(cacertfile, undefined, _) -> + <<>>; +validate_option(cacertfile, Value, _) + when is_binary(Value) -> Value; -validate_option(depth, Value) when is_integer(Value), - Value >= 0, Value =< 255-> +validate_option(cacertfile, Value, _) + when is_list(Value), Value =/= ""-> + binary_filename(Value); +validate_option(cacerts, Value, _) + when Value == undefined; + is_list(Value) -> Value; -validate_option(cert, Value) when Value == undefined; - is_list(Value)-> +validate_option(cb_info, {V1, V2, V3, V4} = Value, _) + when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4) -> Value; -validate_option(cert, Value) when Value == undefined; - is_binary(Value)-> +validate_option(cb_info, {V1, V2, V3, V4, V5} = Value, _) + when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4), + is_atom(V5) -> + Value; +validate_option(cert, Value, _) when Value == undefined; + is_list(Value)-> + Value; +validate_option(cert, Value, _) when Value == undefined; + is_binary(Value)-> [Value]; -validate_option(certfile, undefined = Value) -> +validate_option(certfile, undefined = Value, _) -> Value; -validate_option(certfile, Value) when is_binary(Value) -> +validate_option(certfile, Value, _) + when is_binary(Value) -> Value; -validate_option(certfile, Value) when is_list(Value) -> +validate_option(certfile, Value, _) + when is_list(Value) -> binary_filename(Value); - -validate_option(key, undefined) -> +validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols}, _) + when is_list(PreferredProtocols) -> + validate_binary_list(client_preferred_next_protocols, PreferredProtocols), + validate_npn_ordering(Precedence), + {Precedence, PreferredProtocols, ?NO_PROTOCOL}; +validate_option(client_preferred_next_protocols, + {Precedence, PreferredProtocols, Default} = Value, _) + when is_list(PreferredProtocols), is_binary(Default), + byte_size(Default) > 0, byte_size(Default) < 256 -> + validate_binary_list(client_preferred_next_protocols, PreferredProtocols), + validate_npn_ordering(Precedence), + Value; +validate_option(client_preferred_next_protocols, undefined, _) -> undefined; -validate_option(key, {KeyType, Value}) when is_binary(Value), - KeyType == rsa; %% Backwards compatibility - KeyType == dsa; %% Backwards compatibility - KeyType == 'RSAPrivateKey'; - KeyType == 'DSAPrivateKey'; - KeyType == 'ECPrivateKey'; - KeyType == 'PrivateKeyInfo' -> - {KeyType, Value}; -validate_option(key, #{algorithm := _} = Value) -> +validate_option(client_renegotiation, Value, _) + when is_boolean(Value) -> Value; -validate_option(keyfile, undefined) -> - <<>>; -validate_option(keyfile, Value) when is_binary(Value) -> +validate_option(cookie, Value, _) + when is_boolean(Value) -> Value; -validate_option(keyfile, Value) when is_list(Value), Value =/= "" -> - binary_filename(Value); -validate_option(key_update_at, Value) when is_integer(Value) andalso - Value > 0 -> +validate_option(crl_cache, {Cb, {_Handle, Options}} = Value, _) + when is_atom(Cb) and is_list(Options) -> Value; -validate_option(password, Value) when is_list(Value) -> +validate_option(crl_check, Value, _) + when is_boolean(Value) -> Value; - -validate_option(cacerts, Value) when Value == undefined; - is_list(Value) -> +validate_option(crl_check, Value, _) + when (Value == best_effort) or + (Value == peer) -> Value; -%% certfile must be present in some cases otherwhise it can be set -%% to the empty string. -validate_option(cacertfile, undefined) -> - <<>>; -validate_option(cacertfile, Value) when is_binary(Value) -> +validate_option(customize_hostname_check, Value, _) + when is_list(Value) -> Value; -validate_option(cacertfile, Value) when is_list(Value), Value =/= ""-> - binary_filename(Value); -validate_option(dh, Value) when Value == undefined; - is_binary(Value) -> +validate_option(depth, Value, _) + when is_integer(Value), + Value >= 0, Value =< 255-> Value; -validate_option(dhfile, undefined = Value) -> +validate_option(dh, Value, _) + when Value == undefined; + is_binary(Value) -> Value; -validate_option(dhfile, Value) when is_binary(Value) -> +validate_option(dhfile, undefined = Value, _) -> Value; -validate_option(dhfile, Value) when is_list(Value), Value =/= "" -> - binary_filename(Value); -validate_option(psk_identity, undefined) -> - undefined; -validate_option(psk_identity, Identity) - when is_list(Identity), Identity =/= "", length(Identity) =< 65535 -> - binary_filename(Identity); -validate_option(user_lookup_fun, undefined) -> - undefined; -validate_option(user_lookup_fun, {Fun, _} = Value) when is_function(Fun, 3) -> - Value; -validate_option(srp_identity, undefined) -> - undefined; -validate_option(srp_identity, {Username, Password}) - when is_list(Username), is_list(Password), Username =/= "", length(Username) =< 255 -> - {unicode:characters_to_binary(Username), - unicode:characters_to_binary(Password)}; - -validate_option(reuse_session, undefined) -> - undefined; -validate_option(reuse_session, Value) when is_function(Value) -> +validate_option(dhfile, Value, _) + when is_binary(Value) -> Value; -validate_option(reuse_session, Value) when is_binary(Value) -> +validate_option(dhfile, Value, _) + when is_list(Value), Value =/= "" -> + binary_filename(Value); +validate_option(early_data, Value, server) + when Value =:= disabled orelse + Value =:= enabled -> Value; -validate_option(reuse_session, {Id, Data} = Value) when is_binary(Id) andalso - is_binary(Data) -> +validate_option(early_data = Option, Value, server) -> + throw({error, + {options, role, {Option, {Value, {server, [disabled, enabled]}}}}}); +validate_option(early_data, Value, client) + when is_binary(Value) -> Value; -validate_option(reuse_sessions, Value) when is_boolean(Value) -> +validate_option(early_data = Option, Value, client) -> + throw({error, + {options, type, {Option, {Value, not_binary}}}}); +validate_option(erl_dist, Value, _) + when is_boolean(Value) -> Value; -validate_option(reuse_sessions, save = Value) -> +validate_option(fail_if_no_peer_cert, Value, _) + when is_boolean(Value) -> Value; -validate_option(secure_renegotiate, Value) when is_boolean(Value) -> +validate_option(fallback, Value, _) + when is_boolean(Value) -> Value; -validate_option(keep_secrets, Value) when is_boolean(Value) -> +validate_option(handshake, hello = Value, _) -> Value; -validate_option(client_renegotiation, Value) when is_boolean(Value) -> +validate_option(handshake, full = Value, _) -> Value; -validate_option(renegotiate_at, Value) when is_integer(Value) -> - erlang:min(Value, ?DEFAULT_RENEGOTIATE_AT); - -validate_option(hibernate_after, undefined) -> %% Backwards compatibility +validate_option(hibernate_after, undefined, _) -> %% Backwards compatibility infinity; -validate_option(hibernate_after, infinity) -> +validate_option(hibernate_after, infinity, _) -> infinity; -validate_option(hibernate_after, Value) when is_integer(Value), Value >= 0 -> +validate_option(hibernate_after, Value, _) + when is_integer(Value), Value >= 0 -> Value; - -validate_option(erl_dist,Value) when is_boolean(Value) -> +validate_option(honor_cipher_order, Value, _) + when is_boolean(Value) -> Value; -validate_option(Opt, Value) when Opt =:= alpn_advertised_protocols orelse Opt =:= alpn_preferred_protocols, - is_list(Value) -> - validate_binary_list(Opt, Value), +validate_option(honor_ecc_order, Value, _) + when is_boolean(Value) -> Value; -validate_option(Opt, Value) - when Opt =:= alpn_advertised_protocols orelse Opt =:= alpn_preferred_protocols, - Value =:= undefined -> - undefined; -validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols}) - when is_list(PreferredProtocols) -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), - {Precedence, PreferredProtocols, ?NO_PROTOCOL}; -validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols, Default} = Value) - when is_list(PreferredProtocols), is_binary(Default), - byte_size(Default) > 0, byte_size(Default) < 256 -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), +validate_option(keep_secrets, Value, _) when is_boolean(Value) -> Value; -validate_option(client_preferred_next_protocols, undefined) -> +validate_option(key, undefined, _) -> undefined; -validate_option(log_alert, true) -> - notice; -validate_option(log_alert, false) -> - warning; -validate_option(log_level, Value) when - is_atom(Value) andalso - (Value =:= emergency orelse - Value =:= alert orelse - Value =:= critical orelse - Value =:= error orelse - Value =:= warning orelse - Value =:= notice orelse - Value =:= info orelse - Value =:= debug) -> +validate_option(key, {KeyType, Value}, _) + when is_binary(Value), + KeyType == rsa; %% Backwards compatibility + KeyType == dsa; %% Backwards compatibility + KeyType == 'RSAPrivateKey'; + KeyType == 'DSAPrivateKey'; + KeyType == 'ECPrivateKey'; + KeyType == 'PrivateKeyInfo' -> + {KeyType, Value}; +validate_option(key, #{algorithm := _} = Value, _) -> Value; -validate_option(middlebox_comp_mode, Value) when is_boolean(Value) -> +validate_option(keyfile, undefined, _) -> + <<>>; +validate_option(keyfile, Value, _) + when is_binary(Value) -> Value; -validate_option(next_protocols_advertised, Value) when is_list(Value) -> - validate_binary_list(next_protocols_advertised, Value), +validate_option(keyfile, Value, _) + when is_list(Value), Value =/= "" -> + binary_filename(Value); +validate_option(key_update_at, Value, _) + when is_integer(Value) andalso + Value > 0 -> Value; -validate_option(next_protocols_advertised, undefined) -> - undefined; -validate_option(server_name_indication, Value) when is_list(Value) -> - %% RFC 6066, Section 3: Currently, the only server names supported are - %% DNS hostnames - %% case inet_parse:domain(Value) of - %% false -> - %% throw({error, {options, {{Opt, Value}}}}); - %% true -> - %% Value - %% end; - %% - %% But the definition seems very diffuse, so let all strings through - %% and leave it up to public_key to decide... +validate_option(log_alert, true, _) -> + notice; +validate_option(log_alert, false, _) -> + warning; +validate_option(log_level, Value, _) + when is_atom(Value) andalso + (Value =:= emergency orelse + Value =:= alert orelse + Value =:= critical orelse + Value =:= error orelse + Value =:= warning orelse + Value =:= notice orelse + Value =:= info orelse + Value =:= debug) -> Value; -validate_option(server_name_indication, undefined) -> - undefined; -validate_option(server_name_indication, disable) -> - disable; - %% RFC 6066, Section 4 -validate_option(max_fragment_length, I) when I == ?MAX_FRAGMENT_LENGTH_BYTES_1; I == ?MAX_FRAGMENT_LENGTH_BYTES_2; - I == ?MAX_FRAGMENT_LENGTH_BYTES_3; I == ?MAX_FRAGMENT_LENGTH_BYTES_4 -> +validate_option(max_fragment_length, I, _) + when I == ?MAX_FRAGMENT_LENGTH_BYTES_1; + I == ?MAX_FRAGMENT_LENGTH_BYTES_2; + I == ?MAX_FRAGMENT_LENGTH_BYTES_3; + I == ?MAX_FRAGMENT_LENGTH_BYTES_4 -> I; -validate_option(max_fragment_length, undefined) -> +validate_option(max_fragment_length, undefined, _) -> undefined; - -validate_option(sni_hosts, []) -> - []; -validate_option(sni_hosts, [{Hostname, SSLOptions} | Tail]) when is_list(Hostname) -> - RecursiveSNIOptions = proplists:get_value(sni_hosts, SSLOptions, undefined), - case RecursiveSNIOptions of - undefined -> - [{Hostname, validate_options(SSLOptions)} | validate_option(sni_hosts, Tail)]; - _ -> - throw({error, {options, {sni_hosts, RecursiveSNIOptions}}}) - end; -validate_option(sni_fun, undefined) -> - undefined; -validate_option(sni_fun, Fun) when is_function(Fun) -> - Fun; -validate_option(honor_cipher_order, Value) when is_boolean(Value) -> +validate_option(max_handshake_size, Value, _) + when is_integer(Value) andalso + Value =< ?MAX_UNIT24 -> Value; -validate_option(honor_ecc_order, Value) when is_boolean(Value) -> +validate_option(middlebox_comp_mode, Value, _) + when is_boolean(Value) -> Value; -validate_option(padding_check, Value) when is_boolean(Value) -> +validate_option(next_protocols_advertised, Value, _) when is_list(Value) -> + validate_binary_list(next_protocols_advertised, Value), Value; -validate_option(fallback, Value) when is_boolean(Value) -> +validate_option(next_protocols_advertised, undefined, _) -> + undefined; +validate_option(ocsp_nonce, Value, _) + when Value =:= true orelse + Value =:= false -> Value; -validate_option(cookie, Value) when is_boolean(Value) -> +%% The OCSP responders' certificates can be given as a suggestion and +%% will be used to verify the OCSP response. +validate_option(ocsp_responder_certs, Value, _) + when is_list(Value) -> + [public_key:pkix_decode_cert(CertDer, plain) || CertDer <- Value, + is_binary(CertDer)]; +validate_option(ocsp_stapling, Value, _) + when Value =:= true orelse + Value =:= false -> Value; -validate_option(crl_check, Value) when is_boolean(Value) -> +validate_option(padding_check, Value, _) + when is_boolean(Value) -> Value; -validate_option(crl_check, Value) when (Value == best_effort) or (Value == peer) -> +validate_option(partial_chain, Value, _) + when is_function(Value) -> Value; -validate_option(crl_cache, {Cb, {_Handle, Options}} = Value) when is_atom(Cb) and is_list(Options) -> +validate_option(password, Value, _) + when is_list(Value) -> Value; -validate_option(beast_mitigation, Value) when Value == one_n_minus_one orelse - Value == zero_n orelse - Value == disabled -> - Value; -validate_option(max_handshake_size, Value) when is_integer(Value) andalso Value =< ?MAX_UNIT24 -> +validate_option(protocol, Value = tls, _) -> Value; -validate_option(protocol, Value = tls) -> +validate_option(protocol, Value = dtls, _) -> Value; -validate_option(protocol, Value = dtls) -> +validate_option(psk_identity, undefined, _) -> + undefined; +validate_option(psk_identity, Identity, _) + when is_list(Identity), Identity =/= "", length(Identity) =< 65535 -> + binary_filename(Identity); +validate_option(renegotiate_at, Value, _) when is_integer(Value) -> + erlang:min(Value, ?DEFAULT_RENEGOTIATE_AT); +validate_option(reuse_session, undefined, _) -> + undefined; +validate_option(reuse_session, Value, _) + when is_function(Value) -> Value; -validate_option(handshake, hello = Value) -> +validate_option(reuse_session, Value, _) + when is_binary(Value) -> Value; -validate_option(handshake, full = Value) -> +validate_option(reuse_session, {Id, Data} = Value, _) + when is_binary(Id) andalso + is_binary(Data) -> Value; -validate_option(customize_hostname_check, Value) when is_list(Value) -> +validate_option(reuse_sessions, Value, _) + when is_boolean(Value) -> Value; -validate_option(cb_info, {V1, V2, V3, V4} = Value) when is_atom(V1), - is_atom(V2), - is_atom(V3), - is_atom(V4) - -> +validate_option(reuse_sessions, save = Value, _) -> Value; -validate_option(cb_info, {V1, V2, V3, V4, V5} = Value) when is_atom(V1), - is_atom(V2), - is_atom(V3), - is_atom(V4), - is_atom(V5) - -> +validate_option(secure_renegotiate, Value, _) + when is_boolean(Value) -> Value; -validate_option(use_ticket, Value) when is_list(Value) -> +validate_option(server_name_indication, Value, _) + when is_list(Value) -> + %% RFC 6066, Section 3: Currently, the only server names supported are + %% DNS hostnames + %% case inet_parse:domain(Value) of + %% false -> + %% throw({error, {options, {{Opt, Value}}}}); + %% true -> + %% Value + %% end; + %% + %% But the definition seems very diffuse, so let all strings through + %% and leave it up to public_key to decide... Value; -validate_option(session_tickets, Value) when Value =:= disabled orelse - Value =:= manual orelse - Value =:= auto orelse - Value =:= stateless orelse - Value =:= stateful -> +validate_option(server_name_indication, undefined, _) -> + undefined; +validate_option(server_name_indication, disable, _) -> + disable; +validate_option(session_tickets, Value, server) + when Value =:= disabled orelse + Value =:= stateful orelse + Value =:= stateless -> Value; -validate_option(anti_replay, '10k') -> - %% n = 10000 - %% p = 0.030003564 (1 in 33) - %% m = 72985 (8.91KiB) - %% k = 5 - {10, 5, 72985}; -validate_option(anti_replay, '100k') -> - %% n = 100000 - %% p = 0.03000428 (1 in 33) - %% m = 729845 (89.09KiB) - %% k = 5 - {10, 5, 729845}; -validate_option(anti_replay, Value) when (is_tuple(Value) andalso - tuple_size(Value) =:= 3) -> +validate_option(session_tickets, Value, server) -> + throw({error, + {options, role, + {session_tickets, + {Value, {server, [disabled, stateful, stateless]}}}}}); +validate_option(session_tickets, Value, client) + when Value =:= disabled orelse + Value =:= manual orelse + Value =:= auto -> Value; -validate_option(ocsp_stapling, Value) when Value =:= true orelse - Value =:= false -> +validate_option(session_tickets, Value, client) -> + throw({error, + {options, role, + {session_tickets, + {Value, {client, [disabled, manual, auto]}}}}}); +validate_option(sni_fun, undefined, _) -> + undefined; +validate_option(sni_fun, Fun, _) + when is_function(Fun) -> + Fun; +validate_option(sni_hosts, [], _) -> + []; +validate_option(sni_hosts, [{Hostname, SSLOptions} | Tail], _) + when is_list(Hostname) -> + RecursiveSNIOptions = proplists:get_value(sni_hosts, SSLOptions, undefined), + case RecursiveSNIOptions of + undefined -> + [{Hostname, validate_options(SSLOptions)} | + validate_option(sni_hosts, Tail)]; + _ -> + throw({error, {options, {sni_hosts, RecursiveSNIOptions}}}) + end; +validate_option(srp_identity, undefined, _) -> + undefined; +validate_option(srp_identity, {Username, Password}, _) + when is_list(Username), + is_list(Password), Username =/= "", + length(Username) =< 255 -> + {unicode:characters_to_binary(Username), + unicode:characters_to_binary(Password)}; +validate_option(user_lookup_fun, undefined, _) -> + undefined; +validate_option(user_lookup_fun, {Fun, _} = Value, _) + when is_function(Fun, 3) -> + Value; +validate_option(use_ticket, Value, _) + when is_list(Value) -> Value; -%% The OCSP responders' certificates can be given as a suggestion and -%% will be used to verify the OCSP response. -validate_option(ocsp_responder_certs, Value) when is_list(Value) -> - [public_key:pkix_decode_cert(CertDer, plain) || CertDer <- Value, - is_binary(CertDer)]; -validate_option(ocsp_nonce, Value) when Value =:= true orelse - Value =:= false -> +validate_option(verify, Value, _) + when Value == verify_none; Value == verify_peer -> Value; -validate_option(Opt, undefined = Value) -> +validate_option(verify_fun, undefined, _) -> + undefined; +%% Backwards compatibility +validate_option(verify_fun, Fun, _) when is_function(Fun) -> + {fun(_,{bad_cert, _} = Reason, OldFun) -> + case OldFun([Reason]) of + true -> + {valid, OldFun}; + false -> + {fail, Reason} + end; + (_,{extension, _}, UserState) -> + {unknown, UserState}; + (_, valid, UserState) -> + {valid, UserState}; + (_, valid_peer, UserState) -> + {valid, UserState} + end, Fun}; +validate_option(verify_fun, {Fun, _} = Value, _) when is_function(Fun) -> + Value; +validate_option(versions, Versions, _) -> + validate_versions(Versions, Versions); +validate_option(Opt, undefined = Value, _) -> AllOpts = maps:keys(?RULES), case lists:member(Opt, AllOpts) of true -> @@ -2404,7 +2491,7 @@ validate_option(Opt, undefined = Value) -> false -> throw({error, {options, {Opt, Value}}}) end; -validate_option(Opt, Value) -> +validate_option(Opt, Value, _) -> throw({error, {options, {Opt, Value}}}). handle_cb_info({V1, V2, V3, V4}) -> diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index c263769291..b5b0a23d85 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -51,11 +51,15 @@ %%-------------------------------------------------------------------- -spec trusted_cert_and_paths([der_cert()], db_handle(), certdb_ref(), fun()) -> - [{der_cert() | unknown_ca | selfsigned_peer, [der_cert()]}]. + [{der_cert() | unknown_ca | invalid_issuer | selfsigned_peer, [der_cert()]}]. %% -%% Description: Extracts the root cert (if not presents tries to -%% look it up, if not found {bad_cert, unknown_ca} will be added verification -%% errors. Returns {RootCert | RootCertRelatedError, Path} Path = lists:reverse(Chain) -- Root +%% Description: Construct input to public_key:pkix_path_validation/3, +%% If the ROOT cert is not found {bad_cert, unknown_ca} will be returned +%% instead of the ROOT cert to be handled as a path validation error +%% by the verify_fun. +%% Returns {RootCert | RootCertRelatedError, Path} +%% Note: Path = lists:reverse(Chain) -- Root, that is on the peer cert +%% always comes first in the chain but last in the path. %%-------------------------------------------------------------------- trusted_cert_and_paths([Peer] = Chain, CertDbHandle, CertDbRef, PartialChainHandler) -> OtpCert = public_key:pkix_decode_cert(Peer, otp), @@ -67,6 +71,10 @@ trusted_cert_and_paths([Peer] = Chain, CertDbHandle, CertDbRef, PartialChainHan CertDbHandle, CertDbRef)] end; trusted_cert_and_paths(Chain, CertDbHandle, CertDbRef, PartialChainHandler) -> + %% Construct possible certificate paths from the chain certificates. + %% If the chain contains extraneous certificates there could be + %% more than one possible path such chains might be used to phase out + %% an old certificate. Paths = paths(Chain, CertDbHandle, CertDbRef), lists:map(fun(Path) -> case handle_partial_chain(Path, PartialChainHandler, CertDbHandle, CertDbRef) of @@ -439,24 +447,23 @@ paths([Root], _, _, _, Path) -> paths([Cert1, Cert2 | Rest], Chain, CertDbHandle, CertDbRef, Path) -> case public_key:pkix_is_issuer(Cert1, Cert2) of true -> + %% Chain orded so far paths([Cert2 | Rest], Chain, CertDbHandle, CertDbRef, [Cert1 | Path]); false -> + %% Chain is unorded and/or contains extraneous certificates unorded_or_extraneous(Chain, CertDbHandle, CertDbRef) end. unorded_or_extraneous([Peer | FalseChain], CertDbHandle, CertDbRef) -> - case extraneous_certs(FalseChain) of - [_] -> - [path_candidate(Peer, FalseChain, CertDbHandle, CertDbRef)]; - ChainCandidates -> - lists:map(fun(Candidate) -> - path_candidate(Peer, Candidate, CertDbHandle, CertDbRef) - end, - ChainCandidates) - end. - -path_candidate(Peer, CAs, CertDbHandle, _CertDbRef) -> - {ok, ExtractedCerts} = ssl_pkix_db:extract_trusted_certs({der, CAs}), + ChainCandidates = extraneous_chains(FalseChain), + lists:map(fun(Candidate) -> + path_candidate(Peer, Candidate, CertDbHandle, CertDbRef) + end, + ChainCandidates). + +path_candidate(Peer, ChainCandidateCAs, CertDbHandle, _CertDbRef) -> + {ok, ExtractedCerts} = ssl_pkix_db:extract_trusted_certs({der, ChainCandidateCAs}), + %% certificate_chain/4 will make sure the chain is ordered case certificate_chain(Peer, CertDbHandle, ExtractedCerts, []) of {ok, undefined, Chain} -> lists:reverse(Chain); @@ -466,11 +473,13 @@ path_candidate(Peer, CAs, CertDbHandle, _CertDbRef) -> handle_partial_chain([IssuerCert| Rest] = Path, PartialChainHandler, CertDbHandle, CertDbRef) -> case public_key:pkix_is_self_signed(IssuerCert) of - true -> %% IssuerCert = ROOT + true -> %% IssuerCert = ROOT (That is ROOT was included in chain) {ok, {SerialNr, IssuerId}} = public_key:pkix_issuer_id(IssuerCert, self), case ssl_manager:lookup_trusted_cert(CertDbHandle, CertDbRef, SerialNr, IssuerId) of - {ok, {IssuerCert, _}} -> + {ok, {IssuerCert, _}} -> %% Match sent ROOT to trusted ROOT maybe_shorten_path(Path, PartialChainHandler, {IssuerCert, Rest}); + {ok, _} -> %% Did not match trusted ROOT + maybe_shorten_path(Path, PartialChainHandler, {invalid_issuer, Path}); _ -> maybe_shorten_path(Path, PartialChainHandler, {unknown_ca, Path}) end; @@ -481,8 +490,8 @@ handle_partial_chain([IssuerCert| Rest] = Path, PartialChainHandler, CertDbHandl case ssl_manager:lookup_trusted_cert(CertDbHandle, CertDbRef, SerialNr, IssuerId) of {ok, {NewIssuerCert, _}} -> case public_key:pkix_is_self_signed(NewIssuerCert) of - true -> %% IssuerCert = ROOT - maybe_shorten_path(Path, PartialChainHandler, {IssuerCert, Rest}); + true -> %% NewIssuerCert is a trusted ROOT cert + maybe_shorten_path([NewIssuerCert | Path], PartialChainHandler, {NewIssuerCert, Path}); false -> maybe_shorten_path([NewIssuerCert | Path], PartialChainHandler, {unknown_ca, [NewIssuerCert | Path]}) @@ -496,6 +505,11 @@ handle_partial_chain([IssuerCert| Rest] = Path, PartialChainHandler, CertDbHandl end. maybe_shorten_path(Path, PartialChainHandler, Default) -> + %% This function might shorthen the + %% certificate path to be validated with + %% public_key:pkix_path_validation by letting + %% the user put its trust in an intermidate cert + %% from the certifcate chain sent by the peer. try PartialChainHandler(Path) of {trusted_ca, Root} -> new_trusteded_path(Root, Path, Default); @@ -515,6 +529,8 @@ new_trusteded_path(_, [], Default) -> Default. handle_incomplete_chain([PeerCert| _] = Chain0, PartialChainHandler, Default, CertDbHandle, CertDbRef) -> + %% We received an incomplete chain, that is not all certs expected to be present are present. + %% See if we have the certificates to rebuild it. case certificate_chain(PeerCert, CertDbHandle, CertDbRef) of {ok, _, [PeerCert | _] = Chain} when Chain =/= Chain0 -> %% Chain candidate found handle_partial_chain(lists:reverse(Chain), PartialChainHandler, CertDbHandle, CertDbRef); @@ -522,38 +538,61 @@ handle_incomplete_chain([PeerCert| _] = Chain0, PartialChainHandler, Default, Ce Default end. -extraneous_certs(Certs) -> +extraneous_chains(Certs) -> + %% If some certs claim to be the same cert that is have the same + %% subject field we should create a list of possible chain certs + %% for each such cert. Only one chain, if any, should be + %% verifiable using available ROOT certs. Subjects = [{subject(Cert), Cert} || Cert <- Certs], - SortedCerts = sort_by_subject(Subjects), - extraneous_certs(SortedCerts,[[]]). + Duplicates = find_duplicates(Subjects), + %% Number of certs with duplicates (same subject) has been limited + %% to two and the maximum number of combinations is limited to 4. + build_candidates(Duplicates, 2, 4). + +build_candidates(Map, Duplicates, Combinations) -> + Subjects = maps:keys(Map), + build_candidates(Subjects, Map, Duplicates, 1, Combinations, []). +%% +build_candidates([], _, _, _, _, Acc) -> + Acc; +build_candidates([H|T], Map, Duplicates, Combinations, Max, Acc0) -> + case maps:get(H, Map) of + {Certs, Counter} when Counter > 1 andalso + Duplicates > 0 andalso + Counter * Combinations =< Max -> + case Acc0 of + [] -> + Acc = [[Cert] || Cert <- Certs], + build_candidates(T, Map, Duplicates - 1, Combinations * Counter, Max, Acc); + _Else -> + Acc = [[Cert|L] || Cert <- Certs, L <- Acc0], + build_candidates(T, Map, Duplicates - 1, Combinations * Counter, Max, Acc) + end; + {[Cert|_], _} -> + case Acc0 of + [] -> + Acc = [[Cert]], + build_candidates(T, Map, Duplicates, Combinations, Max, Acc); + _Else -> + Acc = [[Cert|L] || L <- Acc0], + build_candidates(T, Map, Duplicates, Combinations, Max, Acc) + end + end. -extraneous_certs([_], Acc) -> +find_duplicates(Chain) -> + find_duplicates(Chain, #{}). +%% +find_duplicates([], Acc) -> Acc; -extraneous_certs([CA0, CA1 | Certs], Acc) -> - {_, Name1} = public_key:pkix_subject_id(CA0), - {_, Name2} = public_key:pkix_subject_id(CA1), - NormName1 = public_key:pkix_normalize_name(Name1), - NormName2 = public_key:pkix_normalize_name(Name2), - case NormName1 of - NormName2 -> - extraneous_certs([CA1| Certs], path_alts(CA0, CA1, Certs, Acc)); - _ -> - extraneous_certs([CA1 | Certs], lists:map(fun(Candidate) -> [CA0 | Candidate] end, Acc)) +find_duplicates([{Subject, Cert}|T], Acc) -> + case maps:get(Subject, Acc, none) of + none -> + find_duplicates(T, Acc#{Subject => {[Cert], 1}}); + {Certs, Counter} -> + find_duplicates(T, Acc#{Subject => {[Cert|Certs], Counter + 1}}) end. -path_alts(CA1, CA2, CAs, []) -> - [[CA1 | CAs], [CA2 | CAs]]; -path_alts(CA1, CA2, CAs, [[]]) -> - [[CA1 | CAs], [CA2 | CAs]]; -path_alts(CA1, CA2, _, [Alt1, Alt2 |Path]) -> - path_alts(CA1, CA2, Alt1, Path) ++ path_alts(CA1, CA2, Alt2, Path). - -sort_by_subject(Subjects) -> - Sort = lists:keysort(1, Subjects), - lists:map(fun({_, Cert}) -> Cert end, Sort). - subject(Cert) -> - OTPCert =public_key:pkix_decode_cert(Cert, otp), - TBSCert = OTPCert#'OTPCertificate'.tbsCertificate, - Subject = TBSCert#'OTPTBSCertificate'.subject, - public_key:pkix_normalize_name(Subject). + {_Serial,Subject} = public_key:pkix_subject_id(Cert), + Subject. + diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 6bc7f6e353..85042e8612 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -70,7 +70,8 @@ hash_size/1, effective_key_bits/1, key_material/1, - signature_algorithm_to_scheme/1]). + signature_algorithm_to_scheme/1, + bulk_cipher_algorithm/1]). %% RFC 8446 TLS 1.3 -export([generate_client_shares/1, @@ -530,13 +531,15 @@ rsa_suites(0) -> ?TLS_RSA_WITH_AES_128_CBC_SHA, ?TLS_RSA_WITH_3DES_EDE_CBC_SHA ]; -rsa_suites(N) when N =< 4 -> +rsa_suites(N) when N >= 3 -> [ ?TLS_RSA_WITH_AES_256_GCM_SHA384, ?TLS_RSA_WITH_AES_256_CBC_SHA256, ?TLS_RSA_WITH_AES_128_GCM_SHA256, ?TLS_RSA_WITH_AES_128_CBC_SHA256 - ]. + ]; +rsa_suites(_) -> + []. %%-------------------------------------------------------------------- -spec filter(undefined | binary(), [ssl_cipher_format:cipher_suite()], diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 0a7c4560fb..9f2141b6f8 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -260,6 +260,18 @@ %% TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = { 0xC0, 0x0A } -define(TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#0A)>>). +%% TLS_ECDHE_ECDSA_WITH_AES_128_CCM = {0xC0,0xAC} +-define(TLS_ECDHE_ECDSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#AC)>>). + +%% TLS_ECDHE_ECDSA_WITH_AES_256_CCM = {0xC0,0xAD} +-define(TLS_ECDHE_ECDSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#AD)>>). + +%% TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = {0xC0,0xAE} +-define(TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#AE)>>). + +%% TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = {0xC0,0xAF} +-define(TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#AF)>>). + %% ECDH_RSA %% TLS_ECDH_RSA_WITH_NULL_SHA = { 0xC0, 0x0B } diff --git a/lib/ssl/src/ssl_cipher_format.erl b/lib/ssl/src/ssl_cipher_format.erl index 49855f4b74..589b0facf8 100644 --- a/lib/ssl/src/ssl_cipher_format.erl +++ b/lib/ssl/src/ssl_cipher_format.erl @@ -77,13 +77,13 @@ suite_map_to_str(#{key_exchange := Kex, cipher := Cipher, mac := aead, prf := PRF}) -> - "TLS_" ++ string:to_upper(atom_to_list(Kex)) ++ + "TLS_" ++ kex_str(Kex) ++ "_WITH_" ++ string:to_upper(atom_to_list(Cipher)) ++ - "_" ++ string:to_upper(atom_to_list(PRF)); + prf_str("_", PRF); suite_map_to_str(#{key_exchange := Kex, cipher := Cipher, mac := Mac}) -> - "TLS_" ++ string:to_upper(atom_to_list(Kex)) ++ + "TLS_" ++ kex_str(Kex) ++ "_WITH_" ++ string:to_upper(atom_to_list(Cipher)) ++ "_" ++ string:to_upper(atom_to_list(Mac)). @@ -97,12 +97,6 @@ suite_str_to_map(SuiteStr)-> case string:split(Str0, "_WITH_") of [Rest] -> tls_1_3_suite_str_to_map(Rest); - [Prefix, Kex | Rest] when Prefix == "SPR"; - Prefix == "PSK"; - Prefix == "DHE"; - Prefix == "ECDHE" - -> - pre_tls_1_3_suite_str_to_map(Prefix ++ "_" ++ Kex, Rest); [Kex| Rest] -> pre_tls_1_3_suite_str_to_map(Kex, Rest) end. @@ -116,26 +110,36 @@ suite_map_to_openssl_str(#{key_exchange := null} = Suite) -> suite_map_to_str(Suite); suite_map_to_openssl_str(#{key_exchange := rsa = Kex, cipher := Cipher, - mac := Mac}) when Cipher == "des_cbc"; - Cipher == "3des_ede_cbc" -> + mac := aead, + prf := PRF}) when PRF =/= default_prf -> + openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ + "-" ++ string:to_upper(atom_to_list(PRF)); +suite_map_to_openssl_str(#{key_exchange := Kex, + cipher := Cipher, + mac := Mac}) when (Kex == rsa) orelse + (Kex == srp_anon) + andalso + (Cipher == "des_cbc") orelse + (Cipher == "3des_ede_cbc") -> openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ "-" ++ string:to_upper(atom_to_list(Mac)); suite_map_to_openssl_str(#{key_exchange := Kex, cipher := chacha20_poly1305 = Cipher, - mac := aead}) -> - openssl_suite_start(string:to_upper(atom_to_list(Kex)), Cipher) - ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))); + mac := aead, + prf := sha256}) -> + openssl_suite_start(kex_str(Kex), Cipher) + ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))); suite_map_to_openssl_str(#{key_exchange := Kex, cipher := Cipher, mac := aead, prf := PRF}) -> - openssl_suite_start(string:to_upper(atom_to_list(Kex)), Cipher) + openssl_suite_start(kex_str(Kex), Cipher) ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ - "-" ++ string:to_upper(atom_to_list(PRF)); + prf_str("-", PRF); suite_map_to_openssl_str(#{key_exchange := Kex, - cipher := Cipher, - mac := Mac}) -> - openssl_suite_start(string:to_upper(atom_to_list(Kex)), Cipher) + cipher := Cipher, + mac := Mac}) -> + openssl_suite_start(kex_str(Kex), Cipher) ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ "-" ++ string:to_upper(atom_to_list(Mac)). @@ -148,14 +152,20 @@ suite_openssl_str_to_map("DES-CBC3-SHA") -> suite_str_to_map("TLS_RSA_WITH_3DES_EDE_CBC_SHA"); suite_openssl_str_to_map("SRP-DSS-DES-CBC3-SHA") -> suite_str_to_map("TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA"); +suite_openssl_str_to_map("ADH" ++ Rest) -> + suite_openssl_str_to_map("DH-anon", Rest); +suite_openssl_str_to_map("AECDH" ++ Rest) -> + suite_openssl_str_to_map("ECDH-anon", Rest); suite_openssl_str_to_map("EDH-RSA" ++ Rest) -> suite_openssl_str_to_map("DHE-RSA", Rest); +suite_openssl_str_to_map("EDH-DSS-" ++ Rest) -> + suite_openssl_str_to_map("DHE-DSS", Rest); suite_openssl_str_to_map("DHE-RSA-" ++ Rest) -> suite_openssl_str_to_map("DHE-RSA", Rest); suite_openssl_str_to_map("DHE-DSS-" ++ Rest) -> suite_openssl_str_to_map("DHE-DSS", Rest); -suite_openssl_str_to_map("EDH-DSS-" ++ Rest) -> - suite_openssl_str_to_map("DHE-DSS", Rest); +suite_openssl_str_to_map("DHE-PSK-" ++ Rest) -> + suite_openssl_str_to_map("DHE-PSK", Rest); suite_openssl_str_to_map("DES" ++ _ = Rest) -> suite_openssl_str_to_map("RSA", Rest); suite_openssl_str_to_map("AES" ++ _ = Rest) -> @@ -174,8 +184,6 @@ suite_openssl_str_to_map("RSA-PSK-" ++ Rest) -> suite_openssl_str_to_map("RSA-PSK", Rest); suite_openssl_str_to_map("RSA-" ++ Rest) -> suite_openssl_str_to_map("RSA", Rest); -suite_openssl_str_to_map("DHE-PSK-" ++ Rest) -> - suite_openssl_str_to_map("DHE-PSK", Rest); suite_openssl_str_to_map("ECDHE-PSK-" ++ Rest) -> suite_openssl_str_to_map("ECDHE-PSK", Rest); suite_openssl_str_to_map("PSK-" ++ Rest) -> @@ -348,12 +356,12 @@ suite_bin_to_map(?TLS_DH_anon_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dh_anon, cipher => aes_128_cbc, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_DH_anon_WITH_AES_256_CBC_SHA256) -> #{key_exchange => dh_anon, cipher => aes_256_cbc, mac => sha256, - prf => default_prf}; + prf => sha256}; %%% PSK Cipher Suites RFC 4279 suite_bin_to_map(?TLS_PSK_WITH_RC4_128_SHA) -> #{key_exchange => psk, @@ -466,7 +474,7 @@ suite_bin_to_map(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => psk, cipher => aes_128_cbc, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => psk, cipher => aes_256_cbc, @@ -476,7 +484,7 @@ suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dhe_psk, cipher => aes_128_cbc, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => dhe_psk, cipher => aes_256_cbc, @@ -506,7 +514,7 @@ suite_bin_to_map(?TLS_DHE_PSK_WITH_NULL_SHA256) -> #{key_exchange => dhe_psk, cipher => null, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_DHE_PSK_WITH_NULL_SHA384) -> #{key_exchange => dhe_psk, cipher => null, @@ -516,7 +524,7 @@ suite_bin_to_map(?TLS_RSA_PSK_WITH_NULL_SHA256) -> #{key_exchange => rsa_psk, cipher => null, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_RSA_PSK_WITH_NULL_SHA384) -> #{key_exchange => rsa_psk, cipher => null, @@ -547,7 +555,7 @@ suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_cbc, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdhe_psk, cipher => aes_256_cbc, @@ -557,7 +565,7 @@ suite_bin_to_map(?TLS_ECDHE_PSK_WITH_NULL_SHA256) -> #{key_exchange => ecdhe_psk, cipher => null, mac => sha256, - prf => default_prf}; + prf => sha256}; suite_bin_to_map(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> #{key_exchange => ecdhe_psk, cipher => null, mac => sha384, @@ -566,22 +574,22 @@ suite_bin_to_map(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_ccm, - mac => null, + mac => aead, prf => sha256}; suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_ccm_8, - mac => null, + mac => aead, prf => sha256}; %%% SRP Cipher Suites RFC 5054 suite_bin_to_map(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> @@ -680,6 +688,26 @@ suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) -> cipher => aes_256_cbc, mac => sha, prf => default_prf}; +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_128_CCM) -> + #{key_exchange => ecdhe_ecdsa, + cipher => aes_128_ccm, + mac => aead, + prf => default_prf}; +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_CCM) -> + #{key_exchange => ecdhe_ecdsa, + cipher => aes_256_ccm, + mac => aead, + prf => default_prf}; +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8) -> + #{key_exchange => ecdhe_ecdsa, + cipher => aes_128_ccm_8, + mac => aead, + prf => default_prf}; +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8) -> + #{key_exchange => ecdhe_ecdsa, + cipher => aes_256_ccm_8, + mac => aead, + prf => default_prf}; suite_bin_to_map(?TLS_ECDH_RSA_WITH_NULL_SHA) -> #{key_exchange => ecdh_rsa, cipher => null, @@ -840,7 +868,7 @@ suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> suite_bin_to_map(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_dss, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_bin_to_map(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_dss, @@ -902,42 +930,42 @@ suite_bin_to_map(?TLS_PSK_WITH_AES_128_CCM) -> #{key_exchange => psk, cipher => aes_128_ccm, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_PSK_WITH_AES_256_CCM) -> #{key_exchange => psk, cipher => aes_256_ccm, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_CCM) -> #{key_exchange => dhe_psk, cipher => aes_128_ccm, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_CCM) -> #{key_exchange => dhe_psk, cipher => aes_256_ccm, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_PSK_WITH_AES_128_CCM_8) -> #{key_exchange => psk, cipher => aes_128_ccm_8, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_PSK_WITH_AES_256_CCM_8) -> #{key_exchange => psk, cipher => aes_256_ccm_8, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_PSK_DHE_WITH_AES_128_CCM_8) -> #{key_exchange => dhe_psk, cipher => aes_128_ccm_8, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(?TLS_PSK_DHE_WITH_AES_256_CCM_8) -> #{key_exchange => dhe_psk, cipher => aes_256_ccm_8, mac => aead, - prf => sha256}; + prf => default_prf}; suite_bin_to_map(#{key_exchange := psk_dhe, cipher := aes_256_ccm_8, mac := aead, @@ -1297,22 +1325,22 @@ suite_map_to_bin(#{key_exchange := ecdhe_psk, %%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256; suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_ccm_8, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_ccm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; %%% SRP Cipher Suites RFC 5054 @@ -1393,6 +1421,22 @@ suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA; +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, + cipher := aes_128_ccm, + mac := aead}) -> + ?TLS_ECDHE_ECDSA_WITH_AES_128_CCM; +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, + cipher := aes_256_ccm, + mac := aead}) -> + ?TLS_ECDHE_ECDSA_WITH_AES_256_CCM; +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, + cipher := aes_128_ccm_8, + mac := aead}) -> + ?TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8; +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, + cipher := aes_256_ccm_8, + mac := aead}) -> + ?TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8; suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := null, mac := sha}) -> @@ -1616,22 +1660,22 @@ suite_map_to_bin(#{key_exchange := dhe_rsa, suite_map_to_bin(#{key_exchange := psk, cipher := aes_128_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_WITH_AES_128_CCM; suite_map_to_bin(#{key_exchange := psk, cipher := aes_256_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_WITH_AES_256_CCM; suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_128_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_DHE_PSK_WITH_AES_128_CCM; suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_256_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_DHE_PSK_WITH_AES_256_CCM; suite_map_to_bin(#{key_exchange := rsa, cipher := aes_128_ccm, @@ -1641,7 +1685,7 @@ suite_map_to_bin(#{key_exchange := rsa, suite_map_to_bin(#{key_exchange := rsa, cipher := aes_256_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_RSA_WITH_AES_256_CCM; suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_128_ccm, @@ -1651,48 +1695,48 @@ suite_map_to_bin(#{key_exchange := dhe_rsa, suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_256_ccm, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_DHE_RSA_WITH_AES_256_CCM; suite_map_to_bin(#{key_exchange := psk, cipher := aes_128_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_WITH_AES_128_CCM_8; suite_map_to_bin(#{key_exchange := psk, cipher := aes_256_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_WITH_AES_256_CCM_8; suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_128_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_DHE_WITH_AES_128_CCM_8; suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_256_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_PSK_DHE_WITH_AES_256_CCM_8; suite_map_to_bin(#{key_exchange := rsa, cipher := aes_128_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_RSA_WITH_AES_128_CCM_8; suite_map_to_bin(#{key_exchange := rsa, cipher := aes_256_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_RSA_WITH_AES_256_CCM_8; suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_128_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_DHE_RSA_WITH_AES_128_CCM_8; suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_256_ccm_8, mac := aead, - prf := sha256}) -> + prf := default_prf}) -> ?TLS_DHE_RSA_WITH_AES_256_CCM_8; %% TLS 1.3 Cipher Suites RFC8446 @@ -1740,21 +1784,42 @@ pre_tls_1_3_suite_str_to_map(KexStr, Rest) -> cipher => Cipher, prf => Prf }. - -cipher_str_to_algs(_, CipherStr, "CCM"= End) -> %% PRE TLS 1.3 - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), - {Cipher, aead, sha256}; -cipher_str_to_algs(_, CipherStr, "8" = End) -> %% PRE TLS 1.3 - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), - {Cipher, aead, sha256}; -cipher_str_to_algs(_, CipherStr, "CHACHA20_POLY1305" = End) -> %% PRE TLS 1.3 - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), - {Cipher, aead, sha256}; -cipher_str_to_algs(_, CipherStr0, "") -> %% TLS 1.3 + +kex_str(srp_dss) -> + "SRP_SHA_DSS"; +kex_str(srp_rsa) -> + "SRP_SHA_RSA"; +kex_str(srp_anon) -> + "SRP_SHA"; +kex_str(dh_anon) -> + "DH_anon"; +kex_str(ecdh_anon) -> + "ECDH_anon"; +kex_str(Kex) -> + string:to_upper(atom_to_list(Kex)). + +prf_str(_, default_prf) -> + ""; +prf_str(Prefix, PRF) -> + Prefix ++ string:to_upper(atom_to_list(PRF)). + +cipher_str_to_algs(any, CipherStr0, "") -> %% TLS 1.3 [CipherStr, AlgStr] = string:split(CipherStr0, "_", trailing), Hash = algo_str_to_atom(AlgStr), Cipher = algo_str_to_atom(CipherStr), {Cipher, aead, Hash}; +cipher_str_to_algs(_Kex, CipherStr, "CCM"= End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, default_prf}; +cipher_str_to_algs(_Kex, CipherStr, "GCM"= End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, default_prf}; +cipher_str_to_algs(_Kex, CipherStr, "8" = End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, default_prf}; +cipher_str_to_algs(_Kex, "CHACHA20_POLY1305" = CipherStr, "") -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr), + {Cipher, aead, sha256}; cipher_str_to_algs(Kex, CipherStr, HashStr) -> %% PRE TLS 1.3 Hash = algo_str_to_atom(HashStr), Cipher = algo_str_to_atom(CipherStr), @@ -1796,48 +1861,80 @@ openssl_is_aead_cipher("CHACHA20-POLY1305") -> openssl_is_aead_cipher(CipherStr) -> case string:split(CipherStr, "-", trailing) of [_, Rest] -> - (Rest == "GCM") orelse (Rest == "CCM") orelse (Rest == "8"); + (Rest == "GCM") orelse (Rest == "CCM") orelse (Rest == "CCM8"); [_] -> false end. algo_str_to_atom("SRP_SHA_DSS") -> srp_dss; +algo_str_to_atom("SRP_SHA_RSA") -> + srp_rsa; +algo_str_to_atom("SRP_SHA") -> + srp_anon; +algo_str_to_atom("SRP") -> + srp_anon; algo_str_to_atom(AlgoStr) -> erlang:list_to_existing_atom(string:to_lower(AlgoStr)). +openssl_cipher_name(Kex, "3DES_EDE_CBC" ++ _) when Kex == ecdhe_psk; + Kex == srp_anon; + Kex == psk; + Kex == dhe_psk -> + "3DES-EDE-CBC"; openssl_cipher_name(_, "3DES_EDE_CBC" ++ _) -> "DES-CBC3"; openssl_cipher_name(Kex, "AES_128_CBC" ++ _ = CipherStr) when Kex == rsa; Kex == dhe_rsa; + Kex == dhe_dss; + Kex == ecdh_rsa; Kex == ecdhe_rsa; - Kex == ecdhe_ecdsa -> + Kex == ecdh_ecdsa; + Kex == ecdhe_ecdsa; + Kex == ecdh_anon; + Kex == dh_anon -> openssl_name_concat(CipherStr); openssl_cipher_name(Kex, "AES_256_CBC" ++ _ = CipherStr) when Kex == rsa; Kex == dhe_rsa; + Kex == dhe_dss; + Kex == ecdh_rsa; Kex == ecdhe_rsa; - Kex == ecdhe_ecdsa -> + Kex == ecdh_ecdsa; + Kex == ecdhe_ecdsa; + Kex == ecdh_anon; + Kex == dh_anon -> openssl_name_concat(CipherStr); -openssl_cipher_name(Kex, "AES_128_CBC" ++ _ = CipherStr) when Kex == srp; +openssl_cipher_name(Kex, "AES_128_CBC" ++ _ = CipherStr) when Kex == srp_anon; Kex == srp_rsa -> lists:append(string:replace(CipherStr, "_", "-", all)); -openssl_cipher_name(Kex, "AES_256_CBC" ++ _ = CipherStr) when Kex == srp; +openssl_cipher_name(Kex, "AES_256_CBC" ++ _ = CipherStr) when Kex == srp_anon; Kex == srp_rsa -> lists:append(string:replace(CipherStr, "_", "-", all)); openssl_cipher_name(_, "AES_128_CBC" ++ _ = CipherStr) -> openssl_name_concat(CipherStr) ++ "-CBC"; openssl_cipher_name(_, "AES_256_CBC" ++ _ = CipherStr) -> openssl_name_concat(CipherStr) ++ "-CBC"; +openssl_cipher_name(_, "AES_128_GCM_8") -> + openssl_name_concat("AES_128_GCM") ++ "-GCM8"; +openssl_cipher_name(_, "AES_256_GCM_8") -> + openssl_name_concat("AES_256_GCM") ++ "-GCM8"; +openssl_cipher_name(_, "AES_128_CCM_8") -> + openssl_name_concat("AES_128_CCM") ++ "-CCM8"; +openssl_cipher_name(_, "AES_256_CCM_8") -> + openssl_name_concat("AES_256_CCM") ++ "-CCM8"; openssl_cipher_name(_, "AES_128_GCM" ++ _ = CipherStr) -> openssl_name_concat(CipherStr) ++ "-GCM"; openssl_cipher_name(_, "AES_256_GCM" ++ _ = CipherStr) -> openssl_name_concat(CipherStr) ++ "-GCM"; +openssl_cipher_name(_, "AES_128_CCM" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-CCM"; +openssl_cipher_name(_, "AES_256_CCM" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-CCM"; openssl_cipher_name(_, "RC4" ++ _) -> "RC4"; openssl_cipher_name(_, CipherStr) -> lists:append(string:replace(CipherStr, "_", "-", all)). - openssl_suite_start(Kex, Cipher) -> case openssl_kex_name(Kex, Cipher) of "" -> @@ -1848,6 +1945,16 @@ openssl_suite_start(Kex, Cipher) -> openssl_kex_name("RSA", _) -> ""; +openssl_kex_name("DH_anon", _) -> + "ADH"; +openssl_kex_name("ECDH_anon", _) -> + "AECDH"; +openssl_kex_name("SRP_SHA", _) -> + "SRP"; +openssl_kex_name("SRP_SHA_RSA", _) -> + "SRP-RSA"; +openssl_kex_name("SRP_SHA_DSS", _) -> + "SRP-DSS"; openssl_kex_name("DHE_RSA", Cipher) when Cipher == des_cbc; Cipher == '3des_ede_cbc' -> "EDH-RSA"; @@ -1856,7 +1963,9 @@ openssl_kex_name(Kex, _) -> kex_name_from_openssl(Kex) -> case lists:append(string:replace(Kex, "-", "_", all)) of "EDH-RSA" -> - "DHE_RSA"; + "DHE_RSA"; + "SRP" -> + "SRP_SHA"; Str -> Str end. @@ -1865,26 +1974,30 @@ cipher_name_from_openssl("AES128") -> "AES_128_CBC"; cipher_name_from_openssl("AES256") -> "AES_256_CBC"; -cipher_name_from_openssl("AES128-CBC") -> - "AES_128_CBC"; -cipher_name_from_openssl("AES256-CBC") -> - "AES_256_CBC"; -cipher_name_from_openssl("AES-128-CBC") -> - "AES_128_CBC"; -cipher_name_from_openssl("AES-256-CBC") -> - "AES_256_CBC"; -cipher_name_from_openssl("AES128-GCM") -> - "AES_128_GCM"; -cipher_name_from_openssl("AES256-GCM") -> - "AES_256_GCM"; +cipher_name_from_openssl("AES128-CCM8") -> + "AES_128_CCM_8"; +cipher_name_from_openssl("AES256-CCM8") -> + "AES_256_CCM_8"; +cipher_name_from_openssl("AES128-" ++ Suffix) -> + "AES_128_" ++ lists:append(string:replace(Suffix, "-", "_", all)); +cipher_name_from_openssl("AES256-" ++ Suffix) -> + "AES_256_" ++ lists:append(string:replace(Suffix, "-", "_", all)); +cipher_name_from_openssl("AES128_" ++ Suffix) -> + "AES_128_" ++ Suffix; +cipher_name_from_openssl("AES256_" ++ Suffix) -> + "AES_256_" ++ Suffix; cipher_name_from_openssl("DES-CBC") -> "DES_CBC"; cipher_name_from_openssl("DES-CBC3") -> "3DES_EDE_CBC"; +cipher_name_from_openssl("3DES-EDE-CBC") -> + "3DES_EDE_CBC"; cipher_name_from_openssl("RC4") -> "RC4_128"; +cipher_name_from_openssl("CHACHA20-POLY1305") -> + "CHACHA20_POLY1305"; cipher_name_from_openssl(Str) -> - Str. + lists:append(string:replace(Str, "-", "_", all)). openssl_name_concat(Str0) -> [Str, _] = string:split(Str0, "_", trailing), @@ -1894,8 +2007,8 @@ openssl_name_concat(Str0) -> suite_openssl_str_to_map(Kex0, Rest) -> Kex = algo_str_to_atom(kex_name_from_openssl(Kex0)), - [CipherStr, AlgStr] = string:split(Rest, "-", trailing), - {Cipher, Mac, Prf} = openssl_cipher_str_to_algs(Kex, CipherStr, AlgStr), + [Part1, Part2] = string:split(Rest, "-", trailing), + {Cipher, Mac, Prf} = openssl_cipher_str_to_algs(Kex, Part1, Part2), #{key_exchange => Kex, mac => Mac, cipher => Cipher, @@ -1903,19 +2016,25 @@ suite_openssl_str_to_map(Kex0, Rest) -> }. %% Does only need own implementation PRE TLS 1.3 -openssl_cipher_str_to_algs(_, CipherStr, "CCM"= End) -> - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), - {Cipher, aead, sha256}; -openssl_cipher_str_to_algs(_, CipherStr, "8" = End) -> - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), +openssl_cipher_str_to_algs(_, Part1, "CCM" = End) -> + Cipher = algo_str_to_atom(cipher_name_from_openssl(Part1 ++ "_" ++ End)), + {Cipher, aead, default_prf}; +openssl_cipher_str_to_algs(_, Part1, "GCM" = End) -> + Cipher = algo_str_to_atom(cipher_name_from_openssl(Part1 ++ "_" ++ End)), + {Cipher, aead, default_prf}; +openssl_cipher_str_to_algs(_, Part2, "CCM8") -> + Cipher = algo_str_to_atom(cipher_name_from_openssl(Part2 ++ "-CCM-8")), + {Cipher, aead, default_prf}; +openssl_cipher_str_to_algs(_, Part2, "GCM8") -> + Cipher = algo_str_to_atom(cipher_name_from_openssl(Part2 ++ "-GCM-8")), + {Cipher, aead, default_prf}; +openssl_cipher_str_to_algs(_, "CHACHA20", "POLY1305") -> + Cipher = chacha20_poly1305, {Cipher, aead, sha256}; -openssl_cipher_str_to_algs(_, CipherStr, "POLY1305" = End) -> - Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), - {Cipher, aead, sha256}; -openssl_cipher_str_to_algs(Kex, CipherStr, HashStr) -> - Hash = algo_str_to_atom(HashStr), - Cipher = algo_str_to_atom(cipher_name_from_openssl(string:strip(CipherStr, left, $-))), - case openssl_is_aead_cipher(CipherStr) of +openssl_cipher_str_to_algs(Kex, Part1, Part2) -> + Hash = algo_str_to_atom(Part2), + Cipher = algo_str_to_atom(cipher_name_from_openssl(string:strip(Part1, left, $-))), + case openssl_is_aead_cipher(Part1) of true -> {Cipher, aead, Hash}; false -> diff --git a/lib/ssl/src/ssl_config.erl b/lib/ssl/src/ssl_config.erl index ac072564bb..e03f117c82 100644 --- a/lib/ssl/src/ssl_config.erl +++ b/lib/ssl/src/ssl_config.erl @@ -26,8 +26,18 @@ -include("ssl_connection.hrl"). -include_lib("public_key/include/public_key.hrl"). --export([init/2]). +-define(DEFAULT_MAX_SESSION_CACHE, 1000). +-export([init/2, + pre_1_3_session_opts/0, + get_max_early_data_size/0, + get_ticket_lifetime/0, + get_ticket_store_size/0 + ]). + +%%==================================================================== +%% Internal application API +%%==================================================================== init(#{erl_dist := ErlDist, key := Key, keyfile := KeyFile, @@ -44,6 +54,50 @@ init(#{erl_dist := ErlDist, DHParams = init_diffie_hellman(PemCache, DH, DHFile, Role), {ok, Config#{private_key => PrivateKey, dh_params => DHParams}}. +pre_1_3_session_opts() -> + CbOpts = case application:get_env(ssl, session_cb) of + {ok, Cb} when is_atom(Cb) -> + InitArgs = session_cb_init_args(), + #{session_cb => Cb, + session_cb_init_args => InitArgs}; + _ -> + #{session_cb => ssl_server_session_cache_db, + session_cb_init_args => []} + end, + LifeTime = session_lifetime(), + Max = max_session_cache_size(), + [CbOpts#{lifetime => LifeTime, max => Max}]. + + +get_ticket_lifetime() -> + case application:get_env(ssl, server_session_ticket_lifetime) of + {ok, Seconds} when is_integer(Seconds) andalso + Seconds =< 604800 -> %% MUST be less than 7 days + Seconds; + _ -> + 7200 %% Default 2 hours + end. + + +get_ticket_store_size() -> + case application:get_env(ssl, server_session_ticket_store_size) of + {ok, Size} when is_integer(Size) -> + Size; + _ -> + 1000 + end. + +get_max_early_data_size() -> + case application:get_env(ssl, server_session_ticket_max_early_data) of + {ok, Size} when is_integer(Size) -> + Size; + _ -> + ?DEFAULT_MAX_EARLY_DATA_SIZE + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== init_manager_name(false) -> put(ssl_manager, ssl_manager:name(normal)), put(ssl_pem_cache, ssl_pem_cache:name(normal)); @@ -176,3 +230,29 @@ init_diffie_hellman(DbHandle,_, DHParamFile, server) -> _:Reason -> file_error(DHParamFile, {dhfile, Reason}) end. + + +session_cb_init_args() -> + case application:get_env(ssl, session_cb_init_args) of + {ok, Args} when is_list(Args) -> + Args; + _ -> + [] + end. + +session_lifetime() -> + case application:get_env(ssl, session_lifetime) of + {ok, Time} when is_integer(Time) -> + Time; + _ -> + ?'24H_in_sec' + end. + +max_session_cache_size() -> + case application:get_env(ssl, session_cache_server_max) of + {ok, Size} when is_integer(Size) -> + Size; + _ -> + ?DEFAULT_MAX_SESSION_CACHE + end. + diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 371599bbe8..4f9584bb9f 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -64,6 +64,7 @@ resumption = false :: boolean(), %% TLS 1.3 change_cipher_spec_sent = false :: boolean(), %% TLS 1.3 sni_guided_cert_selection = false :: boolean(), %% TLS 1.3 + early_data_accepted = false :: boolean(), %% TLS 1.3 allow_renegotiate = true ::boolean(), %% Ext handling hello, %%:: #client_hello{} | #server_hello{} diff --git a/lib/ssl/src/ssl_dist_connection_sup.erl b/lib/ssl/src/ssl_dist_connection_sup.erl index 28c8692ca5..441a7577be 100644 --- a/lib/ssl/src/ssl_dist_connection_sup.erl +++ b/lib/ssl/src/ssl_dist_connection_sup.erl @@ -42,48 +42,20 @@ start_link() -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= - init([]) -> - - TLSConnetionManager = tls_connection_manager_child_spec(), - %% Handles emulated options so that they inherited by the accept - %% socket, even when setopts is performed on the listen socket - ListenOptionsTracker = listen_options_tracker_child_spec(), - Pre_1_3SessionTracker = ssl_server_session_child_spec(), - - {ok, {{one_for_one, 10, 3600}, [TLSConnetionManager, - ListenOptionsTracker, - Pre_1_3SessionTracker - ]}}. + TLSSup = tls_sup_child_spec(), + {ok, {{one_for_one, 10, 3600}, [TLSSup]}}. %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -tls_connection_manager_child_spec() -> - Name = dist_tls_connection, - StartFunc = {tls_connection_sup, start_link_dist, []}, +tls_sup_child_spec() -> + Name = dist_tls_sup, + StartFunc = {tls_dist_sup, start_link, []}, Restart = permanent, Shutdown = 4000, - Modules = [tls_connection_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -listen_options_tracker_child_spec() -> - Name = dist_tls_socket, - StartFunc = {ssl_listen_tracker_sup, start_link_dist, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [tls_socket], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -ssl_server_session_child_spec() -> - Name = dist_ssl_server_session_cache_sup, - StartFunc = {ssl_server_session_cache_sup, start_link_dist, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [ssl_server_session_cache_sup], + Modules = [tls_dist_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/ssl_dist_sup.erl b/lib/ssl/src/ssl_dist_sup.erl index bea67935d8..ae0887c3d9 100644 --- a/lib/ssl/src/ssl_dist_sup.erl +++ b/lib/ssl/src/ssl_dist_sup.erl @@ -70,16 +70,16 @@ ssl_admin_child_spec() -> StartFunc = {ssl_dist_admin_sup, start_link , []}, Restart = permanent, Shutdown = 4000, - Modules = [ssl_admin_sup], + Modules = [ssl_dist_admin_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. ssl_connection_sup() -> - Name = ssl_dist_connection_sup, - StartFunc = {ssl_dist_connection_sup, start_link, []}, + Name = tls_dist_sup, + StartFunc = {tls_dist_sup, start_link, []}, Restart = permanent, Shutdown = 4000, - Modules = [ssl_connection_sup], + Modules = [tls_dist_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/ssl_gen_statem.erl b/lib/ssl/src/ssl_gen_statem.erl index aa2a5541a1..e6268b4876 100644 --- a/lib/ssl/src/ssl_gen_statem.erl +++ b/lib/ssl/src/ssl_gen_statem.erl @@ -438,50 +438,50 @@ initial_hello({call, From}, {start, Timeout}, #state{static_env = #static_env{role = client = Role, host = Host, port = Port, - protocol_cb = Connection, - transport_cb = Transport, - socket = Socket}, + protocol_cb = Connection}, handshake_env = #handshake_env{renegotiation = {Renegotiation, _}, - ocsp_stapling_state = OcspState0} = HsEnv, + ocsp_stapling_state = OcspState0}, connection_env = CEnv, - ssl_options = #{log_level := LogLevel, - %% Use highest version in initial ClientHello. + ssl_options = #{%% Use highest version in initial ClientHello. %% Versions is a descending list of supported versions. versions := [HelloVersion|_] = Versions, session_tickets := SessionTickets, ocsp_stapling := OcspStaplingOpt, - ocsp_nonce := OcspNonceOpt} = SslOpts, + ocsp_nonce := OcspNonceOpt, + early_data := EarlyData} = SslOpts, session = Session, connection_states = ConnectionStates0 } = State0) -> KeyShare = maybe_generate_client_shares(SslOpts), - %% Update UseTicket in case of automatic session resumption + %% Update UseTicket in case of automatic session resumption. The automatic ticket handling + %% also takes it into account if the ticket is suitable for sending early data not exceeding + %% the max_early_data_size or if it can only be used for session resumption. {UseTicket, State1} = tls_handshake_1_3:maybe_automatic_session_resumption(State0), TicketData = tls_handshake_1_3:get_ticket_data(self(), SessionTickets, UseTicket), OcspNonce = tls_handshake:ocsp_nonce(OcspNonceOpt, OcspStaplingOpt), - Hello = tls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, - Session#session.session_id, - Renegotiation, - Session#session.own_certificates, - KeyShare, - TicketData, - OcspNonce), - - Handshake0 = ssl_handshake:init_handshake_history(), + Hello0 = tls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, + Session#session.session_id, + Renegotiation, + Session#session.own_certificates, + KeyShare, + TicketData, + OcspNonce), + + %% Early Data Indication + Hello1 = tls_handshake_1_3:maybe_add_early_data_indication(Hello0, + EarlyData, + HelloVersion), %% Update pre_shared_key extension with binders (TLS 1.3) - Hello1 = tls_handshake_1_3:maybe_add_binders(Hello, TicketData, HelloVersion), + Hello2 = tls_handshake_1_3:maybe_add_binders(Hello1, TicketData, HelloVersion), MaxFragEnum = maps:get(max_frag_enum, Hello1#client_hello.extensions, undefined), ConnectionStates1 = ssl_record:set_max_fragment_length(MaxFragEnum, ConnectionStates0), + State2 = State1#state{connection_states = ConnectionStates1, + connection_env = CEnv#connection_env{negotiated_version = HelloVersion}}, - {BinMsg, ConnectionStates, Handshake} = - Connection:encode_handshake(Hello1, HelloVersion, ConnectionStates1, Handshake0), - - tls_socket:send(Transport, Socket, BinMsg), - ssl_logger:debug(LogLevel, outbound, 'handshake', Hello1), - ssl_logger:debug(LogLevel, outbound, 'record', BinMsg), + State3 = Connection:queue_handshake(Hello2, State2), %% RequestedVersion is used as the legacy record protocol version and shall be %% {3,3} in case of TLS 1.2 and higher. In all other cases it defaults to the @@ -490,18 +490,31 @@ initial_hello({call, From}, {start, Timeout}, %% negotiated_version is also used by the TLS 1.3 state machine and is set after %% ServerHello is processed. RequestedVersion = tls_record:hello_version(Versions), - State = State1#state{connection_states = ConnectionStates, - connection_env = CEnv#connection_env{ - negotiated_version = RequestedVersion}, - session = Session, - handshake_env = HsEnv#handshake_env{ - tls_handshake_history = Handshake, - ocsp_stapling_state = OcspState0#{ocsp_nonce => OcspNonce}}, - start_or_recv_from = From, - key_share = KeyShare}, - NextState = next_statem_state(Versions, Role), - Connection:next_event(NextState, no_record, State, - [{{timeout, handshake}, Timeout, close}]); + + {Ref,Maybe} = tls_handshake_1_3:maybe(), + try + %% Send Early Data + State4 = Maybe(tls_handshake_1_3:maybe_send_early_data(State3)), + + {#state{handshake_env = HsEnv1} = State5, _} = + Connection:send_handshake_flight(State4), + + State = State5#state{ + connection_env = CEnv#connection_env{ + negotiated_version = RequestedVersion}, + session = Session, + handshake_env = HsEnv1#handshake_env{ + ocsp_stapling_state = OcspState0#{ocsp_nonce => OcspNonce}}, + start_or_recv_from = From, + key_share = KeyShare}, + NextState = next_statem_state(Versions, Role), + Connection:next_event(NextState, no_record, State, + [{{timeout, handshake}, Timeout, close}]) + catch + {Ref, #alert{} = Alert} -> + handle_own_alert(Alert, RequestedVersion, init, + State0#state{start_or_recv_from = From}) + end; initial_hello({call, From}, {start, Timeout}, #state{static_env = #static_env{role = Role, protocol_cb = Connection}, ssl_options = #{versions := Versions}} = State0) -> @@ -1421,11 +1434,17 @@ read_application_data_deliver(State, Front, BufferSize, Rear, SocketOpts0, RecvF State#state{ user_data_buffer = {Front,BufferSize,Rear}, start_or_recv_from = undefined, - bytes_to_read = undefined, + bytes_to_read = undefined, socket_options = SocketOpts }}; true -> %% Try to deliver more data - read_application_data(State, Front, BufferSize, Rear, SocketOpts, undefined, undefined) + %% Process early data if it is accepted. + case (State#state.handshake_env)#handshake_env.early_data_accepted of + false -> + read_application_data(State, Front, BufferSize, Rear, SocketOpts, undefined, undefined); + true -> + read_application_data(State, Front, BufferSize, Rear, SocketOpts, RecvFrom, undefined) + end end. @@ -1782,20 +1801,37 @@ security_info(#state{connection_states = ConnectionStates, #security_parameters{client_random = ClientRand, server_random = ServerRand, master_secret = MasterSecret, - application_traffic_secret = AppTrafSecretRead}} = ReadState, + application_traffic_secret = AppTrafSecretRead, + client_early_data_secret = ServerEarlyData + }} = ReadState, BaseSecurityInfo = [{client_random, ClientRand}, {server_random, ServerRand}, {master_secret, MasterSecret}], if KeepSecrets =/= true -> BaseSecurityInfo; true -> #{security_parameters := - #security_parameters{application_traffic_secret = AppTrafSecretWrite}} = + #security_parameters{application_traffic_secret = AppTrafSecretWrite, + client_early_data_secret = ClientEarlyData + }} = ssl_record:current_connection_state(ConnectionStates, write), - BaseSecurityInfo ++ - if Role == server -> - [{server_traffic_secret_0, AppTrafSecretWrite}, {client_traffic_secret_0, AppTrafSecretRead}]; - true -> - [{client_traffic_secret_0, AppTrafSecretWrite}, {server_traffic_secret_0, AppTrafSecretRead}] - end ++ + if Role == server -> + if ServerEarlyData =/= undefined -> + [{server_traffic_secret_0, AppTrafSecretWrite}, + {client_traffic_secret_0, AppTrafSecretRead}, + {client_early_data_secret, ServerEarlyData}]; + true -> + [{server_traffic_secret_0, AppTrafSecretWrite}, + {client_traffic_secret_0, AppTrafSecretRead}] + end; + true -> + if ClientEarlyData =/= undefined -> + [{client_traffic_secret_0, AppTrafSecretWrite}, + {server_traffic_secret_0, AppTrafSecretRead}, + {client_early_data_secret, ClientEarlyData}]; + true -> + [{client_traffic_secret_0, AppTrafSecretWrite}, + {server_traffic_secret_0, AppTrafSecretRead}] + end + end ++ case ReadState of #{client_handshake_traffic_secret := ClientHSTrafficSecret, server_handshake_traffic_secret := ServerHSTrafficSecret} -> @@ -1803,7 +1839,7 @@ security_info(#state{connection_states = ConnectionStates, {server_handshake_traffic_secret, ServerHSTrafficSecret}]; _ -> [] - end + end ++ BaseSecurityInfo end. record_cb(tls) -> @@ -1981,10 +2017,18 @@ maybe_add_keylog({_, 'tlsv1.3'}, Info) -> ServerTrafficSecret0 = keylog_secret(ServerTrafficSecret0Bin, Prf), ClientHSecret = keylog_secret(ClientHSecretBin, Prf), ServerHSecret = keylog_secret(ServerHSecretBin, Prf), - Keylog = [io_lib:format("CLIENT_HANDSHAKE_TRAFFIC_SECRET ~64.16.0B ", [ClientRandom]) ++ ClientHSecret, - io_lib:format("SERVER_HANDSHAKE_TRAFFIC_SECRET ~64.16.0B ", [ClientRandom]) ++ ServerHSecret, - io_lib:format("CLIENT_TRAFFIC_SECRET_0 ~64.16.0B ", [ClientRandom]) ++ ClientTrafficSecret0, - io_lib:format("SERVER_TRAFFIC_SECRET_0 ~64.16.0B ", [ClientRandom]) ++ ServerTrafficSecret0], + Keylog0 = [io_lib:format("CLIENT_HANDSHAKE_TRAFFIC_SECRET ~64.16.0B ", [ClientRandom]) ++ ClientHSecret, + io_lib:format("SERVER_HANDSHAKE_TRAFFIC_SECRET ~64.16.0B ", [ClientRandom]) ++ ServerHSecret, + io_lib:format("CLIENT_TRAFFIC_SECRET_0 ~64.16.0B ", [ClientRandom]) ++ ClientTrafficSecret0, + io_lib:format("SERVER_TRAFFIC_SECRET_0 ~64.16.0B ", [ClientRandom]) ++ ServerTrafficSecret0], + Keylog = case lists:keyfind(client_early_data_secret, 1, Info) of + {client_early_data_secret, EarlySecret} -> + ClientEarlySecret = keylog_secret(EarlySecret, Prf), + [io_lib:format("CLIENT_EARLY_TRAFFIC_SECRET ~64.16.0B ", [ClientRandom]) ++ ClientEarlySecret + | Keylog0]; + _ -> + Keylog0 + end, Info ++ [{keylog,Keylog}] catch _Cxx:_Exx -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 7d6c21438e..783f729386 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -43,7 +43,7 @@ -type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | #client_key_exchange{} | #finished{} | #certificate_verify{} | - #hello_request{} | #next_protocol{}. + #hello_request{} | #next_protocol{} | #end_of_early_data{}. %% Create handshake messages -export([hello_request/0, server_hello/4, server_hello_done/0, @@ -768,7 +768,13 @@ encode_extensions([#cookie{cookie = Cookie} | Rest], Acc) -> CookieLen = byte_size(Cookie), Len = CookieLen + 2, encode_extensions(Rest, <<?UINT16(?COOKIE_EXT), ?UINT16(Len), ?UINT16(CookieLen), - Cookie/binary, Acc/binary>>). + Cookie/binary, Acc/binary>>); +encode_extensions([#early_data_indication{} | Rest], Acc) -> + encode_extensions(Rest, <<?UINT16(?EARLY_DATA_EXT), + ?UINT16(0), Acc/binary>>); +encode_extensions([#early_data_indication_nst{indication = MaxSize} | Rest], Acc) -> + encode_extensions(Rest, <<?UINT16(?EARLY_DATA_EXT), + ?UINT16(4), ?UINT32(MaxSize), Acc/binary>>). encode_cert_status_req( StatusType, @@ -1309,7 +1315,9 @@ get_identities_binders(TicketData) -> %% get_identities_binders([], {Identities, Binders}, _) -> {lists:reverse(Identities), lists:reverse(Binders)}; -get_identities_binders([{Key, _, Identity, _, _, HKDF}|T], {I0, B0}, N) -> +get_identities_binders([#ticket_data{key = Key, + identity = Identity, + cipher_suite = {_, HKDF}}|T], {I0, B0}, N) -> %% Use dummy binder for proper calculation of packet size when creating %% the real binder value. Binder = dummy_binder(HKDF), @@ -2813,6 +2821,7 @@ decode_extensions(<<?UINT16(?COOKIE_EXT), ?UINT16(Len), ?UINT16(CookieLen), when Len == CookieLen + 2 -> decode_extensions(Rest, Version, MessageType, Acc#{cookie => #cookie{cookie = Cookie}}); + %% RFC6066, if a server returns a "CertificateStatus" message, then %% the server MUST have included an extension of type "status_request" %% with empty "extension_data" in the extended server hello. @@ -2838,6 +2847,18 @@ decode_extensions(<<?UINT16(?STATUS_REQUEST), ?UINT16(Len), decode_extensions(Rest, Version, MessageType, Acc) end; +decode_extensions(<<?UINT16(?EARLY_DATA_EXT), ?UINT16(0), Rest/binary>>, + Version, MessageType, Acc) -> + decode_extensions(Rest, Version, MessageType, + Acc#{early_data => #early_data_indication{}}); + +decode_extensions(<<?UINT16(?EARLY_DATA_EXT), ?UINT16(4), ?UINT32(MaxSize), + Rest/binary>>, + Version, MessageType, Acc) -> + decode_extensions(Rest, Version, MessageType, + Acc#{early_data => + #early_data_indication_nst{indication = MaxSize}}); + %% Ignore data following the ClientHello (i.e., %% extensions) if not understood. decode_extensions(<<?UINT16(_), ?UINT16(Len), _Unknown:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 6477f5ab57..b080016458 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -115,6 +115,8 @@ %% 2^24.5 * 2^14 = 2^38.5 -define(KEY_USAGE_LIMIT_AES_GCM, 388736063997). +-define(DEFAULT_MAX_EARLY_DATA_SIZE, 16384). + %% This map stores all supported options with default values and %% list of dependencies: %% #{<option> => {<default_value>, [<option>]}, @@ -140,6 +142,9 @@ depth => {10, [versions]}, dh => {undefined, [versions]}, dhfile => {undefined, [versions]}, + early_data => {undefined, [versions, + session_tickets, + use_ticket]}, eccs => {undefined, [versions]}, erl_dist => {false, [versions]}, fail_if_no_peer_cert => {false, [versions]}, @@ -236,6 +241,17 @@ {stop, any(), any()}. -type ssl_options() :: map(). +%% Internal ticket data record holding pre-processed ticket data. +-record(ticket_data, + {key, %% key in client ticket store + pos, %% ticket position in binders list + identity, %% opaque ticket binary + psk, %% pre-shared key + nonce, %% ticket nonce + cipher_suite, %% cipher suite - hash, bulk cipher algorithm + max_size %% max early data size allowed by this ticket + }). + -endif. % -ifdef(ssl_internal). diff --git a/lib/ssl/src/ssl_listen_tracker_sup.erl b/lib/ssl/src/ssl_listen_tracker_sup.erl index f7e97bcb76..6afd1c0009 100644 --- a/lib/ssl/src/ssl_listen_tracker_sup.erl +++ b/lib/ssl/src/ssl_listen_tracker_sup.erl @@ -69,4 +69,4 @@ init(_O) -> tracker_name(normal) -> ?MODULE; tracker_name(dist) -> - list_to_atom(atom_to_list(?MODULE) ++ "dist"). + list_to_atom(atom_to_list(?MODULE) ++ "_dist"). diff --git a/lib/ssl/src/ssl_logger.erl b/lib/ssl/src/ssl_logger.erl index d8bf5b3a2b..027dbe8732 100644 --- a/lib/ssl/src/ssl_logger.erl +++ b/lib/ssl/src/ssl_logger.erl @@ -254,9 +254,13 @@ parse_handshake(Direction, #key_update{} = KeyUpdate) -> Header = io_lib:format("~s Post-Handshake, KeyUpdate", [header_prefix(Direction)]), Message = io_lib:format("~p", [?rec_info(key_update, KeyUpdate)]), + {Header, Message}; +parse_handshake(Direction, #end_of_early_data{} = EndOfEarlyData) -> + Header = io_lib:format("~s Handshake, EndOfEarlyData", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(end_of_early_data, EndOfEarlyData)]), {Header, Message}. - parse_cipher_suites([_|_] = Ciphers) -> [format_cipher(C) || C <- Ciphers]. diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 47a9f11829..040c4f1ebc 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -41,8 +41,12 @@ set_client_verify_data/3, set_server_verify_data/3, set_max_fragment_length/2, - empty_connection_state/2, initial_connection_state/2, record_protocol_role/1, - step_encryption_state/1]). + empty_connection_state/2, + empty_connection_state/3, + record_protocol_role/1, + step_encryption_state/1, + step_encryption_state_read/1, + step_encryption_state_write/1]). %% Compression -export([compress/3, uncompress/3, compressions/0]). @@ -138,6 +142,17 @@ step_encryption_state(#state{connection_states = ConnStates#{current_read => NewRead, current_write => NewWrite}}. +step_encryption_state_read(#state{connection_states = + #{pending_read := PendingRead} = ConnStates} = State) -> + NewRead = PendingRead#{sequence_number => 0}, + State#state{connection_states = + ConnStates#{current_read => NewRead}}. + +step_encryption_state_write(#state{connection_states = + #{pending_write := PendingWrite} = ConnStates} = State) -> + NewWrite = PendingWrite#{sequence_number => 0}, + State#state{connection_states = + ConnStates#{current_write => NewWrite}}. %%-------------------------------------------------------------------- -spec set_security_params(#security_parameters{}, #security_parameters{}, @@ -445,6 +460,10 @@ nonce_seed(_,_, CipherState) -> %%-------------------------------------------------------------------- empty_connection_state(ConnectionEnd, BeastMitigation) -> + MaxEarlyDataSize = ssl_config:get_max_early_data_size(), + empty_connection_state(ConnectionEnd, BeastMitigation, MaxEarlyDataSize). +%% +empty_connection_state(ConnectionEnd, BeastMitigation, MaxEarlyDataSize) -> SecParams = empty_security_params(ConnectionEnd), #{security_parameters => SecParams, beast_mitigation => BeastMitigation, @@ -454,7 +473,10 @@ empty_connection_state(ConnectionEnd, BeastMitigation) -> secure_renegotiation => undefined, client_verify_data => undefined, server_verify_data => undefined, - max_fragment_length => undefined + max_early_data_size => MaxEarlyDataSize, + max_fragment_length => undefined, + trial_decryption => false, + early_data_limit => false }. empty_security_params(ConnectionEnd = ?CLIENT) -> @@ -481,20 +503,6 @@ record_protocol_role(client) -> record_protocol_role(server) -> ?SERVER. -initial_connection_state(ConnectionEnd, BeastMitigation) -> - #{security_parameters => - initial_security_params(ConnectionEnd), - sequence_number => 0, - beast_mitigation => BeastMitigation, - compression_state => undefined, - cipher_state => undefined, - mac_secret => undefined, - secure_renegotiation => undefined, - client_verify_data => undefined, - server_verify_data => undefined, - max_fragment_length => undefined - }. - initial_security_params(ConnectionEnd) -> SecParams = #security_parameters{connection_end = ConnectionEnd, compression_algorithm = ?NULL}, diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index 9aa598daed..d142ecf6da 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -68,6 +68,7 @@ master_secret, % opaque 48 resumption_master_secret, application_traffic_secret, + client_early_data_secret, client_random, % opaque 32 server_random, % opaque 32 exportable % boolean diff --git a/lib/ssl/src/ssl_server_session_cache.erl b/lib/ssl/src/ssl_server_session_cache.erl index 022255258e..44862e5cad 100644 --- a/lib/ssl/src/ssl_server_session_cache.erl +++ b/lib/ssl/src/ssl_server_session_cache.erl @@ -66,6 +66,8 @@ {error, Error :: {already_started, pid()}} | {error, Error :: term()} | ignore. +start_link(ssl_unknown_listener = Listner, Map) -> + gen_server:start_link({local, Listner}, ?MODULE, [Listner, Map], []); start_link(Listner, Map) -> gen_server:start_link(?MODULE, [Listner, Map], []). @@ -113,7 +115,7 @@ init([Listner, #{lifetime := Lifetime, max := Max }]) -> process_flag(trap_exit, true), - erlang:monitor(process, Listner), + Monitor = monitor_listener(Listner), DbRef = init(Cb, [{role, server} | InitArgs]), State = #state{store_cb = Cb, lifetime = Lifetime, @@ -121,7 +123,7 @@ init([Listner, #{lifetime := Lifetime, max = Max, session_index = #{}, id_generator = crypto:strong_rand_bytes(16), - listner = Listner + listner = Monitor }, {ok, State}. @@ -196,7 +198,7 @@ handle_cast({register_session, #session{session_id = SessionId} = Session}, -spec handle_info(Info :: timeout() | term(), State :: term()) -> {noreply, NewState :: term()}. -handle_info({'DOWN', _, process, Listner, _}, #state{listner = Listner} = State) -> +handle_info({'DOWN', Monitor, _, _, _}, #state{listner = Monitor} = State) -> {stop, normal, State}; handle_info(_, State) -> {noreply, State}. @@ -270,3 +272,10 @@ size(Cb,Cache) -> error:undef -> Cb:foldl(fun(_, Acc) -> Acc + 1 end, 0, Cache) end. + +monitor_listener(ssl_unknown_listener) -> + %% Backwards compatible Erlang node + %% global process. + undefined; +monitor_listener(Listen) when is_port(Listen) -> + erlang:monitor(port, Listen). diff --git a/lib/ssl/src/ssl_server_session_cache_sup.erl b/lib/ssl/src/ssl_server_session_cache_sup.erl index ef8a24f19b..2f0c3dc823 100644 --- a/lib/ssl/src/ssl_server_session_cache_sup.erl +++ b/lib/ssl/src/ssl_server_session_cache_sup.erl @@ -29,30 +29,21 @@ -include("ssl_internal.hrl"). %% API --export([start_link/0, - start_link_dist/0]). --export([start_child/1, - start_child_dist/1, - session_opts/0]). +-export([start_link/0]). +-export([start_child/1]). %% Supervisor callback -export([init/1]). --define(DEFAULT_MAX_SESSION_CACHE, 1000). %%%========================================================================= %%% API %%%========================================================================= start_link() -> - supervisor:start_link({local, tracker_name(normal)}, ?MODULE, []). + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_link_dist() -> - supervisor:start_link({local, tracker_name(dist)}, ?MODULE, []). +start_child(Listner) -> + supervisor:start_child(?MODULE, [Listner | ssl_config:pre_1_3_session_opts()]). -start_child(Args) -> - supervisor:start_child(tracker_name(normal), [self() | Args]). - -start_child_dist(Args) -> - supervisor:start_child(tracker_name(dist), [self() | Args]). %%%========================================================================= %%% Supervisor callback @@ -72,45 +63,3 @@ init(_O) -> ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. -tracker_name(normal) -> - ?MODULE; -tracker_name(dist) -> - list_to_atom(atom_to_list(?MODULE) ++ "dist"). - -session_opts() -> - CbOpts = case application:get_env(ssl, session_cb) of - {ok, Cb} when is_atom(Cb) -> - InitArgs = session_cb_init_args(), - #{session_cb => Cb, - session_cb_init_args => InitArgs}; - _ -> - #{session_cb => ssl_server_session_cache_db, - session_cb_init_args => []} - end, - LifeTime = session_lifetime(), - Max = max_session_cache_size(), - [CbOpts#{lifetime => LifeTime, max => Max}]. - -session_cb_init_args() -> - case application:get_env(ssl, session_cb_init_args) of - {ok, Args} when is_list(Args) -> - Args; - _ -> - [] - end. - -session_lifetime() -> - case application:get_env(ssl, session_lifetime) of - {ok, Time} when is_integer(Time) -> - Time; - _ -> - ?'24H_in_sec' - end. - -max_session_cache_size() -> - case application:get_env(ssl, session_cache_server_max) of - {ok, Size} when is_integer(Size) -> - Size; - _ -> - ?DEFAULT_MAX_SESSION_CACHE - end. diff --git a/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl b/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl new file mode 100644 index 0000000000..936ffcc0ac --- /dev/null +++ b/lib/ssl/src/ssl_upgrade_server_session_cache_sup.erl @@ -0,0 +1,90 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2020-2020. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- +%% Purpose: Supervisor for a listen options tracker +%%---------------------------------------------------------------------- +-module(ssl_upgrade_server_session_cache_sup). + +-behaviour(supervisor). + +-include("ssl_internal.hrl"). + +%% API +-export([start_link/0, + start_link_dist/0]). +-export([start_child/1]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= +start_link() -> + supervisor:start_link({local, sup_name(normal)}, ?MODULE, []). + +start_link_dist() -> + supervisor:start_link({local, sup_name(dist)}, ?MODULE, []). + +start_child(Type) -> + SupName = sup_name(Type), + Children = supervisor:count_children(SupName), + Workers = proplists:get_value(workers, Children), + case Workers of + 0 -> + %% In case two upgrade servers are started very close to each other + %% only one will be able to grab the local name and we will use + %% that process for handling pre TLS-1.3 sessions for + %% servers with to us unknown listeners. + case supervisor:start_child(SupName, [ssl_unknown_listener | ssl_config:pre_1_3_session_opts()]) of + {error, {already_started, Child}} -> + {ok, Child}; + {ok, _} = Return -> + Return + end; + 1 -> + [{_,Child,_, _}] = supervisor:which_children(SupName), + {ok, Child} + end. + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= +init(_O) -> + RestartStrategy = simple_one_for_one, + MaxR = 3, + MaxT = 3600, + + Name = undefined, % As simple_one_for_one is used. + StartFunc = {ssl_server_session_cache, start_link, []}, + Restart = transient, % Should be restarted only on abnormal termination + Shutdown = 4000, + Modules = [ssl_server_session_cache], + Type = worker, + + ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, + {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. + +sup_name(normal) -> + ?MODULE; +sup_name(dist) -> + list_to_atom(atom_to_list(?MODULE) ++ "_dist"). diff --git a/lib/ssl/src/tls_client_ticket_store.erl b/lib/ssl/src/tls_client_ticket_store.erl index 179c86ba8d..eb10adc9f1 100644 --- a/lib/ssl/src/tls_client_ticket_store.erl +++ b/lib/ssl/src/tls_client_ticket_store.erl @@ -25,10 +25,11 @@ -module(tls_client_ticket_store). -behaviour(gen_server). +-include("ssl_internal.hrl"). -include("tls_handshake_1_3.hrl"). %% API --export([find_ticket/3, +-export([find_ticket/5, get_tickets/2, lock_tickets/2, remove_tickets/1, @@ -51,7 +52,7 @@ -record(data, { pos = undefined, - hkdf, + cipher_suite, sni, psk, timestamp, @@ -69,8 +70,8 @@ start_link(Max, Lifetime) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Max, Lifetime], []). -find_ticket(Pid, HashAlgos, SNI) -> - gen_server:call(?MODULE, {find_ticket, Pid, HashAlgos, SNI}, infinity). +find_ticket(Pid, Ciphers, HashAlgos, SNI, EarlyDataSize) -> + gen_server:call(?MODULE, {find_ticket, Pid, Ciphers, HashAlgos, SNI, EarlyDataSize}, infinity). get_tickets(Pid, Keys) -> gen_server:call(?MODULE, {get_tickets, Pid, Keys}, infinity). @@ -85,8 +86,8 @@ remove_tickets([]) -> remove_tickets(Keys) -> gen_server:cast(?MODULE, {remove_tickets, Keys}). -store_ticket(Ticket, HKDF, SNI, PSK) -> - gen_server:call(?MODULE, {store_ticket, Ticket, HKDF, SNI, PSK}, infinity). +store_ticket(Ticket, CipherSuite, SNI, PSK) -> + gen_server:call(?MODULE, {store_ticket, Ticket, CipherSuite, SNI, PSK}, infinity). unlock_tickets(Pid, Keys) -> gen_server:call(?MODULE, {unlock, Pid, Keys}, infinity). @@ -107,8 +108,8 @@ init(Args) -> -spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> {reply, Reply :: term(), NewState :: term()} . -handle_call({find_ticket, Pid, HashAlgos, SNI}, _From, State) -> - Key = do_find_ticket(State, Pid, HashAlgos, SNI), +handle_call({find_ticket, Pid, Ciphers, HashAlgos, SNI, EarlyDataSize}, _From, State) -> + Key = do_find_ticket(State, Pid, Ciphers, HashAlgos, SNI, EarlyDataSize), {reply, Key, State}; handle_call({get_tickets, Pid, Keys}, _From, State) -> Data = get_tickets(State, Pid, Keys), @@ -116,8 +117,8 @@ handle_call({get_tickets, Pid, Keys}, _From, State) -> handle_call({lock, Pid, Keys}, _From, State0) -> State = lock_tickets(State0, Pid, Keys), {reply, ok, State}; -handle_call({store_ticket, Ticket, HKDF, SNI, PSK}, _From, State0) -> - State = store_ticket(State0, Ticket, HKDF, SNI, PSK), +handle_call({store_ticket, Ticket, CipherSuite, SNI, PSK}, _From, State0) -> + State = store_ticket(State0, Ticket, CipherSuite, SNI, PSK), {reply, ok, State}; handle_call({unlock, Pid, Keys}, _From, State0) -> State = unlock_tickets(State0, Pid, Keys), @@ -170,42 +171,78 @@ inital_state([Max, Lifetime]) -> max = Max }. - -do_find_ticket(_, _, [], _) -> - undefined; +do_find_ticket(Iter, Pid, Ciphers, HashAlgos, SNI, EarlyDataSize) -> + do_find_ticket(Iter, Pid, Ciphers, HashAlgos, SNI, EarlyDataSize, []). +%% +do_find_ticket(_, _, _, [], _, _, []) -> + {undefined, undefined}; +do_find_ticket(_, _, _, [], _, _, Acc) -> + {undefined, last_elem(Acc)}; do_find_ticket(#state{db = Db, - lifetime = Lifetime} = State, Pid, [Hash|T], SNI) -> - case iterate_tickets(gb_trees:iterator(Db), Pid, Hash, SNI, Lifetime) of - none -> - do_find_ticket(State, Pid, T, SNI); + lifetime = Lifetime} = State, Pid, Ciphers, [Hash|T], SNI, EarlyDataSize, Acc) -> + case iterate_tickets(gb_trees:iterator(Db), Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize) of + {undefined, undefined} -> + do_find_ticket(State, Pid, Ciphers, T, SNI, EarlyDataSize, Acc); + {undefined, Key} -> + do_find_ticket(State, Pid, Ciphers, T, SNI, EarlyDataSize, [Key|Acc]); Key -> Key end. -iterate_tickets(Iter0, Pid, Hash, SNI, Lifetime) -> +iterate_tickets(Iter0, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize) -> + iterate_tickets(Iter0, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, []). +%% +iterate_tickets(Iter0, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, Acc) -> case gb_trees:next(Iter0) of - {Key, #data{hkdf = Hash, + {Key, #data{cipher_suite = {Cipher, Hash}, sni = TicketSNI, + ticket = #new_session_ticket{ + extensions = Extensions}, timestamp = Timestamp, lock = Lock}, Iter} when Lock =:= undefined orelse Lock =:= Pid -> + MaxEarlyData = tls_handshake_1_3:get_max_early_data(Extensions), Age = erlang:system_time(seconds) - Timestamp, if Age < Lifetime -> case verify_ticket_sni(SNI, TicketSNI) of match -> - Key; + case lists:member(Cipher, Ciphers) of + true -> + Front = last_elem(Acc), + %% 'Key' can be used with early_data as both + %% block cipher and hash algorithm matches. + %% 'Front' can only be used for session + %% resumption. + case EarlyDataSize =:= undefined orelse + EarlyDataSize =< MaxEarlyData of + true -> + {Key, Front}; + false -> + %% 'Key' cannot be used for early_data as the data + %% to be sent exceeds the max limit for this ticket. + iterate_tickets(Iter, Pid, Ciphers, Hash, SNI, + Lifetime, EarlyDataSize,[Key|Acc]) + end; + false -> + iterate_tickets(Iter, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, [Key|Acc]) + end; nomatch -> - iterate_tickets(Iter, Pid, Hash, SNI, Lifetime) + iterate_tickets(Iter, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, Acc) end; true -> - iterate_tickets(Iter, Pid, Hash, SNI, Lifetime) + iterate_tickets(Iter, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, Acc) end; {_, _, Iter} -> - iterate_tickets(Iter, Pid, Hash, SNI, Lifetime); + iterate_tickets(Iter, Pid, Ciphers, Hash, SNI, Lifetime, EarlyDataSize, Acc); none -> - none + {undefined, last_elem(Acc)} end. +last_elem([_|_] = L) -> + lists:last(L); +last_elem([]) -> + undefined. + verify_ticket_sni(undefined, _) -> match; verify_ticket_sni(SNI, SNI) -> @@ -224,7 +261,7 @@ get_tickets(_, _, [], Acc) -> get_tickets(#state{db = Db} = State, Pid, [Key|T], Acc) -> try gb_trees:get(Key, Db) of #data{pos = Pos, - hkdf = HKDF, + cipher_suite = CipherSuite, psk = PSK, timestamp = Timestamp, ticket = NewSessionTicket, @@ -235,14 +272,23 @@ get_tickets(#state{db = Db} = State, Pid, [Key|T], Acc) -> ticket_age_add = AgeAdd, ticket_nonce = Nonce, ticket = Ticket, - extensions = _Extensions + extensions = Extensions } = NewSessionTicket, TicketAge = erlang:system_time(seconds) - Timestamp, ObfuscatedTicketAge = obfuscate_ticket_age(TicketAge, AgeAdd), Identity = #psk_identity{ identity = Ticket, obfuscated_ticket_age = ObfuscatedTicketAge}, - get_tickets(State, Pid, T, [{Key, Pos, Identity, PSK, Nonce, HKDF}|Acc]) + MaxEarlyData = tls_handshake_1_3:get_max_early_data(Extensions), + TicketData = #ticket_data{ + key = Key, + pos = Pos, + identity = Identity, + psk = PSK, + nonce = Nonce, + cipher_suite = CipherSuite, + max_size = MaxEarlyData}, + get_tickets(State, Pid, T, [TicketData|Acc]) catch _:_ -> get_tickets(State, Pid, T, Acc) @@ -296,7 +342,7 @@ collect_invalid_tickets(Iter0, Lifetime, Acc) -> end. -store_ticket(#state{db = Db0, max = Max} = State, Ticket, HKDF, SNI, PSK) -> +store_ticket(#state{db = Db0, max = Max} = State, Ticket, CipherSuite, SNI, PSK) -> Timestamp = erlang:system_time(seconds), Size = gb_trees:size(Db0), Db1 = if Size =:= Max -> @@ -306,7 +352,7 @@ store_ticket(#state{db = Db0, max = Max} = State, Ticket, HKDF, SNI, PSK) -> end, Key = {erlang:monotonic_time(), erlang:unique_integer([monotonic])}, Db = gb_trees:insert(Key, - #data{hkdf = HKDF, + #data{cipher_suite = CipherSuite, sni = SNI, psk = PSK, timestamp = Timestamp, diff --git a/lib/ssl/src/tls_connection_1_3.erl b/lib/ssl/src/tls_connection_1_3.erl index fc4b4f673f..6c81933c23 100644 --- a/lib/ssl/src/tls_connection_1_3.erl +++ b/lib/ssl/src/tls_connection_1_3.erl @@ -132,6 +132,7 @@ wait_sh/3, wait_ee/3, wait_cert_cr/3, + wait_eoed/3, connection/3, downgrade/3 ]). @@ -168,8 +169,8 @@ update_cipher_key(ConnStateName, CS0) -> ApplicationTrafficSecret = tls_v1:update_traffic_secret(HKDF, ApplicationTrafficSecret0), %% Calculate traffic keys - #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), - {Key, IV} = tls_v1:calculate_traffic_keys(HKDF, Cipher, ApplicationTrafficSecret), + KeyLength = tls_v1:key_length(CipherSuite), + {Key, IV} = tls_v1:calculate_traffic_keys(HKDF, KeyLength, ApplicationTrafficSecret), SecParams = SecParams0#security_parameters{application_traffic_secret = ApplicationTrafficSecret}, CipherState = CipherState0#cipher_state{key = Key, iv = IV}, @@ -307,8 +308,8 @@ negotiated(internal, Message, State0) -> negotiated(info, Msg, State) -> tls_gen_connection:handle_info(Msg, ?FUNCTION_NAME, State). -wait_cert(internal, #change_cipher_spec{}, State) -> - tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State); +wait_cert(internal, #change_cipher_spec{}, State0) -> + tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State0); wait_cert(internal, #certificate_1_3{} = Certificate, State0) -> case tls_handshake_1_3:do_wait_cert(Certificate, State0) of @@ -337,9 +338,8 @@ wait_cv(info, Msg, State) -> wait_cv(Type, Msg, State) -> ssl_gen_statem:handle_common_event(Type, Msg, ?FUNCTION_NAME, State). - -wait_finished(internal, #change_cipher_spec{}, State) -> - tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State); +wait_finished(internal, #change_cipher_spec{}, State0) -> + tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State0); wait_finished(internal, #finished{} = Finished, State0) -> case tls_handshake_1_3:do_wait_finished(Finished, State0) of @@ -417,6 +417,20 @@ wait_cert_cr(info, Msg, State) -> wait_cert_cr(Type, Msg, State) -> ssl_gen_statem:handle_common_event(Type, Msg, ?FUNCTION_NAME, State). +wait_eoed(internal, #change_cipher_spec{}, State) -> + tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State); +wait_eoed(internal, #end_of_early_data{} = EOED, State0) -> + case tls_handshake_1_3:do_wait_eoed(EOED, State0) of + {#alert{} = Alert, State} -> + ssl_gen_statem:handle_own_alert(Alert, {3,4}, wait_eoed, State); + {State1, NextState} -> + tls_gen_connection:next_event(NextState, no_record, State1) + end; +wait_eoed(info, Msg, State) -> + tls_gen_connection:handle_info(Msg, ?FUNCTION_NAME, State); +wait_eoed(Type, Msg, State) -> + ssl_gen_statem:handle_common_event(Type, Msg, ?FUNCTION_NAME, State). + connection(internal, #new_session_ticket{} = NewSessionTicket, State) -> handle_new_session_ticket(NewSessionTicket, State), tls_gen_connection:next_event(?FUNCTION_NAME, no_record, State); @@ -450,7 +464,8 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #{erl_dist := IsErlDist, client_renegotiation := ClientRenegotiation} = SSLOptions, - ConnectionStates = tls_record:init_connection_states(Role, disabled), + MaxEarlyDataSize = init_max_early_data_size(Role), + ConnectionStates = tls_record:init_connection_states(Role, disabled, MaxEarlyDataSize), InternalActiveN = case application:get_env(ssl, internal_active_n) of {ok, N} when is_integer(N) andalso (not IsErlDist) -> N; @@ -504,10 +519,12 @@ handle_new_session_ticket(#new_session_ticket{ticket_nonce = Nonce} = NewSession when SessionTickets =:= manual -> #{security_parameters := SecParams} = ssl_record:current_connection_state(ConnectionStates, read), + CipherSuite = SecParams#security_parameters.cipher_suite, + #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), HKDF = SecParams#security_parameters.prf_algorithm, RMS = SecParams#security_parameters.resumption_master_secret, PSK = tls_v1:pre_shared_key(RMS, Nonce, HKDF), - send_ticket_data(User, NewSessionTicket, HKDF, SNI, PSK); + send_ticket_data(User, NewSessionTicket, {Cipher, HKDF}, SNI, PSK); handle_new_session_ticket(#new_session_ticket{ticket_nonce = Nonce} = NewSessionTicket, #state{connection_states = ConnectionStates, ssl_options = #{session_tickets := SessionTickets, @@ -515,21 +532,21 @@ handle_new_session_ticket(#new_session_ticket{ticket_nonce = Nonce} = NewSession when SessionTickets =:= auto -> #{security_parameters := SecParams} = ssl_record:current_connection_state(ConnectionStates, read), + CipherSuite = SecParams#security_parameters.cipher_suite, + #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), HKDF = SecParams#security_parameters.prf_algorithm, RMS = SecParams#security_parameters.resumption_master_secret, PSK = tls_v1:pre_shared_key(RMS, Nonce, HKDF), - tls_client_ticket_store:store_ticket(NewSessionTicket, HKDF, SNI, PSK). - + tls_client_ticket_store:store_ticket(NewSessionTicket, {Cipher, HKDF}, SNI, PSK). -%% Send ticket data to user as opaque binary -send_ticket_data(User, NewSessionTicket, HKDF, SNI, PSK) -> +send_ticket_data(User, NewSessionTicket, CipherSuite, SNI, PSK) -> Timestamp = erlang:system_time(seconds), - TicketData = #{hkdf => HKDF, + TicketData = #{cipher_suite => CipherSuite, sni => SNI, psk => PSK, timestamp => Timestamp, ticket => NewSessionTicket}, - User ! {ssl, session_ticket, {SNI, erlang:term_to_binary(TicketData)}}. + User ! {ssl, session_ticket, TicketData}. handle_key_update(#key_update{request_update = update_not_requested}, State0) -> %% Update read key in connection @@ -545,3 +562,12 @@ handle_key_update(#key_update{request_update = update_requested}, {error, Reason} -> {error, State1, ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, Reason)} end. + +init_max_early_data_size(client) -> + %% Disable trial decryption on the client side + %% Servers do trial decryption of max_early_data bytes of plain text. + %% Setting it to 0 means that a decryption error will result in an Alert. + 0; +init_max_early_data_size(server) -> + ssl_config:get_max_early_data_size(). + diff --git a/lib/ssl/src/tls_connection_sup.erl b/lib/ssl/src/tls_connection_sup.erl index 9813250a46..b7f80ad524 100644 --- a/lib/ssl/src/tls_connection_sup.erl +++ b/lib/ssl/src/tls_connection_sup.erl @@ -40,13 +40,13 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). start_link_dist() -> - supervisor:start_link({local, ssl_connection_sup_dist}, ?MODULE, []). + supervisor:start_link({local, tls_dist_connection_sup}, ?MODULE, []). start_child(Args) -> supervisor:start_child(?MODULE, Args). start_child_dist(Args) -> - supervisor:start_child(ssl_connection_sup_dist, Args). + supervisor:start_child(tls_dist_connection_sup, Args). %%%========================================================================= %%% Supervisor callback diff --git a/lib/ssl/src/tls_dist_server_sup.erl b/lib/ssl/src/tls_dist_server_sup.erl new file mode 100644 index 0000000000..96603a7495 --- /dev/null +++ b/lib/ssl/src/tls_dist_server_sup.erl @@ -0,0 +1,89 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(tls_dist_server_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + ListenTracker = listen_options_tracker_child_spec(), + SessionTracker = tls_server_session_child_spec(), + Pre_1_3SessionTracker = ssl_server_session_child_spec(), + + {ok, {{one_for_all, 10, 3600}, [ListenTracker, + SessionTracker, + Pre_1_3SessionTracker + ]}}. + + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +%% Handles emulated options so that they inherited by the accept +%% socket, even when setopts is performed on the listen socket +listen_options_tracker_child_spec() -> + Name = dist_tls_socket, + StartFunc = {ssl_listen_tracker_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_listen_tracker_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +tls_server_session_child_spec() -> + Name = dist_tls_server_session_ticket, + StartFunc = {tls_server_session_ticket_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_server_session_ticket_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +ssl_server_session_child_spec() -> + Name = dist_ssl_server_session_cache_sup, + StartFunc = {ssl_upgrade_server_session_cache_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_server_session_cache_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + diff --git a/lib/ssl/src/tls_dist_sup.erl b/lib/ssl/src/tls_dist_sup.erl new file mode 100644 index 0000000000..54e0a6a514 --- /dev/null +++ b/lib/ssl/src/tls_dist_sup.erl @@ -0,0 +1,75 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(tls_dist_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + + TLSConnetionSup = tls_connection_child_spec(), + ServerInstanceSup = server_instance_child_spec(), + + {ok, {{one_for_one, 10, 3600}, [TLSConnetionSup, + ServerInstanceSup + ]}}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +tls_connection_child_spec() -> + Name = dist_tls_connection, + StartFunc = {tls_connection_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_connection_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +server_instance_child_spec() -> + Name = dist_tls_server_sup, + StartFunc = {tls_dist_server_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_dist_server_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/tls_gen_connection.erl b/lib/ssl/src/tls_gen_connection.erl index 7c16bfa3de..5da87e79d6 100644 --- a/lib/ssl/src/tls_gen_connection.erl +++ b/lib/ssl/src/tls_gen_connection.erl @@ -659,6 +659,17 @@ next_record(#state{connection_env = #connection_env{negotiated_version = {3,4} = [_|_] -> next_record(State, CipherTexts, ConnectionStates, Check, [Fragment|Acc]) end; + {trial_decryption_failed, ConnectionStates} -> + case CipherTexts of + [] -> + %% End of cipher texts - build and deliver an ?APPLICATION_DATA record + %% from the accumulated fragments + next_record_done(State, [], ConnectionStates, + #ssl_tls{type = ?APPLICATION_DATA, + fragment = iolist_to_binary(lists:reverse(Acc))}); + [_|_] -> + next_record(State, CipherTexts, ConnectionStates, Check, Acc) + end; {Record, ConnectionStates} when Acc =:= [] -> %% Singelton non-?APPLICATION_DATA record - deliver next_record_done(State, CipherTexts, ConnectionStates, Record); diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index dbde7ad476..640b999884 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -51,12 +51,19 @@ do_wait_sh/2, do_wait_ee/2, do_wait_cert_cr/2, + do_wait_eoed/2, + early_data_size/1, get_ticket_data/3, maybe_add_binders/3, maybe_add_binders/4, - maybe_automatic_session_resumption/1]). + maybe_add_early_data_indication/3, + maybe_automatic_session_resumption/1, + maybe_send_early_data/1, + update_current_read/3]). --export([is_valid_binder/4]). +-export([get_max_early_data/1, + is_valid_binder/4, + maybe/0]). %% crypto:hash(sha256, "HelloRetryRequest"). -define(HELLO_RETRY_REQUEST_RANDOM, <<207,33,173,116,229,154,97,17, @@ -178,12 +185,18 @@ encrypted_extensions(#state{handshake_env = HandshakeEnv}) -> MaxFragEnum -> E1#{max_frag_enum => MaxFragEnum} end, - E = case HandshakeEnv#handshake_env.sni_guided_cert_selection of + E3 = case HandshakeEnv#handshake_env.sni_guided_cert_selection of false -> E2; true -> E2#{sni => #sni{hostname = ""}} end, + E = case HandshakeEnv#handshake_env.early_data_accepted of + false -> + E3; + true -> + E3#{early_data => #early_data_indication{}} + end, #encrypted_extensions{ extensions = E }. @@ -595,9 +608,11 @@ do_start(#client_hello{cipher_suites = ClientCiphers, supported_groups := ServerGroups0, alpn_preferred_protocols := ALPNPreferredProtocols, keep_secrets := KeepSecrets, - honor_cipher_order := HonorCipherOrder}} = State0) -> + honor_cipher_order := HonorCipherOrder, + early_data := EarlyDataEnabled}} = State0) -> SNI = maps:get(sni, Extensions, undefined), ClientGroups0 = maps:get(elliptic_curves, Extensions, undefined), + EarlyDataIndication = maps:get(early_data, Extensions, undefined), {Ref,Maybe} = maybe(), try ClientGroups = Maybe(get_supported_groups(ClientGroups0)), @@ -618,7 +633,7 @@ do_start(#client_hello{cipher_suites = ClientCiphers, CookieExt = maps:get(cookie, Extensions, undefined), Cookie = get_cookie(CookieExt), - + #state{connection_states = ConnectionStates0, session = #session{own_certificates = [Cert | _]}} = State1 = Maybe(ssl_gen_statem:handle_sni_extension(SNI, State0)), @@ -668,14 +683,14 @@ do_start(#client_hello{cipher_suites = ClientCiphers, State2 end, - State = update_start_state(State3, - #{cipher => Cipher, - key_share => KeyShare, - session_id => SessionId, - group => Group, - sign_alg => SelectedSignAlg, - peer_public_key => ClientPubKey, - alpn => ALPNProtocol}), + State4 = update_start_state(State3, + #{cipher => Cipher, + key_share => KeyShare, + session_id => SessionId, + group => Group, + sign_alg => SelectedSignAlg, + peer_public_key => ClientPubKey, + alpn => ALPNProtocol}), %% 4.1.4. Hello Retry Request %% @@ -683,13 +698,15 @@ do_start(#client_hello{cipher_suites = ClientCiphers, %% message if it is able to find an acceptable set of parameters but the %% ClientHello does not contain sufficient information to proceed with %% the handshake. - case Maybe(send_hello_retry_request(State, ClientPubKey, KeyShare, SessionId)) of + case Maybe(send_hello_retry_request(State4, ClientPubKey, KeyShare, SessionId)) of {_, start} = NextStateTuple -> NextStateTuple; - {_, negotiated} = NextStateTuple -> + {State5, negotiated} -> + %% Determine if early data is accepted + State = handle_early_data(State5, EarlyDataEnabled, EarlyDataIndication), %% Exclude any incompatible PSKs. PSK = Maybe(handle_pre_shared_key(State, OfferedPSKs, Cipher)), - Maybe(session_resumption(NextStateTuple, PSK)) + Maybe(session_resumption({State, negotiated}, PSK)) end catch {Ref, #alert{} = Alert} -> @@ -790,6 +807,9 @@ do_start(#server_hello{cipher_suite = SelectedCipherSuite, do_negotiated({start_handshake, PSK0}, #state{connection_states = ConnectionStates0, + handshake_env = + #handshake_env{ + early_data_accepted = EarlyDataAccepted}, static_env = #static_env{protocol_cb = Connection}, session = #session{session_id = SessionId, ecc = SelectedGroup, @@ -802,7 +822,6 @@ do_negotiated({start_handshake, PSK0}, ssl_record:pending_connection_state(ConnectionStates0, read), #security_parameters{prf_algorithm = HKDF} = SecParamsR, - {Ref,Maybe} = maybe(), try %% Create server_hello @@ -810,39 +829,52 @@ do_negotiated({start_handshake, PSK0}, State1 = Connection:queue_handshake(ServerHello, State0), %% D.4. Middlebox Compatibility Mode State2 = maybe_queue_change_cipher_spec(State1, last), - {State3, _} = Connection:send_handshake_flight(State2), PSK = get_pre_shared_key(PSK0, HKDF), - State4 = + State3 = calculate_handshake_secrets(ClientPublicKey, ServerPrivateKey, SelectedGroup, - PSK, State3), + PSK, State2), - State5 = ssl_record:step_encryption_state(State4), + %% Step only write state if early_data is accepted + State4 = + case EarlyDataAccepted of + true -> + ssl_record:step_encryption_state_write(State3); + false -> + %% Read state is overwritten when hanshake secrets are set. + %% Trial_decryption and early_data_limit must be set here! + update_current_read( + ssl_record:step_encryption_state(State3), + true, %% trial_decryption + false %% early data limit + ) + + end, %% Create EncryptedExtensions - EncryptedExtensions = encrypted_extensions(State5), + EncryptedExtensions = encrypted_extensions(State4), %% Encode EncryptedExtensions - State6 = Connection:queue_handshake(EncryptedExtensions, State5), + State5 = Connection:queue_handshake(EncryptedExtensions, State4), %% Create and send CertificateRequest ({verify, verify_peer}) - {State7, NextState} = maybe_send_certificate_request(State6, SslOpts, PSK0), + {State6, NextState} = maybe_send_certificate_request(State5, SslOpts, PSK0), %% Create and send Certificate (if PSK is undefined) - State8 = Maybe(maybe_send_certificate(State7, PSK0)), + State7 = Maybe(maybe_send_certificate(State6, PSK0)), %% Create and send CertificateVerify (if PSK is undefined) - State9 = Maybe(maybe_send_certificate_verify(State8, PSK0)), + State8 = Maybe(maybe_send_certificate_verify(State7, PSK0)), %% Create Finished - Finished = finished(State9), + Finished = finished(State8), %% Encode Finished - State10= Connection:queue_handshake(Finished, State9), + State9 = Connection:queue_handshake(Finished, State8), %% Send first flight - {State, _} = Connection:send_handshake_flight(State10), + {State, _} = Connection:send_handshake_flight(State9), {State, NextState} @@ -907,18 +939,20 @@ do_wait_finished(#finished{verify_data = VerifyData}, Maybe(validate_finished(State0, VerifyData)), %% D.4. Middlebox Compatibility Mode State1 = maybe_queue_change_cipher_spec(State0, first), + %% Signal change of cipher + State2 = maybe_send_end_of_early_data(State1), %% Maybe send Certificate + CertificateVerify - State2 = Maybe(maybe_queue_cert_cert_cv(State1)), - Finished = finished(State2), + State3 = Maybe(maybe_queue_cert_cert_cv(State2)), + Finished = finished(State3), %% Encode Finished - State3 = Connection:queue_handshake(Finished, State2), + State4 = Connection:queue_handshake(Finished, State3), %% Send first flight - {State4, _} = Connection:send_handshake_flight(State3), - State5 = calculate_traffic_secrets(State4), - State6 = maybe_calculate_resumption_master_secret(State5), - State7 = forget_master_secret(State6), + {State5, _} = Connection:send_handshake_flight(State4), + State6 = calculate_traffic_secrets(State5), + State7 = maybe_calculate_resumption_master_secret(State6), + State8 = forget_master_secret(State7), %% Configure traffic keys - ssl_record:step_encryption_state(State7) + ssl_record:step_encryption_state(State8) catch {Ref, #alert{} = Alert} -> Alert @@ -973,8 +1007,8 @@ do_wait_sh(#server_hello{cipher_suite = SelectedCipherSuite, PSK = Maybe(get_pre_shared_key(SessionTickets, UseTicket, HKDFAlgo, SelectedIdentity)), State3 = calculate_handshake_secrets(ServerPublicKey, ClientPrivateKey, SelectedGroup, PSK, State2), - State4 = ssl_record:step_encryption_state(State3), - + %% State4 = ssl_record:step_encryption_state(State3), + State4 = ssl_record:step_encryption_state_read(State3), {State4, wait_ee} catch @@ -989,6 +1023,7 @@ do_wait_ee(#encrypted_extensions{extensions = Extensions}, State0) -> ALPNProtocol0 = maps:get(alpn, Extensions, undefined), ALPNProtocol = get_alpn(ALPNProtocol0), + EarlyDataIndication = maps:get(early_data, Extensions, undefined), {Ref, Maybe} = maybe(), @@ -996,14 +1031,17 @@ do_wait_ee(#encrypted_extensions{extensions = Extensions}, State0) -> %% RFC 6066: handle received/expected maximum fragment length Maybe(maybe_max_fragment_length(Extensions, State0)), + %% Check if early_data is accepted/rejected + State1 = maybe_check_early_data_indication(EarlyDataIndication, State0), + %% Go to state 'wait_finished' if using PSK. - Maybe(maybe_resumption(State0)), + Maybe(maybe_resumption(State1)), %% Update state - #state{handshake_env = HsEnv} = State0, - State1 = State0#state{handshake_env = HsEnv#handshake_env{alpn = ALPNProtocol}}, + #state{handshake_env = HsEnv} = State1, + State2 = State1#state{handshake_env = HsEnv#handshake_env{alpn = ALPNProtocol}}, - {State1, wait_cert_cr} + {State2, wait_cert_cr} catch {Ref, {State, StateName}} -> {State, StateName}; @@ -1032,6 +1070,25 @@ do_wait_cert_cr(#certificate_request_1_3{} = CertificateRequest, State0) -> end. +do_wait_eoed(#end_of_early_data{}, State0) -> + {Ref,_Maybe} = maybe(), + try + %% Step read state to enable reading handshake messages from the client. + %% Write state is already stepped in state 'negotiated'. + State1 = ssl_record:step_encryption_state_read(State0), + + %% Early data has been received, no more early data is expected. + HsEnv = (State1#state.handshake_env)#handshake_env{early_data_accepted = false}, + State2 = State1#state{handshake_env = HsEnv}, + {State2, wait_finished} + catch + {Ref, #alert{} = Alert} -> + {Alert, State0}; + {Ref, {#alert{} = Alert, State}} -> + {Alert, State} + end. + + %% For reasons of backward compatibility with middleboxes (see %% Appendix D.4), the HelloRetryRequest message uses the same structure %% as the ServerHello, but with Random set to the special value of the @@ -1225,12 +1282,33 @@ session_resumption({#state{ssl_options = #{session_tickets := disabled}} = State session_resumption({#state{ssl_options = #{session_tickets := Tickets}} = State, negotiated}, undefined) when Tickets =/= disabled -> {ok, {State, negotiated}}; -session_resumption({#state{ssl_options = #{session_tickets := Tickets}} = State0, negotiated}, PSK) +session_resumption({#state{ssl_options = #{session_tickets := Tickets}, + handshake_env = #handshake_env{ + early_data_accepted = false}} = State0, negotiated}, PSK) when Tickets =/= disabled -> State = handle_resumption(State0, ok), - {ok, {State, negotiated, PSK}}. - - + {ok, {State, negotiated, PSK}}; +session_resumption({#state{ssl_options = #{session_tickets := Tickets}, + handshake_env = #handshake_env{ + early_data_accepted = true}} = State0, negotiated}, PSK0) + when Tickets =/= disabled -> + State1 = handle_resumption(State0, ok), + %% TODO Refactor PSK-tuple {Index, PSK}, index might not be needed. + {_ , PSK} = PSK0, + State2 = calculate_client_early_traffic_secret(State1, PSK), + %% Set 0-RTT traffic keys for reading early_data + State3 = ssl_record:step_encryption_state_read(State2), + State = update_current_read(State3, true, true), + {ok, {State, negotiated, PSK0}}. + +%% Session resumption with early_data +maybe_send_certificate_request(#state{ + handshake_env = + #handshake_env{ + early_data_accepted = true}} = State, + _, PSK) when PSK =/= undefined -> + %% Go wait for End of Early Data + {State, wait_eoed}; %% Do not send CR during session resumption maybe_send_certificate_request(State, _, PSK) when PSK =/= undefined -> {State, wait_finished}; @@ -1508,16 +1586,15 @@ calculate_handshake_secrets(PublicKey, PrivateKey, SelectedGroup, PSK, %% Calculate [sender]_handshake_traffic_secret {Messages, _} = HHistory, - ClientHSTrafficSecret = tls_v1:client_handshake_traffic_secret(HKDFAlgo, HandshakeSecret, lists:reverse(Messages)), ServerHSTrafficSecret = tls_v1:server_handshake_traffic_secret(HKDFAlgo, HandshakeSecret, lists:reverse(Messages)), %% Calculate traffic keys - #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), - {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ClientHSTrafficSecret), - {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ServerHSTrafficSecret), + KeyLength = tls_v1:key_length(CipherSuite), + {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, ClientHSTrafficSecret), + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, ServerHSTrafficSecret), %% Calculate Finished Keys ReadFinishedKey = tls_v1:finished_key(ClientHSTrafficSecret, HKDFAlgo), @@ -1530,6 +1607,67 @@ calculate_handshake_secrets(PublicKey, PrivateKey, SelectedGroup, PSK, ReadKey, ReadIV, ReadFinishedKey, WriteKey, WriteIV, WriteFinishedKey). +%% Server +calculate_client_early_traffic_secret(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = {Hist, _}}} = State, PSK) -> + + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{cipher_suite = CipherSuite} = SecParamsR, + #{cipher := Cipher, + prf := HKDF} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + calculate_client_early_traffic_secret(Hist, PSK, Cipher, HKDF, State). + +%% Client +calculate_client_early_traffic_secret( + ClientHello, PSK, Cipher, HKDFAlgo, + #state{connection_states = ConnectionStates, + ssl_options = #{keep_secrets := KeepSecrets}, + static_env = #static_env{role = Role}} = State0) -> + EarlySecret = tls_v1:key_schedule(early_secret, HKDFAlgo , {psk, PSK}), + ClientEarlyTrafficSecret = + tls_v1:client_early_traffic_secret(HKDFAlgo, EarlySecret, ClientHello), + %% Calculate traffic key + KeyLength = ssl_cipher:key_material(Cipher), + {Key, IV} = + tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, ClientEarlyTrafficSecret), + %% Update pending connection states + case Role of + client -> + PendingWrite0 = ssl_record:pending_connection_state(ConnectionStates, write), + PendingWrite1 = maybe_store_early_data_secret(KeepSecrets, ClientEarlyTrafficSecret, + PendingWrite0), + PendingWrite = update_connection_state(PendingWrite1, undefined, undefined, + undefined, + Key, IV, undefined), + State0#state{connection_states = ConnectionStates#{pending_write => PendingWrite}}; + server -> + PendingRead0 = ssl_record:pending_connection_state(ConnectionStates, read), + PendingRead1 = maybe_store_early_data_secret(KeepSecrets, ClientEarlyTrafficSecret, + PendingRead0), + PendingRead2 = update_connection_state(PendingRead1, undefined, undefined, + undefined, + Key, IV, undefined), + %% Signal start of early data. This is to prevent handshake messages to be + %% counted in max_early_data_size. + PendingRead = PendingRead2#{count_early_data => true}, + State0#state{connection_states = ConnectionStates#{pending_read => PendingRead}} + end. + +update_current_read(#state{connection_states = CS} = State, TrialDecryption, EarlyDataLimit) -> + Read0 = ssl_record:current_connection_state(CS, read), + Read = Read0#{trial_decryption => TrialDecryption, + early_data_limit => EarlyDataLimit}, + State#state{connection_states = CS#{current_read => Read}}. + +maybe_store_early_data_secret(true, EarlySecret, State) -> + #{security_parameters := SecParams0} = State, + SecParams = SecParams0#security_parameters{client_early_data_secret = EarlySecret}, + State#{security_parameters := SecParams}; +maybe_store_early_data_secret(false, _, State) -> + State. %% Server get_pre_shared_key(undefined, HKDFAlgo) -> @@ -1554,7 +1692,7 @@ get_pre_shared_key(manual = SessionTickets, UseTicket, HKDFAlgo, SelectedIdentit {ok, binary:copy(<<0>>, ssl_cipher:hash_size(HKDFAlgo))}; illegal_parameter -> {error, ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)}; - {_, PSK} -> + {_, PSK, _, _, _} -> {ok, PSK} end; get_pre_shared_key(auto = SessionTickets, UseTicket, HKDFAlgo, SelectedIdentity) -> @@ -1566,19 +1704,35 @@ get_pre_shared_key(auto = SessionTickets, UseTicket, HKDFAlgo, SelectedIdentity) illegal_parameter -> tls_client_ticket_store:unlock_tickets(self(), UseTicket), {error, ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)}; - {Key, PSK} -> + {Key, PSK, _, _, _} -> tls_client_ticket_store:remove_tickets([Key]), %% Remove single-use ticket tls_client_ticket_store:unlock_tickets(self(), UseTicket -- [Key]), {ok, PSK} end. - +%% +%% Early Data +get_pre_shared_key_early_data(SessionTickets, UseTicket) -> + TicketData = get_ticket_data(self(), SessionTickets, UseTicket), + case choose_psk(TicketData, 0) of + undefined -> %% Should not happen + {error, ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)}; + illegal_parameter -> + {error, ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)}; + {_Key, PSK, Cipher, HKDF, MaxSize} -> + {ok, {PSK, Cipher, HKDF, MaxSize}} + end. choose_psk(undefined, _) -> undefined; choose_psk([], _) -> illegal_parameter; -choose_psk([{Key, SelectedIdentity, _, PSK, _, _}|_], SelectedIdentity) -> - {Key, PSK}; +choose_psk([#ticket_data{ + key = Key, + pos = SelectedIdentity, + psk = PSK, + cipher_suite = {Cipher, HKDF}, + max_size = MaxSize}|_], SelectedIdentity) -> + {Key, PSK, Cipher, HKDF, MaxSize}; choose_psk([_|T], SelectedIdentity) -> choose_psk(T, SelectedIdentity). @@ -1608,9 +1762,9 @@ calculate_traffic_secrets(#state{ tls_v1:server_application_traffic_secret_0(HKDFAlgo, MasterSecret, lists:reverse(Messages)), %% Calculate traffic keys - #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), - {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ClientAppTrafficSecret0), - {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ServerAppTrafficSecret0), + KeyLength = tls_v1:key_length(CipherSuite), + {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, ClientAppTrafficSecret0), + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, ServerAppTrafficSecret0), update_pending_connection_states(State0, MasterSecret, undefined, ClientAppTrafficSecret0, ServerAppTrafficSecret0, @@ -2396,18 +2550,18 @@ maybe_add_binders(Hello0, {[HRR,MessageHash|_], _}, TicketData, Version) when Ve maybe_add_binders(Hello, _, _, Version) when Version =< {3,3} -> Hello. - create_binders(Context, TicketData) -> create_binders(Context, TicketData, []). %% create_binders(_, [], Acc) -> lists:reverse(Acc); -create_binders(Context, [{_, _, _, PSK, _, HKDF}|T], Acc) -> +create_binders(Context, [#ticket_data{ + psk = PSK, + cipher_suite = {_, HKDF}}|T], Acc) -> FinishedKey = calculate_finished_key(PSK, HKDF), Binder = calculate_binder(FinishedKey, HKDF, Context), create_binders(Context, T, [Binder|Acc]). - %% Removes the binders list from the ClientHello. %% opaque PskBinderEntry<32..255>; %% @@ -2441,6 +2595,18 @@ truncate_client_hello(HelloBin0) -> {Truncated, _} = split_binary(HelloBin0, size(HelloBin0) - BindersSize - 2), Truncated. +maybe_add_early_data_indication(#client_hello{ + extensions = Extensions0} = ClientHello, + EarlyData, + Version) + when Version =:= {3,4} andalso + is_binary(EarlyData) andalso + size(EarlyData) > 0 -> + Extensions = Extensions0#{early_data => + #early_data_indication{}}, + ClientHello#client_hello{extensions = Extensions}; +maybe_add_early_data_indication(ClientHello, _, _) -> + ClientHello. %% The PskBinderEntry is computed in the same way as the Finished %% message (Section 4.4.4) but with the BaseKey being the binder_key @@ -2475,6 +2641,7 @@ update_binders(#client_hello{extensions = maybe_automatic_session_resumption(#state{ ssl_options = #{versions := [Version|_], ciphers := UserSuites, + early_data := EarlyData, session_tickets := SessionTickets, server_name_indication := SNI} = SslOpts0 } = State0) @@ -2482,7 +2649,13 @@ maybe_automatic_session_resumption(#state{ SessionTickets =:= auto -> AvailableCipherSuites = ssl_handshake:available_suites(UserSuites, Version), HashAlgos = cipher_hash_algos(AvailableCipherSuites), - UseTicket = tls_client_ticket_store:find_ticket(self(), HashAlgos, SNI), + Ciphers = ciphers_for_early_data(AvailableCipherSuites), + %% Find a pair of tickets KeyPair = {Ticket0, Ticket2} where Ticket0 satisfies + %% requirements for early_data and session resumption while Ticket2 can only + %% be used for session resumption. + EarlyDataSize = early_data_size(EarlyData), + KeyPair = tls_client_ticket_store:find_ticket(self(), Ciphers, HashAlgos, SNI, EarlyDataSize), + UseTicket = choose_ticket(KeyPair, EarlyData), tls_client_ticket_store:lock_tickets(self(), [UseTicket]), State = State0#state{ssl_options = SslOpts0#{use_ticket => [UseTicket]}}, {[UseTicket], State}; @@ -2491,6 +2664,144 @@ maybe_automatic_session_resumption(#state{ } = State) -> {UseTicket, State}. +early_data_size(undefined) -> + undefined; +early_data_size(EarlyData) when is_binary(EarlyData) -> + byte_size(EarlyData). + +%% Choose a ticket based on the intention of the user. The first argument is +%% a 2-tuple of ticket keys where the first element refers to a ticket that +%% fulfills all criteria for sending early_data (hash, cipher, early data size). +%% Second argument refers to a ticket that can only be used for session +%% resumption. +choose_ticket({Key, _}, _) when Key =/= undefined -> + Key; +choose_ticket({_, Key}, EarlyData) when EarlyData =:= undefined -> + Key; +choose_ticket(_, _) -> + %% No tickets found that fulfills the original intention of the user + %% (sending early_data). It is possible to do session resumption but + %% in that case the configured early data would have to be removed + %% and that would contradict the will of the user. Returning undefined + %% here prevents session resumption instead. + undefined. + +maybe_send_early_data(#state{ + handshake_env = #handshake_env{tls_handshake_history = {Hist, _}}, + protocol_specific = #{sender := _Sender}, + ssl_options = #{versions := [Version|_], + use_ticket := UseTicket, + session_tickets := SessionTickets, + early_data := EarlyData} = _SslOpts0 + } = State0) when Version =:= {3,4} andalso + UseTicket =/= [undefined] andalso + EarlyData =/= undefined -> + %% D.4. Middlebox Compatibility Mode + State1 = maybe_queue_change_cipher_spec(State0, last), + %% Early traffic secret + EarlyDataSize = early_data_size(EarlyData), + case get_pre_shared_key_early_data(SessionTickets, UseTicket) of + {ok, {PSK, Cipher, HKDF, MaxSize}} when EarlyDataSize =< MaxSize -> + State2 = calculate_client_early_traffic_secret(Hist, PSK, Cipher, HKDF, State1), + %% Set 0-RTT traffic keys for sending early_data and EndOfEarlyData + State3 = ssl_record:step_encryption_state_write(State2), + {ok, encode_early_data(Cipher, State3)}; + {ok, {_, _, _, _MaxSize}} -> + {error, ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER, too_much_early_data)}; + {error, Alert} -> + {error, Alert} + end; +maybe_send_early_data(State) -> + {ok, State}. + +encode_early_data(Cipher, + #state{ + flight_buffer = Flight0, + protocol_specific = #{sender := _Sender}, + ssl_options = #{versions := [Version|_], + early_data := EarlyData} = _SslOpts0 + } = State0) -> + #state{connection_states = + #{current_write := + #{security_parameters := SecurityParameters0} = Write0} = ConnectionStates0} = State0, + BulkCipherAlgo = ssl_cipher:bulk_cipher_algorithm(Cipher), + SecurityParameters = SecurityParameters0#security_parameters{ + cipher_type = ?AEAD, + bulk_cipher_algorithm = BulkCipherAlgo}, + Write = Write0#{security_parameters => SecurityParameters}, + ConnectionStates1 = ConnectionStates0#{current_write => Write}, + {BinEarlyData, ConnectionStates} = tls_record:encode_data([EarlyData], Version, ConnectionStates1), + State0#state{connection_states = ConnectionStates, + flight_buffer = Flight0 ++ [BinEarlyData]}. + +maybe_send_end_of_early_data( + #state{ + handshake_env = #handshake_env{early_data_accepted = true}, + protocol_specific = #{sender := _Sender}, + ssl_options = #{versions := [Version|_], + use_ticket := UseTicket, + early_data := EarlyData}, + static_env = #static_env{protocol_cb = Connection} + } = State0) when Version =:= {3,4} andalso + UseTicket =/= [undefined] andalso + EarlyData =/= undefined -> + %% EndOfEarlydata is encrypted with the 0-RTT traffic keys + State1 = Connection:queue_handshake(#end_of_early_data{}, State0), + %% Use handshake keys after EndOfEarlyData is sent + ssl_record:step_encryption_state_write(State1); +maybe_send_end_of_early_data(State) -> + State. + +maybe_check_early_data_indication(EarlyDataIndication, + #state{ + handshake_env = HsEnv, + ssl_options = #{versions := [Version|_], + use_ticket := UseTicket, + early_data := EarlyData} + } = State) when Version =:= {3,4} andalso + UseTicket =/= [undefined] andalso + EarlyData =/= undefined andalso + EarlyDataIndication =/= undefined -> + signal_user_early_data(State, accepted), + State#state{handshake_env = HsEnv#handshake_env{early_data_accepted = true}}; +maybe_check_early_data_indication(EarlyDataIndication, + #state{ + protocol_specific = #{sender := _Sender}, + ssl_options = #{versions := [Version|_], + use_ticket := UseTicket, + early_data := EarlyData} = _SslOpts0 + } = State) when Version =:= {3,4} andalso + UseTicket =/= [undefined] andalso + EarlyData =/= undefined andalso + EarlyDataIndication =:= undefined -> + signal_user_early_data(State, rejected), + %% Use handshake keys if early_data is rejected. + ssl_record:step_encryption_state_write(State); +maybe_check_early_data_indication(_, State) -> + %% Use handshake keys if there is no early_data. + ssl_record:step_encryption_state_write(State). + +signal_user_early_data(#state{ + connection_env = + #connection_env{ + user_application = {_, User}}, + static_env = + #static_env{ + socket = Socket, + protocol_cb = Connection, + transport_cb = Transport, + trackers = Trackers}} = State, + Result) -> + CPids = Connection:pids(State), + SslSocket = Connection:socket(CPids, Transport, Socket, Trackers), + User ! {ssl, SslSocket, {early_data, Result}}. + +handle_early_data(State, enabled, #early_data_indication{}) -> + %% Accept early data + HsEnv = (State#state.handshake_env)#handshake_env{early_data_accepted = true}, + State#state{handshake_env = HsEnv}; +handle_early_data(State, _, _) -> + State. cipher_hash_algos(Ciphers) -> Fun = fun(Cipher) -> @@ -2499,6 +2810,14 @@ cipher_hash_algos(Ciphers) -> end, lists:map(Fun, Ciphers). +ciphers_for_early_data(CipherSuites0) -> + %% Use only supported TLS 1.3 cipher suites + Supported = lists:filter(fun(CipherSuite) -> + lists:member(CipherSuite, tls_v1:exclusive_suites(4)) end, + CipherSuites0), + %% Return supported block cipher algorithms + lists:map(fun(#{cipher := Cipher}) -> Cipher end, + lists:map(fun ssl_cipher_format:suite_bin_to_map/1, Supported)). get_ticket_data(_, undefined, _) -> undefined; @@ -2523,30 +2842,43 @@ process_user_tickets([H|T], Acc, N) -> process_user_tickets(T, [TicketData|Acc], N + 1) end. -process_ticket(Bin, N) when is_binary(Bin) -> - try erlang:binary_to_term(Bin, [safe]) of - #{hkdf := HKDF, - sni := _SNI, %% TODO: Handle SNI? - psk := PSK, - timestamp := Timestamp, - ticket := NewSessionTicket} -> - #new_session_ticket{ - ticket_lifetime = _LifeTime, - ticket_age_add = AgeAdd, - ticket_nonce = Nonce, - ticket = Ticket, - extensions = _Extensions - } = NewSessionTicket, - TicketAge = erlang:system_time(seconds) - Timestamp, - ObfuscatedTicketAge = obfuscate_ticket_age(TicketAge, AgeAdd), - Identity = #psk_identity{ - identity = Ticket, - obfuscated_ticket_age = ObfuscatedTicketAge}, - {undefined, N, Identity, PSK, Nonce, HKDF}; - _Else -> - error - catch error:badarg -> - error +%% Used when session_tickets = manual +process_ticket(#{cipher_suite := CipherSuite, + sni := _SNI, %% TODO user's responsibility to handle SNI? + psk := PSK, + timestamp := Timestamp, + ticket := NewSessionTicket}, N) -> + #new_session_ticket{ + ticket_lifetime = _LifeTime, + ticket_age_add = AgeAdd, + ticket_nonce = Nonce, + ticket = Ticket, + extensions = Extensions + } = NewSessionTicket, + TicketAge = erlang:system_time(seconds) - Timestamp, + ObfuscatedTicketAge = obfuscate_ticket_age(TicketAge, AgeAdd), + Identity = #psk_identity{ + identity = Ticket, + obfuscated_ticket_age = ObfuscatedTicketAge}, + MaxEarlyData = get_max_early_data(Extensions), + #ticket_data{ + key = undefined, + pos = N, + identity = Identity, + psk = PSK, + nonce = Nonce, + cipher_suite = CipherSuite, + max_size = MaxEarlyData}; +process_ticket(_, _) -> + error. + +get_max_early_data(Extensions) -> + EarlyDataIndication = maps:get(early_data, Extensions, undefined), + case EarlyDataIndication of + undefined -> + undefined; + #early_data_indication_nst{indication = MaxSize} -> + MaxSize end. %% The "obfuscated_ticket_age" diff --git a/lib/ssl/src/tls_handshake_1_3.hrl b/lib/ssl/src/tls_handshake_1_3.hrl index 9f41c84708..d506821f6c 100644 --- a/lib/ssl/src/tls_handshake_1_3.hrl +++ b/lib/ssl/src/tls_handshake_1_3.hrl @@ -87,9 +87,11 @@ %% RFC 8446 4.2.10. Early Data Indication -record(empty, { }). --record(early_data_indication, { - indication % uint32 max_early_data_size (new_session_ticket) | - %% #empty{} (client_hello, encrypted_extensions) + +%% #empty{} (client_hello, encrypted_extensions) +-record(early_data_indication, {}). +-record(early_data_indication_nst, { + indication % uint32 max_early_data_size (new_session_ticket) }). %% RFC 8446 4.2.11. Pre-Shared Key Extension diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 6e5c30760d..9ec5490aa6 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -33,7 +33,9 @@ -include_lib("kernel/include/logger.hrl"). %% Handling of incoming data --export([get_tls_records/5, init_connection_states/2]). +-export([get_tls_records/5, + init_connection_states/2, + init_connection_states/3]). %% Encoding TLS records -export([encode_handshake/3, encode_alert_record/3, @@ -64,16 +66,29 @@ %% Handling of incoming data %%==================================================================== %%-------------------------------------------------------------------- --spec init_connection_states(client | server, one_n_minus_one | zero_n | disabled) -> - ssl_record:connection_states(). +-spec init_connection_states(Role, BeastMitigation) -> + ssl_record:connection_states() when + Role :: client | server, + BeastMitigation :: one_n_minus_one | zero_n | disabled. + %% %% Description: Creates a connection_states record with appropriate %% values for the initial SSL connection setup. %%-------------------------------------------------------------------- init_connection_states(Role, BeastMitigation) -> + MaxEarlyDataSize = ssl_config:get_max_early_data_size(), + init_connection_states(Role, BeastMitigation, MaxEarlyDataSize). +%% +-spec init_connection_states(Role, BeastMitigation, MaxEarlyDataSize) -> + ssl_record:connection_states() when + Role :: client | server, + BeastMitigation :: one_n_minus_one | zero_n | disabled, + MaxEarlyDataSize :: non_neg_integer(). + +init_connection_states(Role, BeastMitigation, MaxEarlyDataSize) -> ConnectionEnd = ssl_record:record_protocol_role(Role), - Current = initial_connection_state(ConnectionEnd, BeastMitigation), - Pending = ssl_record:empty_connection_state(ConnectionEnd, BeastMitigation), + Current = initial_connection_state(ConnectionEnd, BeastMitigation, MaxEarlyDataSize), + Pending = ssl_record:empty_connection_state(ConnectionEnd, BeastMitigation, MaxEarlyDataSize), #{current_read => Current, pending_read => Pending, current_write => Current, @@ -181,7 +196,8 @@ encode_data(Data, Version, %%-------------------------------------------------------------------- -spec decode_cipher_text(tls_version(), #ssl_tls{}, ssl_record:connection_states(), boolean()) -> - {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. + {#ssl_tls{} | trial_decryption_failed, + ssl_record:connection_states()}| #alert{}. %% %% Description: Decode cipher text %%-------------------------------------------------------------------- @@ -465,7 +481,7 @@ split_iovec(Data, MaximumFragmentLength) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -initial_connection_state(ConnectionEnd, BeastMitigation) -> +initial_connection_state(ConnectionEnd, BeastMitigation, MaxEarlyDataSize) -> #{security_parameters => ssl_record:initial_security_params(ConnectionEnd), sequence_number => 0, @@ -476,7 +492,10 @@ initial_connection_state(ConnectionEnd, BeastMitigation) -> secure_renegotiation => undefined, client_verify_data => undefined, server_verify_data => undefined, - max_fragment_length => undefined + max_early_data_size => MaxEarlyDataSize, + max_fragment_length => undefined, + trial_decryption => false, + early_data_limit => false }. %% Used by logging to recreate the received bytes diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl index a9ba415099..3e42e3bf97 100644 --- a/lib/ssl/src/tls_record_1_3.erl +++ b/lib/ssl/src/tls_record_1_3.erl @@ -107,7 +107,8 @@ encode_iolist(Type, Data, ConnectionStates0) -> %%-------------------------------------------------------------------- -spec decode_cipher_text(#ssl_tls{}, ssl_record:connection_states()) -> - {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. + {#ssl_tls{} | trial_decryption_failed, + ssl_record:connection_states()}| #alert{}. %% %% Description: Decode cipher text, use legacy type ssl_tls instead of tls_cipher_text %% in decoding context so that we can reuse the code from erlier versions. @@ -124,12 +125,25 @@ decode_cipher_text(#ssl_tls{type = ?OPAQUE_TYPE, #security_parameters{ cipher_type = ?AEAD, bulk_cipher_algorithm = - BulkCipherAlgo} + BulkCipherAlgo}, + max_early_data_size := MaxEarlyDataSize0, + trial_decryption := TrialDecryption, + early_data_limit := EarlyDataLimit } = ReadState0} = ConnectionStates0) -> case decipher_aead(CipherFragment, BulkCipherAlgo, Key, Seq, IV, TagLen) of + #alert{} when TrialDecryption =:= true andalso + MaxEarlyDataSize0 > 0 -> %% Trial decryption + trial_decrypt(ConnectionStates0, ReadState0, MaxEarlyDataSize0, + BulkCipherAlgo, CipherFragment); #alert{} = Alert -> Alert; - PlainFragment -> + PlainFragment0 when EarlyDataLimit =:= true andalso + MaxEarlyDataSize0 > 0 -> + PlainFragment = remove_padding(PlainFragment0), + process_early_data(ConnectionStates0, ReadState0, MaxEarlyDataSize0, Seq, + BulkCipherAlgo, CipherFragment, PlainFragment); + PlainFragment0 -> + PlainFragment = remove_padding(PlainFragment0), ConnectionStates = ConnectionStates0#{current_read => ReadState0#{sequence_number => Seq + 1}}, @@ -189,9 +203,55 @@ decode_cipher_text(#ssl_tls{type = Type}, _) -> %% Version mismatch is already asserted ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, {record_type_mismatch, Type}). + + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +trial_decrypt(ConnectionStates0, ReadState0, MaxEarlyDataSize0, + BulkCipherAlgo, CipherFragment) -> + MaxEarlyDataSize = update_max_early_date_size(MaxEarlyDataSize0, BulkCipherAlgo, CipherFragment), + ConnectionStates = + ConnectionStates0#{current_read => + ReadState0#{max_early_data_size => MaxEarlyDataSize}}, + if MaxEarlyDataSize < 0 -> + %% More early data is trial decrypted as the configured limit + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed); + true -> + {trial_decryption_failed, ConnectionStates} + end. + +process_early_data(ConnectionStates0, ReadState0, _MaxEarlyDataSize0, Seq, + _BulkCipherAlgo, _CipherFragment, PlainFragment) + when PlainFragment =:= <<5,0,0,0,22>> -> + %% struct { + %% opaque content[TLSPlaintext.length]; <<5,0,0,0>> - 5 = EndOfEarlyData + %% 0 = (uint24) size + %% ContentType type; <<22>> - Handshake + %% uint8 zeros[length_of_padding]; <<>> - no padding + %% } TLSInnerPlaintext; + %% EndOfEarlyData should not be counted into early data + ConnectionStates = + ConnectionStates0#{current_read => + ReadState0#{sequence_number => Seq + 1}}, + {decode_inner_plaintext(PlainFragment), ConnectionStates}; +process_early_data(ConnectionStates0, ReadState0, MaxEarlyDataSize0, Seq, + BulkCipherAlgo, CipherFragment, PlainFragment) -> + %% First packet is deciphered anyway so we must check if more early data is received + %% than the configured limit (max_early_data_size). + MaxEarlyDataSize = + update_max_early_date_size(MaxEarlyDataSize0, BulkCipherAlgo, CipherFragment), + if MaxEarlyDataSize < 0 -> + %% Too much early data received, send alert unexpected_message + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, too_much_early_data); + true -> + ConnectionStates = + ConnectionStates0#{current_read => + ReadState0#{sequence_number => Seq + 1, + max_early_data_size => MaxEarlyDataSize}}, + {decode_inner_plaintext(PlainFragment), ConnectionStates} + end. + inner_plaintext(Type, Data, Length) -> #inner_plaintext{ content = Data, @@ -299,8 +359,6 @@ aead_ciphertext_split(CipherTextFragment, TagLen) decode_inner_plaintext(PlainText) -> case binary:last(PlainText) of - 0 -> - decode_inner_plaintext(init_binary(PlainText)); Type when Type =:= ?APPLICATION_DATA orelse Type =:= ?HANDSHAKE orelse Type =:= ?ALERT -> @@ -315,3 +373,30 @@ init_binary(B) -> {Init, _} = split_binary(B, byte_size(B) - 1), Init. + +remove_padding(InnerPlainText) -> + case binary:last(InnerPlainText) of + 0 -> + remove_padding(init_binary(InnerPlainText)); + _ -> + InnerPlainText + end. + +update_max_early_date_size(MaxEarlyDataSize, BulkCipherAlgo, CipherFragment) -> + %% CipherFragment is the binary encoded form of a TLSInnerPlaintext: + %% + %% struct { + %% opaque content[TLSPlaintext.length]; + %% ContentType type; + %% uint8 zeros[length_of_padding]; + %% } TLSInnerPlaintext; + %% + TypeLen = 1, + PaddingLen = 0, %% TODO Update formula when padding is implemented! + MaxEarlyDataSize - (byte_size(CipherFragment) - TypeLen - PaddingLen - + bca_tag_len(BulkCipherAlgo)). + +bca_tag_len(?AES_CCM_8) -> + 8; +bca_tag_len(_) -> + 16. diff --git a/lib/ssl/src/tls_server_session_ticket.erl b/lib/ssl/src/tls_server_session_ticket.erl index 5f278cb939..b9bafa6e36 100644 --- a/lib/ssl/src/tls_server_session_ticket.erl +++ b/lib/ssl/src/tls_server_session_ticket.erl @@ -31,7 +31,7 @@ -include("ssl_cipher.hrl"). %% API --export([start_link/4, +-export([start_link/5, new/3, use/4 ]). @@ -46,18 +46,19 @@ stateless, stateful, nonce, - lifetime + lifetime, + max_early_data_size }). %%%=================================================================== %%% API %%%=================================================================== --spec start_link(atom(), integer(), integer(), tuple()) -> {ok, Pid :: pid()} | +-spec start_link(atom(), integer(), integer(), integer(), tuple()) -> {ok, Pid :: pid()} | {error, Error :: {already_started, pid()}} | {error, Error :: term()} | ignore. -start_link(Mode, Lifetime, TicketStoreSize, AntiReplay) -> - gen_server:start_link(?MODULE, [Mode, Lifetime, TicketStoreSize, AntiReplay], []). +start_link(Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay) -> + gen_server:start_link(?MODULE, [Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay], []). new(Pid, Prf, MasterSecret) -> gen_server:call(Pid, {new_session_ticket, Prf, MasterSecret}, infinity). @@ -81,10 +82,11 @@ init(Args) -> handle_call({new_session_ticket, Prf, MasterSecret}, _From, #state{nonce = Nonce, lifetime = LifeTime, + max_early_data_size = MaxEarlyDataSize, stateful = #{id_generator := IdGen}} = State0) -> Id = stateful_psk_ticket_id(IdGen), PSK = tls_v1:pre_shared_key(MasterSecret, ticket_nonce(Nonce), Prf), - SessionTicket = new_session_ticket(Id, Nonce, LifeTime), + SessionTicket = new_session_ticket(Id, Nonce, LifeTime, MaxEarlyDataSize), State = stateful_ticket_store(Id, SessionTicket, Prf, PSK, State0), {reply, SessionTicket, State}; handle_call({new_session_ticket, Prf, MasterSecret}, _From, @@ -142,27 +144,30 @@ format_status(_Opt, Status) -> %%% Internal functions %%%=================================================================== -inital_state([stateless, Lifetime, _, undefined]) -> +inital_state([stateless, Lifetime, _, MaxEarlyDataSize, undefined]) -> #state{nonce = 0, stateless = #{seed => {crypto:strong_rand_bytes(16), crypto:strong_rand_bytes(32)}, window => undefined}, - lifetime = Lifetime + lifetime = Lifetime, + max_early_data_size = MaxEarlyDataSize }; -inital_state([stateless, Lifetime, _, {Window, K, M}]) -> +inital_state([stateless, Lifetime, _, MaxEarlyDataSize, {Window, K, M}]) -> erlang:send_after(Window * 1000, self(), rotate_bloom_filters), #state{nonce = 0, stateless = #{bloom_filter => tls_bloom_filter:new(K, M), seed => {crypto:strong_rand_bytes(16), crypto:strong_rand_bytes(32)}, window => Window}, - lifetime = Lifetime + lifetime = Lifetime, + max_early_data_size = MaxEarlyDataSize }; -inital_state([stateful, Lifetime, TicketStoreSize|_]) -> +inital_state([stateful, Lifetime, TicketStoreSize, MaxEarlyDataSize|_]) -> %% statfeful servers replay %% protection is that it saves %% all valid tickets #state{lifetime = Lifetime, + max_early_data_size = MaxEarlyDataSize, nonce = 0, stateful = #{db => stateful_store(), max => TicketStoreSize, @@ -187,17 +192,21 @@ ticket_nonce(I) -> <<?UINT64(I)>>. new_session_ticket_base(#state{nonce = Nonce, - lifetime = Lifetime}) -> - new_session_ticket(undefined, Nonce, Lifetime). + lifetime = Lifetime, + max_early_data_size = MaxEarlyDataSize}) -> + new_session_ticket(undefined, Nonce, Lifetime, MaxEarlyDataSize). -new_session_ticket(Id, Nonce, Lifetime) -> +new_session_ticket(Id, Nonce, Lifetime, MaxEarlyDataSize) -> TicketAgeAdd = ticket_age_add(), + Extensions = #{early_data => + #early_data_indication_nst{ + indication = MaxEarlyDataSize}}, #new_session_ticket{ ticket = Id, ticket_lifetime = Lifetime, ticket_age_add = TicketAgeAdd, ticket_nonce = ticket_nonce(Nonce), - extensions = #{} + extensions = Extensions }. @@ -322,7 +331,7 @@ generate_stateless_ticket(#new_session_ticket{ticket_nonce = Nonce, timestamp = Timestamp }, Shard, IV), Ticket#new_session_ticket{ticket = Encrypted}. - + stateless_use(#offered_psks{ identities = Identities, binders = Binders diff --git a/lib/ssl/src/tls_server_session_ticket_sup.erl b/lib/ssl/src/tls_server_session_ticket_sup.erl index 7ee4bb7b2c..bdde94ecea 100644 --- a/lib/ssl/src/tls_server_session_ticket_sup.erl +++ b/lib/ssl/src/tls_server_session_ticket_sup.erl @@ -27,26 +27,34 @@ -behaviour(supervisor). %% API --export([start_link/0, start_link_dist/0]). --export([start_child/1, start_child_dist/1]). +-export([start_link/0, + start_link_dist/0]). +-export([start_child/1, + start_child_dist/1]). %% Supervisor callback --export([init/1]). +-export([init/1, + sup_name/1]). %%%========================================================================= %%% API %%%========================================================================= start_link() -> - supervisor:start_link({local, tracker_name(normal)}, ?MODULE, []). + supervisor:start_link({local, sup_name(normal)}, ?MODULE, []). start_link_dist() -> - supervisor:start_link({local, tracker_name(dist)}, ?MODULE, []). + supervisor:start_link({local, sup_name(dist)}, ?MODULE, []). start_child(Args) -> - supervisor:start_child(tracker_name(normal), Args). + supervisor:start_child(sup_name(normal), Args). start_child_dist(Args) -> - supervisor:start_child(tracker_name(dist), Args). + supervisor:start_child(sup_name(dist), Args). + +sup_name(normal) -> + ?MODULE; +sup_name(dist) -> + list_to_atom(atom_to_list(?MODULE) ++ "_dist"). %%%========================================================================= %%% Supervisor callback @@ -66,7 +74,3 @@ init(_O) -> ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. -tracker_name(normal) -> - ?MODULE; -tracker_name(dist) -> - list_to_atom(atom_to_list(?MODULE) ++ "dist"). diff --git a/lib/ssl/src/tls_server_sup.erl b/lib/ssl/src/tls_server_sup.erl index 5d3278fe32..b2f011f221 100644 --- a/lib/ssl/src/tls_server_sup.erl +++ b/lib/ssl/src/tls_server_sup.erl @@ -47,10 +47,12 @@ init([]) -> ListenTracker = listen_options_tracker_child_spec(), SessionTracker = tls_server_session_child_spec(), Pre_1_3SessionTracker = ssl_server_session_child_spec(), - + Pre_1_3UpgradeSessionTracker = ssl_upgrade_server_session_child_spec(), + {ok, {{one_for_all, 10, 3600}, [ListenTracker, SessionTracker, - Pre_1_3SessionTracker + Pre_1_3SessionTracker, + Pre_1_3UpgradeSessionTracker ]}}. @@ -86,3 +88,12 @@ ssl_server_session_child_spec() -> Modules = [ssl_server_session_cache_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +ssl_upgrade_server_session_child_spec() -> + Name = ssl_upgrade_server_session_cache_sup, + StartFunc = {ssl_upgrade_server_session_cache_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_upgrade_server_session_cache_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/tls_socket.erl b/lib/ssl/src/tls_socket.erl index 66740832b0..91fdad4e44 100644 --- a/lib/ssl/src/tls_socket.erl +++ b/lib/ssl/src/tls_socket.erl @@ -49,7 +49,7 @@ start_link/3, terminate/2, inherit_tracker/3, - session_id_tracker/1, + session_id_tracker/2, emulated_socket_options/2, get_emulated_opts/1, set_emulated_opts/2, @@ -63,7 +63,7 @@ -record(state, { emulated_opts, - port, + listen_monitor, ssl_opts }). @@ -79,12 +79,14 @@ listen(Transport, Port, #config{transport_info = {Transport, _, _, _, _}, case Transport:listen(Port, Options ++ internal_inet_values()) of {ok, ListenSocket} -> {ok, Tracker} = inherit_tracker(ListenSocket, EmOpts, SslOpts), - LifeTime = get_ticket_lifetime(), - TicketStoreSize = get_ticket_store_size(), + LifeTime = ssl_config:get_ticket_lifetime(), + TicketStoreSize = ssl_config:get_ticket_store_size(), + MaxEarlyDataSize = ssl_config:get_max_early_data_size(), %% TLS-1.3 session handling - {ok, SessionHandler} = session_tickets_tracker(LifeTime, TicketStoreSize, SslOpts), + {ok, SessionHandler} = + session_tickets_tracker(LifeTime, TicketStoreSize, MaxEarlyDataSize, SslOpts), %% PRE TLS-1.3 session handling - {ok, SessionIdHandle} = session_id_tracker(SslOpts), + {ok, SessionIdHandle} = session_id_tracker(ListenSocket, SslOpts), Trackers = [{option_tracker, Tracker}, {session_tickets_tracker, SessionHandler}, {session_id_tracker, SessionIdHandle}], Socket = #sslsocket{pid = {ListenSocket, Config#config{trackers = Trackers}}}, @@ -261,27 +263,40 @@ inherit_tracker(ListenSocket, EmOpts, #{erl_dist := false} = SslOpts) -> inherit_tracker(ListenSocket, EmOpts, #{erl_dist := true} = SslOpts) -> ssl_listen_tracker_sup:start_child_dist([ListenSocket, EmOpts, SslOpts]). -session_tickets_tracker(_, _, #{erl_dist := false, - session_tickets := disabled}) -> +session_tickets_tracker(_, _, _, #{erl_dist := false, + session_tickets := disabled}) -> {ok, disabled}; -session_tickets_tracker(Lifetime, TicketStoreSize, #{erl_dist := false, - session_tickets := Mode, - anti_replay := AntiReplay}) -> - tls_server_session_ticket_sup:start_child([Mode, Lifetime, TicketStoreSize, AntiReplay]); -session_tickets_tracker(Lifetime, TicketStoreSize, #{erl_dist := true, - session_tickets := Mode}) -> - tls_server_session_ticket_sup:start_child_dist([Mode, Lifetime, TicketStoreSize]). - -session_id_tracker(#{versions := [{3,4}]}) -> +session_tickets_tracker(Lifetime, TicketStoreSize, MaxEarlyDataSize, + #{erl_dist := false, + session_tickets := Mode, + anti_replay := AntiReplay}) -> + tls_server_session_ticket_sup:start_child([Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay]); +session_tickets_tracker(Lifetime, TicketStoreSize, MaxEarlyDataSize, + #{erl_dist := true, + session_tickets := Mode, + anti_replay := AntiReplay}) -> + SupName = tls_server_session_ticket_sup:sup_name(dist), + Children = supervisor:count_children(SupName), + Workers = proplists:get_value(workers, Children), + case Workers of + 0 -> + tls_server_session_ticket_sup:start_child([Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay]); + 1 -> + [{_,Child,_, _}] = supervisor:which_children(SupName), + {ok, Child} + end. +session_id_tracker(_, #{versions := [{3,4}]}) -> {ok, not_relevant}; %% Regardless of the option reuse_sessions we need the session_id_tracker %% to generate session ids, but no sessions will be stored unless %% reuse_sessions = true. -session_id_tracker(#{erl_dist := false}) -> - ssl_server_session_cache_sup:start_child(ssl_server_session_cache_sup:session_opts()); -session_id_tracker(#{erl_dist := true}) -> - ssl_server_session_cache_sup:start_child_dist(ssl_server_session_cache_sup:session_opts()). - +session_id_tracker(ssl_unknown_listener, #{erl_dist := false}) -> + ssl_upgrade_server_session_cache_sup:start_child(normal); +session_id_tracker(ListenSocket, #{erl_dist := false}) -> + ssl_server_session_cache_sup:start_child(ListenSocket); +session_id_tracker(_, #{erl_dist := true}) -> + ssl_upgrade_server_session_cache_sup:start_child(dist). + get_emulated_opts(TrackerPid) -> call(TrackerPid, get_emulated_opts). set_emulated_opts(TrackerPid, InetValues) -> @@ -303,10 +318,12 @@ start_link(Port, SockOpts, SslOpts) -> %% %% Description: Initiates the server %%-------------------------------------------------------------------- -init([Port, Opts, SslOpts]) -> +init([Listen, Opts, SslOpts]) -> process_flag(trap_exit, true), - true = link(Port), - {ok, #state{emulated_opts = do_set_emulated_opts(Opts, []), port = Port, ssl_opts = SslOpts}}. + Monitor = monitor_listen(Listen), + {ok, #state{emulated_opts = do_set_emulated_opts(Opts, []), + listen_monitor = Monitor, + ssl_opts = SslOpts}}. %%-------------------------------------------------------------------- -spec handle_call(msg(), from(), #state{}) -> {reply, reply(), #state{}}. @@ -351,7 +368,7 @@ handle_cast(_, State)-> %% %% Description: Handling all non call/cast messages %%------------------------------------------------------------------- -handle_info({'EXIT', Port, _}, #state{port = Port} = State) -> +handle_info({'DOWN', Monitor, _, _, _}, #state{listen_monitor = Monitor} = State) -> {stop, normal, State}. @@ -380,6 +397,9 @@ code_change(_OldVsn, State, _Extra) -> call(Pid, Msg) -> gen_server:call(Pid, Msg, infinity). +monitor_listen(Listen) when is_port(Listen) -> + erlang:monitor(port, Listen). + split_options(Opts) -> split_options(Opts, emulated_options(), [], []). split_options([], _, SocketOpts, EmuOpts) -> @@ -486,19 +506,3 @@ validate_inet_option(active, Value) validate_inet_option(_, _) -> ok. -get_ticket_lifetime() -> - case application:get_env(ssl, server_session_ticket_lifetime) of - {ok, Seconds} when is_integer(Seconds) andalso - Seconds =< 604800 -> %% MUST be less than 7 days - Seconds; - _ -> - 7200 %% Default 2 hours - end. - -get_ticket_store_size() -> - case application:get_env(ssl, server_session_ticket_store_size) of - {ok, Size} when is_integer(Size) -> - Size; - _ -> - 1000 - end. diff --git a/lib/ssl/src/tls_sup.erl b/lib/ssl/src/tls_sup.erl index 25c1db0272..a425ae31e2 100644 --- a/lib/ssl/src/tls_sup.erl +++ b/lib/ssl/src/tls_sup.erl @@ -45,10 +45,10 @@ start_link() -> init([]) -> - TLSConnetionManager = tls_connection_manager_child_spec(), + TLSConnetionSup = tls_connection_child_spec(), ServerInstanceSup = server_instance_child_spec(), - {ok, {{one_for_one, 10, 3600}, [TLSConnetionManager, + {ok, {{one_for_one, 10, 3600}, [TLSConnetionSup, ServerInstanceSup ]}}. @@ -57,7 +57,7 @@ init([]) -> %%% Internal functions %%-------------------------------------------------------------------- -tls_connection_manager_child_spec() -> +tls_connection_child_spec() -> Name = tls_connection, StartFunc = {tls_connection_sup, start_link, []}, Restart = permanent, diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 8e6807d0ab..59c425ecbe 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -56,6 +56,7 @@ hkdf_expand_label/5, hkdf_extract/3, hkdf_expand/4, + key_length/1, key_schedule/3, key_schedule/4, create_info/3, @@ -455,13 +456,20 @@ update_traffic_secret(Algo, Secret) -> %% %% [sender]_write_key = HKDF-Expand-Label(Secret, "key", "", key_length) %% [sender]_write_iv = HKDF-Expand-Label(Secret, "iv", "", iv_length) --spec calculate_traffic_keys(atom(), atom(), binary()) -> {binary(), binary()}. -calculate_traffic_keys(HKDFAlgo, Cipher, Secret) -> - Key = hkdf_expand_label(Secret, <<"key">>, <<>>, ssl_cipher:key_material(Cipher), HKDFAlgo), +-spec calculate_traffic_keys(atom(), integer(), binary()) -> {binary(), binary()}. +calculate_traffic_keys(HKDFAlgo, KeyLength, Secret) -> + Key = hkdf_expand_label(Secret, <<"key">>, <<>>, KeyLength, HKDFAlgo), %% TODO: remove hard coded IV size IV = hkdf_expand_label(Secret, <<"iv">>, <<>>, 12, HKDFAlgo), {Key, IV}. +-spec key_length(CipherSuite) -> KeyLength when + CipherSuite :: binary(), + KeyLength :: 0 | 8 | 16 | 24 | 32. +key_length(CipherSuite) -> + #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + ssl_cipher:key_material(Cipher). + %% TLS v1.3 --------------------------------------------------- %% TLS 1.0 -1.2 --------------------------------------------------- @@ -486,21 +494,7 @@ mac_hash(Method, Mac_write_secret, Seq_num, Type, {Major, Minor}, -spec suites(1|2|3|4) -> [ssl_cipher_format:cipher_suite()]. suites(Minor) when Minor == 1; Minor == 2 -> - [ - ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA, - ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, - ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, - - ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA, - ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, - ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA - ]; + exclusive_suites(2); suites(3) -> exclusive_suites(3) ++ suites(2); @@ -518,36 +512,42 @@ exclusive_suites(3) -> [?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + ?TLS_ECDHE_ECDSA_WITH_AES_256_CCM, + ?TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8, + ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + + ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + + ?TLS_ECDHE_ECDSA_WITH_AES_128_CCM, + ?TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8, + ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384, ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384, ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384, - ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, - - ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, - - ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - - ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, + ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, - ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, - ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, - ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256, + ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, + ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + + ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, @@ -564,19 +564,19 @@ exclusive_suites(3) -> ]; exclusive_suites(Minor) when Minor == 1; Minor == 2 -> [ - ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA, - ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, - ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, - - ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA, - ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, - ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA + ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA, + ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, + ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, + + ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, + ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA ]. signature_algs({3, 4}, HashSigns) -> diff --git a/lib/ssl/test/openssl_cipher_suite_SUITE.erl b/lib/ssl/test/openssl_cipher_suite_SUITE.erl index 64c3899889..fb1f28aa4a 100644 --- a/lib/ssl/test/openssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/openssl_cipher_suite_SUITE.erl @@ -82,19 +82,26 @@ aes_128_gcm_sha256/1, chacha20_poly1305_sha256/1, aes_128_ccm_sha256/1, - aes_128_ccm_8_sha256/1 + aes_128_ccm_8_sha256/1, + ecdhe_ecdsa_with_aes_128_ccm/1, + ecdhe_ecdsa_with_aes_256_ccm/1, + ecdhe_ecdsa_with_aes_128_ccm_8/1, + ecdhe_ecdsa_with_aes_256_ccm_8/1 ]). --define(DEFAULT_TIMEOUT, {seconds, 6}). +-define(DEFAULT_TIMEOUT, {seconds, 10}). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- all() -> - [ - {group, openssl_server}, - {group, openssl_client} - ]. + case ssl_test_lib:working_openssl_client() of + true -> + [{group, openssl_server}, + {group, openssl_client}]; + false -> + [{group, openssl_server}] + end. all_protocol_groups() -> [ @@ -140,7 +147,11 @@ groups() -> ecdhe_ecdsa_aes_128_gcm, ecdhe_ecdsa_aes_256_cbc, ecdhe_ecdsa_aes_256_gcm, - ecdhe_ecdsa_chacha20_poly1305 + ecdhe_ecdsa_chacha20_poly1305, + ecdhe_ecdsa_with_aes_128_ccm, + ecdhe_ecdsa_with_aes_256_ccm, + ecdhe_ecdsa_with_aes_128_ccm_8, + ecdhe_ecdsa_with_aes_256_ccm_8 ]}, {rsa, [], [rsa_des_cbc, rsa_3des_ede_cbc, @@ -298,7 +309,8 @@ do_init_per_group(ecdhe_ecdsa = GroupName, Config) -> end; do_init_per_group(dhe_dss = GroupName, Config) -> PKAlg = proplists:get_value(public_keys, crypto:supports()), - case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) + andalso (ssl_test_lib:openssl_dsa_suites() =/= []) of true -> init_certs(GroupName, Config); false -> @@ -306,7 +318,8 @@ do_init_per_group(dhe_dss = GroupName, Config) -> end; do_init_per_group(srp_dss = GroupName, Config) -> PKAlg = proplists:get_value(public_keys, crypto:supports()), - case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) + andalso (ssl_test_lib:openssl_dsa_suites() =/= []) of true -> init_certs(GroupName, Config); false -> @@ -339,11 +352,11 @@ do_init_per_group(dhe_rsa = GroupName, Config) -> end; do_init_per_group(rsa = GroupName, Config) -> PKAlg = proplists:get_value(public_keys, crypto:supports()), - case lists:member(rsa, PKAlg) of + case lists:member(rsa, PKAlg) andalso ssl_test_lib:openssl_support_rsa_kex() of true -> init_certs(GroupName, Config); false -> - {skip, "Missing SRP crypto support"} + {skip, "Missing RSA key exchange support"} end; do_init_per_group(dh_anon = GroupName, Config) -> PKAlg = proplists:get_value(public_keys, crypto:supports()), @@ -375,7 +388,7 @@ init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; SupCiphers = proplists:get_value(ciphers, crypto:supports()), case lists:member(des_ede3, SupCiphers) of true -> - ct:timetrap({seconds, 5}), + ct:timetrap({seconds, ?DEFAULT_TIMEOUT}), Config; _ -> {skip, "Missing 3DES crypto support"} @@ -478,6 +491,28 @@ init_per_testcase(aes_128_ccm_8_sha256, Config) -> {skip, "Missing AES_128_CCM_8 crypto support"} end; +init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_ccm; + TestCase == ecdhe_ecdsa_with_aes_128_ccm_8-> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap(?DEFAULT_TIMEOUT), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; + +init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_256_ccm; + TestCase == ecdhe_ecdsa_with_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap(?DEFAULT_TIMEOUT), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; + init_per_testcase(TestCase, Config) -> Cipher = ssl_test_lib:test_cipher(TestCase, Config), SupCiphers = proplists:get_value(ciphers, crypto:supports()), @@ -728,6 +763,18 @@ ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). + +ecdhe_ecdsa_with_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_ccm', Config). + +ecdhe_ecdsa_with_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_ccm', Config). + +ecdhe_ecdsa_with_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_ccm_8', Config). + +ecdhe_ecdsa_with_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_ccm_8', Config). %%-------------------------------------------------------------------- %% DHE_DSS -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -897,7 +944,7 @@ run_ciphers_test(Kex, Cipher, Config) -> {skip, {not_sup, Kex, Cipher, Version}} end. -cipher_suite_test(CipherSuite, _Version, Config) -> +cipher_suite_test(CipherSuite, Version, Config) -> #{server_config := SOpts, client_config := COpts} = proplists:get_value(tls_config, Config), ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), @@ -905,11 +952,17 @@ cipher_suite_test(CipherSuite, _Version, Config) -> ct:log("Testing CipherSuite ~p~n", [CipherSuite]), ct:log("Server Opts ~p~n", [ServerOpts]), ct:log("Client Opts ~p~n", [ClientOpts]), - ssl_test_lib:basic_test([{ciphers, [CipherSuite]} | COpts], SOpts, Config). - + case proplists:get_value(server_type, Config) of + erlang -> + ssl_test_lib:basic_test([{ciphers, ssl:cipher_suites(all, Version)} | COpts], + [{ciphers, [CipherSuite]} | SOpts], Config); + _ -> + ssl_test_lib:basic_test([{versions, [Version]}, {ciphers, [CipherSuite]} | COpts], + [{ciphers, ssl_test_lib:openssl_ciphers()} | SOpts], Config) + end. test_ciphers(Kex, Cipher, Version) -> - Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, Version) ++ ssl:cipher_suites(anonymous, Version), + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, Version) ++ ssl:cipher_suites(anonymous, Version), [{key_exchange, fun(Kex0) when (Kex0 == Kex) andalso (Version =/= 'tlsv1.3') -> true; (Kex0) when (Kex0 == any) andalso (Version == 'tlsv1.3') -> true; @@ -928,3 +981,5 @@ test_ciphers(Kex, Cipher, Version) -> end, Ciphers). +openssl_suitestr_to_map(OpenSSLSuiteStrs) -> + [ssl_cipher_format:suite_openssl_str_to_map(SuiteStr) || SuiteStr <- OpenSSLSuiteStrs]. diff --git a/lib/ssl/test/openssl_client_cert_SUITE.erl b/lib/ssl/test/openssl_client_cert_SUITE.erl index 7adfdf32a5..0248956056 100644 --- a/lib/ssl/test/openssl_client_cert_SUITE.erl +++ b/lib/ssl/test/openssl_client_cert_SUITE.erl @@ -156,10 +156,15 @@ init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - Config + case ssl_test_lib:working_openssl_client() of + true -> + ssl_test_lib:clean_start(), + Config; + false -> + {skip, "Broken OpenSSL s_client"} + end catch _:_ -> - {skip, "Crypto did not start"} + {skip, "Crypto did not start"} end. end_per_suite(_Config) -> @@ -167,9 +172,9 @@ end_per_suite(_Config) -> application:unload(ssl), application:stop(crypto). -init_per_group(openssl_client, Config0) -> - Config = proplists:delete(server_type, proplists:delete(client_type, Config0)), +init_per_group(openssl_client, Config) -> [{client_type, openssl}, {server_type, erlang} | Config]; + init_per_group(Group, Config0) when Group == rsa; Group == rsa_1_3 -> Config = ssl_test_lib:make_rsa_cert(Config0), @@ -292,7 +297,8 @@ end_per_group(GroupName, Config) -> init_per_testcase(TestCase, Config) when TestCase == client_auth_empty_cert_accepted; TestCase == client_auth_empty_cert_rejected -> - Version = proplists:get_value(version,Config), + Version = ssl_test_lib:protocol_version(Config), + case Version of sslv3 -> %% Openssl client sends "No Certificate Reserved" warning ALERT diff --git a/lib/ssl/test/openssl_server_cert_SUITE.erl b/lib/ssl/test/openssl_server_cert_SUITE.erl index 4402765ea2..e71bfc8e5c 100644 --- a/lib/ssl/test/openssl_server_cert_SUITE.erl +++ b/lib/ssl/test/openssl_server_cert_SUITE.erl @@ -275,7 +275,7 @@ init_per_group(ecdsa_1_3 = Group, Config0) -> COpts = proplists:get_value(client_ecdsa_opts, Config), SOpts = proplists:get_value(server_ecdsa_opts, Config), %% Make sure ecdh* suite is choosen by ssl_test_lib:start_server - Version = proplists:get_value(version,Config), + Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_cert_tests:test_ciphers(undefined, Version), case Ciphers of [_|_] -> @@ -301,7 +301,7 @@ init_per_group(Group, Config0) when Group == dsa -> COpts = proplists:get_value(client_dsa_opts, Config), SOpts = proplists:get_value(server_dsa_opts, Config), %% Make sure dhe_dss* suite is choosen by ssl_test_lib:start_server - Version = proplists:get_value(version,Config), + Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_cert_tests:test_ciphers(fun(dh_dss) -> true; (dhe_dss) -> diff --git a/lib/ssl/test/openssl_session_SUITE.erl b/lib/ssl/test/openssl_session_SUITE.erl index 1ea6398b2d..08369733dc 100644 --- a/lib/ssl/test/openssl_session_SUITE.erl +++ b/lib/ssl/test/openssl_session_SUITE.erl @@ -124,19 +124,24 @@ init_per_testcase(reuse_session_erlang_client, Config) -> ssl:start(), Config; init_per_testcase(reuse_session_erlang_server, Config) -> - Version = ssl_test_lib:protocol_version(Config), - case ssl_test_lib:is_dtls_version(Version) of + case ssl_test_lib:working_openssl_client() of true -> - case ssl_test_lib:openssl_sane_dtls_session_reuse() of + Version = ssl_test_lib:protocol_version(Config), + case ssl_test_lib:is_dtls_version(Version) of true -> - ct:timetrap(?TIMEOUT), - Config; + case ssl_test_lib:openssl_sane_dtls_session_reuse() of + true -> + ct:timetrap(?TIMEOUT), + Config; + false -> + {skip, "Broken OpenSSL DTLS session reuse"} + end; false -> - {skip, "Broken OpenSSL DTLS session reuse"} + ct:timetrap(?TIMEOUT), + Config end; - false -> - ct:timetrap(?TIMEOUT), - Config + false -> + {skip, "Broken OpenSSL s_client"} end; init_per_testcase(_TestCase, Config) -> ct:timetrap(?TIMEOUT), @@ -171,7 +176,8 @@ reuse_session_erlang_server(Config) when is_list(Config) -> {from, self()}, {mfa, {ssl_test_lib, active_recv, [length(Data)]}}, {reconnect_times, 5}, - {options, [{ciphers, Ciphers} | ServerOpts]}]), + {options, [{ciphers, Ciphers}, + {versions, [Version]}| ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), @@ -206,7 +212,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> {from, self()}, {options, [{reuse_sessions, save}, {verify, verify_peer}, - {ciphers, Ciphers} | ClientOpts]}]), + {ciphers, Ciphers}, + {versions, [Version]} | ClientOpts]}]), SID = receive {Client0, Id0} -> @@ -219,7 +226,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [ {ciphers, Ciphers}, + {from, self()}, {options, [ {ciphers, Ciphers}, + {versions, [Version]}, {reuse_session, SID} | ClientOpts]}]), receive {Client1, SID} -> @@ -237,7 +245,8 @@ reuse_session_erlang_client(Config) when is_list(Config) -> ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {mfa, {ssl_test_lib, session_id, []}}, - {from, self()}, {options, [{ciphers, Ciphers} | ClientOpts]}]), + {from, self()}, {options, [{ciphers, Ciphers}, + {versions, [Version]} | ClientOpts]}]), receive {Client2, ID} -> case ID of diff --git a/lib/ssl/test/openssl_session_ticket_SUITE.erl b/lib/ssl/test/openssl_session_ticket_SUITE.erl index 9986a492b1..effcedc7ee 100644 --- a/lib/ssl/test/openssl_session_ticket_SUITE.erl +++ b/lib/ssl/test/openssl_session_ticket_SUITE.erl @@ -40,7 +40,21 @@ openssl_client_basic/0, openssl_client_basic/1, openssl_client_hrr/0, - openssl_client_hrr/1]). + openssl_client_hrr/1, + openssl_server_early_data_basic/0, + openssl_server_early_data_basic/1, + openssl_server_early_data_big/0, + openssl_server_early_data_big/1, + openssl_server_early_data_manual/0, + openssl_server_early_data_manual/1, + openssl_server_early_data_manual_big/0, + openssl_server_early_data_manual_big/1, + openssl_server_early_data_manual_2_tickets/0, + openssl_server_early_data_manual_2_tickets/1, + openssl_server_early_data_manual_2_chacha_tickets/0, + openssl_server_early_data_manual_2_chacha_tickets/1, + openssl_client_early_data_basic/0, + openssl_client_early_data_basic/1]). -include("tls_handshake.hrl"). @@ -62,14 +76,21 @@ groups() -> {group, openssl_server}]}, {openssl_server, [], [openssl_server_basic, openssl_server_hrr, - openssl_server_hrr_multiple_tickets + openssl_server_hrr_multiple_tickets, + openssl_server_early_data_basic, + openssl_server_early_data_big, + openssl_server_early_data_manual, + openssl_server_early_data_manual_big, + openssl_server_early_data_manual_2_tickets, + openssl_server_early_data_manual_2_chacha_tickets ]}, {stateful, [], session_tests()}, {stateless, [], session_tests()}]. session_tests() -> [openssl_client_basic, - openssl_client_hrr]. + openssl_client_hrr, + openssl_client_early_data_basic]. init_per_suite(Config0) -> catch crypto:stop(), @@ -316,7 +337,7 @@ openssl_server_hrr_multiple_tickets(Config) when is_list(Config) -> {versions, ['tlsv1.2','tlsv1.3']}, {supported_groups,[secp256r1, x25519]}|ClientOpts0], - + Server = ssl_test_lib:start_server(openssl, [{groups, "X448:X25519"}], [{server_opts, ServerOpts} | Config]), @@ -350,3 +371,348 @@ openssl_server_hrr_multiple_tickets(Config) when is_list(Config) -> ssl_test_lib:close(Client1), ssl_test_lib:close(Server). + +openssl_server_early_data_basic() -> + [{doc,"Test early data (erlang client - openssl server)"}]. +openssl_server_early_data_basic(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + Server = ssl_test_lib:start_server(openssl, [{early_data, 16384}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply]}}, + {from, self()}, {options, ClientOpts1}]), + %% Wait for session ticket + ct:sleep(100), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true, no_reply, no_tickets, + {verify_early_data, accepted}]}}, + {from, self()}, + {options, ClientOpts2}]), + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_server_early_data_big() -> + [{doc,"Send more early data than the max_early_data_size (erlang client - openssl server)"}]. +openssl_server_early_data_big(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + Server = ssl_test_lib:start_server(openssl, [{early_data, 5}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply]}}, + {from, self()}, {options, ClientOpts1}]), + %% Wait for session ticket + ct:sleep(100), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use ticket + %% The tickets received cannot be used for sending more early data than the + %% max_early_data_size. They are filtered by the automatic ticket handling + %% mechanism and there will be no session resumption. + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply, no_tickets]}}, + {from, self()}, + {options, ClientOpts2}]), + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_server_early_data_manual() -> + [{doc,"Test sending early data - manual ticket handling (erlang client - openssl server)"}]. +openssl_server_early_data_manual(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + Server = ssl_test_lib:start_server(openssl, [{early_data, 16384}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + + Tickets0 = ssl_test_lib:check_tickets(Client0), + + ct:pal("Received tickets: ~p~n", [Tickets0]), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use tickets + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true, no_reply, no_tickets, + {verify_early_data, accepted}]}}, + {from, self()}, + {options, [{use_ticket, Tickets0}|ClientOpts2]}]), + + process_flag(trap_exit, false), + + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_server_early_data_manual_big() -> + [{doc,"Test sending more early data than the max_early_data_size - manual ticket handling " + "(erlang client - openssl server)"}]. +openssl_server_early_data_manual_big(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + Server = ssl_test_lib:start_server(openssl, [{early_data, 5}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + + Tickets0 = ssl_test_lib:check_tickets(Client0), + + ct:pal("Received tickets: ~p~n", [Tickets0]), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use tickets + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true, no_reply, no_tickets]}}, + {from, self()}, + {options, [{use_ticket, Tickets0}|ClientOpts2]}]), + ssl_test_lib:check_client_alert(Client1, illegal_parameter), + process_flag(trap_exit, false), + + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_server_early_data_manual_2_tickets() -> + [{doc,"Test sending early data - manual ticket handling, 2 tickets (erlang client - openssl server)"}]. +openssl_server_early_data_manual_2_tickets(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + Server = ssl_test_lib:start_server(openssl, [{early_data, 16384}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply, {tickets, 2}]}}, + {from, self()}, {options, ClientOpts1}]), + + Tickets0 = ssl_test_lib:check_tickets(Client0), + + ct:pal("Received tickets: ~p~n", [Tickets0]), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use tickets + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true, no_reply, no_tickets, + {verify_early_data, accepted}]}}, + {from, self()}, + {options, [{use_ticket, Tickets0}|ClientOpts2]}]), + process_flag(trap_exit, false), + + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_server_early_data_manual_2_chacha_tickets() -> + [{doc,"Test sending early data - manual ticket handling, 2 tickets - chacha (erlang client - openssl server)"}]. +openssl_server_early_data_manual_2_chacha_tickets(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {ciphers, ["TLS_CHACHA20_POLY1305_SHA256"]}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + ClientOpts2 = [{early_data, <<"SampleData">>}|ClientOpts1], + + %% openssl s_server seems to select a cipher_suite that satisfies the requirements + %% for early_data. + Server = ssl_test_lib:start_server(openssl, [{early_data, 16384}], + [{server_opts, ServerOpts} | Config]), + + Port = ssl_test_lib:inet_port(Server), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply, {tickets, 2}]}}, + {from, self()}, {options, ClientOpts1}]), + + %% Receive 2 tickets that used Chacha20-Poly1305 and sha256 + Tickets0 = ssl_test_lib:check_tickets(Client0), + + ct:pal("Received tickets: ~p~n", [Tickets0]), + + %% Close previous connection as s_server can only handle one at a time + ssl_test_lib:close(Client0), + + %% Use tickets + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true, no_reply, no_tickets, + {verify_early_data, accepted}]}}, + {from, self()}, + {options, [{use_ticket, Tickets0}|ClientOpts2]}]), + process_flag(trap_exit, false), + + ssl_test_lib:close(Client1), + ssl_test_lib:close(Server). + +openssl_client_early_data_basic() -> + [{doc,"Test early data (openssl client - erlang server)"}]. +openssl_client_early_data_basic(Config) when is_list(Config) -> + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + ClientOpts = proplists:get_value(client_rsa_opts, Config), + + {_, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + TicketFile0 = filename:join([proplists:get_value(priv_dir, Config), "session_ticket0"]), + TicketFile1 = filename:join([proplists:get_value(priv_dir, Config), "session_ticket1"]), + RequestFile = filename:join([proplists:get_value(priv_dir, Config), "request"]), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + %% Create request file to be used with early data + EarlyData = <<"HEAD / HTTP/1.1\nHost: \nConnection: close\n\n">>, + create_request(RequestFile, EarlyData), + + %% Configure session tickets + ServerOpts = [{session_tickets, ServerTicketMode}, + {early_data, enabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + + Port0 = ssl_test_lib:inet_port(Server0), + + Client0 = ssl_test_lib:start_client(openssl, [{port, Port0}, + {options, ClientOpts}, + {session_args, ["-sess_out", TicketFile0]}], Config), + ssl_test_lib:check_result(Server0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_server_early_data, + [wait_reply, EarlyData]}}}, + + %% %% Wait for session ticket + ct:sleep(100), + ssl_test_lib:close(Client0), + + Client1 = ssl_test_lib:start_client(openssl, [{port, Port0}, + {options, ClientOpts}, + {session_args, ["-sess_in", TicketFile0, + "-sess_out", TicketFile1, + "-early_data", RequestFile]}], + Config), + + ssl_test_lib:check_result(Server0, ok), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + +%%-------------------------------------------------------------------- +%% Internal functions ------------------------------------------------ +%%-------------------------------------------------------------------- + +create_request(File, EarlyData) -> + {ok, S} = file:open(File, [write]), + io:format(S, "~s", [binary_to_list(EarlyData)]), + file:close(S). + diff --git a/lib/ssl/test/property_test/ssl_eqc_cipher_format.erl b/lib/ssl/test/property_test/ssl_eqc_cipher_format.erl index b661ec8806..cf6ed755f7 100644 --- a/lib/ssl/test/property_test/ssl_eqc_cipher_format.erl +++ b/lib/ssl/test/property_test/ssl_eqc_cipher_format.erl @@ -73,6 +73,21 @@ prop_tls_cipher_suite_rfc_name() -> prop_tls_cipher_suite_openssl_name() -> ?FORALL({CipherSuite, _TLSVersion}, ?LET(Version, tls_version(), {cipher_suite(Version), Version}), case ssl:str_to_suite(ssl:suite_to_openssl_str(CipherSuite)) of + CipherSuite -> + case ssl:suite_to_openssl_str(CipherSuite) of + "TLS_" ++ _ -> + true; + OpensslName -> + lists:member(OpensslName, openssl_legacy_names()) + end; + _ -> + false + end + ). + +prop_tls_anon_cipher_suite_rfc_name() -> + ?FORALL({CipherSuite, _TLSVersion}, ?LET(Version, pre_tls_1_3_version(), {anon_cipher_suite(Version), Version}), + case ssl:str_to_suite(ssl:suite_to_str(CipherSuite)) of CipherSuite -> true; _ -> @@ -80,6 +95,15 @@ prop_tls_cipher_suite_openssl_name() -> end ). +prop_tls_anon_cipher_suite_openssl_name() -> + ?FORALL({CipherSuite, _TLSVersion}, ?LET(Version, pre_tls_1_3_version(), {anon_cipher_suite(Version), Version}), + case ssl:str_to_suite(ssl:suite_to_openssl_str(CipherSuite)) of + CipherSuite -> + lists:member(ssl:suite_to_openssl_str(CipherSuite), openssl_legacy_names()); + _ -> + false + end + ). %%-------------------------------------------------------------------- %% Generators ----------------------------------------------- @@ -87,9 +111,164 @@ prop_tls_cipher_suite_openssl_name() -> tls_version() -> oneof([?'TLS_v1.3', ?'TLS_v1.2', ?'TLS_v1.1', ?'TLS_v1']). +pre_tls_1_3_version() -> + oneof([?'TLS_v1.2', ?'TLS_v1.1', ?'TLS_v1']). + cipher_suite(Version) -> oneof(cipher_suites(Version)). cipher_suites(Version) -> - ssl:cipher_suites(all, Version). + ssl:cipher_suites(default, Version). + +anon_cipher_suite(Version) -> + oneof(ssl:cipher_suites(anonymous, Version)). + +openssl_legacy_names() -> + %% Only include names that we support + [ + %% Legacy with RSA keyexchange + "AES128-SHA", + "AES256-SHA", + "AES128-SHA256", + "AES256-SHA256", + "AES256-GCM-SHA256", + "AES256-GCM-SHA384", + "DES-CBC-SHA", + "DES-CBC3-SHA", + "RC4-MD5", + "RC4-SHA", + + %% DH based + "DH-RSA-AES128-SHA", + "DH-RSA-AES256-SHA", + "DH-RSA-AES128-SHA256", + "DH-RSA-AES256-SHA256", + "DH-DSS-AES128-SHA", + "DH-DSS-AES256-SHA", + "DH-DSS-AES128-SHA256", + "DH-DSS-AES256-SHA256", + "EDH-RSA-DES-CBC-SHA", + "EDH-RSA-DES-CBC3-SHA", + "DHE-RSA-AES128-SHA", + "DHE-RSA-AES256-SHA", + "DHE-RSA-AES128-SHA256", + "DHE-RSA-AES256-SHA256", + "DHE-RSA-AES256-SHA384", + "DHE-RSA-AES128-GCM-SHA256", + "DHE-RSA-AES256-GCM-SHA384", + "DHE-RSA-AES128-CCM-SHA256", + "DHE-RSA-AES256-CCM-SHA384", + "DHE-RSA-AES128-CCM8-SHA256", + "DHE-RSA-AES256-CCM8-SHA384", + "DHE-RSA-CHACHA20-POLY1305", + "EDH-DSS-DES-CBC-SHA", + "EDH-DSS-DES-CBC3-SHA", + "DHE-DSS-AES128-SHA", + "DHE-DSS-AES256-SHA", + "DHE-DSS-AES128-SHA256", + "DHE-DSS-AES256-SHA256", + "DHE-DSS-AES256-SHA384", + "DHE-DSS-AES128-GCM-SHA256", + "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-RC4-SHA", + "ADH-AES128-SHA256", + "ADH-AES256-SHA256", + "ADH-AES128-CBC-SHA256", + "ADH-AES128-GCM-SHA256", + "ADH-AES256-GCM-SHA384", + "ADH-RC4-MD5", + "ADH-DES-CBC-SHA", + "ADH-DES-CBC3-SHA", + "ADH-AES256-SHA", + "ADH-AES256-SHA256", + + %% ECDH based + "ECDH-ECDSA-AES128-SHA", + "ECDH-ECDSA-AES256-SHA", + "ECDH-ECDSA-AES128-SHA256", + "ECDH-ECDSA-AES256-SHA384", + "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-ECDSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-CCM", + "ECDHE-ECDSA-AES128-CCM8", + "ECDHE-ECDSA-AES256-CCM", + "ECDHE-ECDSA-AES256-CCM8", + "ECDH-ECDSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDH-RSA-AES128-SHA", + "ECDH-RSA-AES256-SHA", + "ECDH-RSA-AES128-SHA256", + "ECDH-RSA-AES256-SHA384", + "ECDH-RSA-AES128-GCM-SHA256", + "ECDH-RSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES128-SHA", + "ECDHE-RSA-AES256-SHA", + "ECDHE-RSA-AES128-SHA256", + "ECDHE-RSA-AES256-SHA384", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA256", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-RSA-CHACHA20-POLY1305", + "ECDHE-PSK-RC4-SHA", + "ECDHE-PSK-3DES-EDE-CBC-SHA", + "ECDHE-PSK-AES128-CBC-SHA", + "ECDHE-PSK-AES128-CBC-SHA256", + "ECDHE-PSK-AES256-CBC-SHA384", + "ECDHE-PSK-AES128-GCM-SHA256", + "ECDHE-PSK-AES256-GCM-SHA384", + "ECDHE-PSK-AES128-CCM-SHA256", + "ECDHE-PSK-AES128-CCM8-SHA256", + "ECDHE-PSK-CHACHA20-POLY1305", + "AECDH-DES-CBC3-SHA", + "AECDH-AES128-SHA", + "AECDH-AES256-SHA", + %% PSK based + "DHE-PSK-NULL-SHA", + "DHE-PSK-RC4-SHA", + "DHE-PSK-3DES-EDE-CBC-SHA", + "DHE-PSK-AES128-CBC-SHA", + "DHE-PSK-AES256-CBC-SHA", + "DHE-PSK-AES128-CBC-SHA256", + "DHE-PSK-AES256-CBC-SHA384", + "DHE-PSK-AES128-GCM-SHA256", + "DHE-PSK-AES256-GCM-SHA384", + "DHE-PSK-AES128-CCM", + "DHE-PSK-AES128-CCM8", + "DHE-PSK-AES256-CCM", + "DHE-PSK-AES256-CCM8", + "DHE-PSK-AES128-CCM-SHA256", + "DHE-PSK-AES128-CCM8-SHA256", + "DHE-PSK-CHACHA20-POLY1305", + "PSK-NULL-SHA", + "PSK-RC4-SHA", + "PSK-3DES-EDE-CBC-SHA", + "PSK-AES128-CBC-SHA", + "PSK-AES256-CBC-SHA", + "PSK-AES128-CBC-SHA256", + "PSK-AES256-CBC-SHA256", + "PSK-AES128-CCM", + "PSK-AES128-CCM8", + "PSK-AES256-CCM", + "PSK-AES256-CCM8", + "PSK-AES128-GCM-SHA256", + "PSK-AES256-CBC-SHA384", + "PSK-AES256-GCM-SHA384", + "PSK-CHACHA20-POLY1305", + "RSA-PSK-NULL-SHA", + "RSA-PSK-CHACHA20-POLY1305", + + %% SRP based + "SRP-3DES-EDE-CBC-SHA", + "SRP-RSA-3DES-EDE-CBC-SHA", + "SRP-DSS-3DES-EDE-CBC-SHA", + "SRP-AES-128-CBC-SHA", + "SRP-AES-256-CBC-SHA" + ]. diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index f834ac4105..c628f3b6d9 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -125,7 +125,8 @@ end_per_suite(_Config) -> %%-------------------------------------------------------------------- init_per_group(GroupName, Config) -> case ssl_test_lib:is_protocol_version(GroupName) of - true -> + true -> + ct:log("Ciphers: ~p~n ", [ssl:cipher_suites(default, GroupName)]), ssl_test_lib:init_per_group(GroupName, [{client_type, erlang}, {server_type, erlang}, @@ -141,7 +142,6 @@ end_per_group(GroupName, Config) -> init_per_testcase(TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), - ct:log("Ciphers: ~p~n ", [ ssl:cipher_suites()]), end_per_testcase(TestCase, Config), ssl:start(), ct:timetrap({seconds, 15}), diff --git a/lib/ssl/test/ssl_api_SUITE.erl b/lib/ssl/test/ssl_api_SUITE.erl index 82dc9fe3b6..8f325bec90 100644 --- a/lib/ssl/test/ssl_api_SUITE.erl +++ b/lib/ssl/test/ssl_api_SUITE.erl @@ -160,6 +160,10 @@ client_options_negative_dependency_stateless/1, client_options_negative_dependency_role/0, client_options_negative_dependency_role/1, + client_options_negative_early_data/0, + client_options_negative_early_data/1, + server_options_negative_early_data/0, + server_options_negative_early_data/1, server_options_negative_version_gap/0, server_options_negative_version_gap/1, server_options_negative_dependency_role/0, @@ -319,6 +323,8 @@ tls13_group() -> client_options_negative_dependency_version, client_options_negative_dependency_stateless, client_options_negative_dependency_role, + client_options_negative_early_data, + server_options_negative_early_data, server_options_negative_version_gap, server_options_negative_dependency_role, invalid_options_tls13, @@ -389,6 +395,15 @@ init_per_testcase(connection_information_with_srp, Config) -> false -> {skip, "Missing SRP crypto support"} end; +init_per_testcase(conf_signature_algs, Config) -> + case ssl_test_lib:appropriate_sha(crypto:supports()) of + sha256 -> + ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:timetrap({seconds, 10}), + Config; + sha -> + {skip, "Tests needs certs with sha256"} + end; init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 10}), @@ -688,14 +703,16 @@ conf_signature_algs(Config) when is_list(Config) -> ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ServerOpts]}]), + {options, [{active, false}, {signature_algs, [{sha256, rsa}]}, + {versions, ['tlsv1.2']} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ClientOpts]}]), + {options, [{active, false}, {signature_algs, [{sha256, rsa}]}, + {versions, ['tlsv1.2']} | ClientOpts]}]), ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), @@ -776,7 +793,11 @@ handshake_continue_tls13_client(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - + SCiphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, 'tlsv1.3'), + [{key_exchange, fun(srp_rsa) -> false; + (srp_anon) -> false; + (srp_dss) -> false; + (_) -> true end}]), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -784,6 +805,7 @@ handshake_continue_tls13_client(Config) when is_list(Config) -> {options, ssl_test_lib:ssl_options([{reuseaddr, true}, {log_level, debug}, {verify, verify_peer}, + {ciphers, SCiphers}, {handshake, hello} | ServerOpts ], Config)}, @@ -826,6 +848,7 @@ handshake_continue_tls13_client(Config) when is_list(Config) -> 'tlsv1.1', 'tlsv1' ]}, + {ciphers, ssl:cipher_suites(all, 'tlsv1.3')}, {verify, verify_peer} | ClientOpts ], Config)}, @@ -1859,7 +1882,7 @@ new_options_in_handshake(Config) when is_list(Config) -> (ecdh_rsa) -> true; (rsa) -> - true; + false; (_) -> false end @@ -2206,6 +2229,91 @@ client_options_negative_dependency_role(Config) when is_list(Config) -> {session_tickets,{stateless,{client,[disabled,manual,auto]}}}}). %%-------------------------------------------------------------------- +client_options_negative_early_data() -> + [{doc,"Test client option early_data."}]. +client_options_negative_early_data(Config) when is_list(Config) -> + start_client_negative(Config, [{versions, ['tlsv1.2']}, + {early_data, "test"}], + {options,dependency, + {early_data,{versions,['tlsv1.3']}}}), + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {early_data, "test"}], + {options,dependency, + {early_data,{session_tickets,[manual,auto]}}}), + + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, stateful}, + {early_data, "test"}], + {options,role, + {session_tickets, + {stateful,{client,[disabled,manual,auto]}}}}), + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, disabled}, + {early_data, "test"}], + {options,dependency, + {early_data,{session_tickets,[manual,auto]}}}), + + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}, + {early_data, "test"}], + {options,dependency, + {early_data, use_ticket}}), + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}, + {use_ticket, [<<"ticket">>]}, + {early_data, "test"}], + {options, type, + {early_data, {"test", not_binary}}}), + %% All options are ok but there is no server + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}, + {use_ticket, [<<"ticket">>]}, + {early_data, <<"test">>}], + econnrefused), + + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, auto}, + {early_data, "test"}], + {options, type, + {early_data, {"test", not_binary}}}), + %% All options are ok but there is no server + start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, auto}, + {early_data, <<"test">>}], + econnrefused). + +%%-------------------------------------------------------------------- +server_options_negative_early_data() -> + [{doc,"Test server option early_data."}]. +server_options_negative_early_data(Config) when is_list(Config) -> + start_server_negative(Config, [{versions, ['tlsv1.2']}, + {early_data, "test"}], + {options,dependency, + {early_data,{versions,['tlsv1.3']}}}), + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {early_data, "test"}], + {options,dependency, + {early_data,{session_tickets,[stateful,stateless]}}}), + + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, manual}, + {early_data, "test"}], + {options,role, + {session_tickets, + {manual,{server,[disabled,stateful,stateless]}}}}), + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, disabled}, + {early_data, "test"}], + {options,dependency, + {early_data,{session_tickets,[stateful,stateless]}}}), + + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, stateful}, + {early_data, "test"}], + {options,role, + {early_data,{"test",{server,[disabled,enabled]}}}}). + +%%-------------------------------------------------------------------- server_options_negative_version_gap() -> [{doc,"Test server options with faulty version gap."}]. server_options_negative_version_gap(Config) when is_list(Config) -> @@ -2554,8 +2662,8 @@ prf_ciphers_and_expected(TlsVer, PRFs, Results) -> case TlsVer of TlsVer when TlsVer == sslv3 orelse TlsVer == tlsv1 orelse TlsVer == 'tlsv1.1' orelse TlsVer == 'dtlsv1' -> - Ciphers = ssl:cipher_suites(), - {_, Expected} = lists:keyfind(md5sha, 1, Results), + Ciphers = ssl:cipher_suites(default, TlsVer), + Expected = [Expect#{prf := md5sha} || Expect <- Results], [[{tls_ver, TlsVer}, {ciphers, Ciphers}, {expected, Expected}, {prf, md5sha}]]; TlsVer when TlsVer == 'tlsv1.2' orelse TlsVer == 'dtlsv1.2'-> lists:foldl( @@ -2566,22 +2674,23 @@ prf_ciphers_and_expected(TlsVer, PRFs, Results) -> ct:log("No ciphers for PRF algorithm ~p. Skipping.", [PRF]), Acc; Ciphers -> - {_, Expected} = lists:keyfind(PRF, 1, Results), + Expected = [Expect#{prf := PRF} || Expect <- Results], [[{tls_ver, TlsVer}, {ciphers, Ciphers}, {expected, Expected}, {prf, PRF}] | Acc] end end, [], PRFs) end. -prf_get_ciphers(_, PRF) -> - lists:filter( - fun(C) when tuple_size(C) == 4 andalso - element(4, C) == PRF -> - true; - (_) -> - false - end, - ssl:cipher_suites()). +prf_get_ciphers(TlsVer, PRF) -> + PrfFilter = fun(Value) -> + case Value of + PRF -> + true; + _ -> + false + end + end, + ssl:filter_cipher_suites(ssl:cipher_suites(default, TlsVer), [{prf, PrfFilter}]). prf_run_test(_, TlsVer, [], _, Prf) -> ct:fail({error, cipher_list_empty, TlsVer, Prf}); @@ -2761,10 +2870,12 @@ cookie_extension(Config, Cookie) -> start_client_negative(Config, Options, Error) -> ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, 0}, + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Port = ssl_test_lib:inet_port(ServerNode), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, + {return_error, econnrefused}, {mfa, {?MODULE, connection_info_result, []}}, {options, Options ++ ClientOpts}]), ct:pal("Actual: ~p~nExpected: ~p", [Client, {connect_failed, Error}]), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 1426c7ef49..a5ece4fad8 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -25,6 +25,7 @@ -behaviour(ct_suite). -include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). -include_lib("ssl/src/ssl_api.hrl"). %% Callback functions @@ -64,12 +65,21 @@ unordered_protocol_versions_server/0, unordered_protocol_versions_server/1, unordered_protocol_versions_client/0, - unordered_protocol_versions_client/1 + unordered_protocol_versions_client/1, + fake_root/0, + fake_root/1, + fake_root_legacy/0, + fake_root_legacy/1, + fake_root_no_intermediate/0, + fake_root_no_intermediate/1, + fake_root_no_intermediate_legacy/0, + fake_root_no_intermediate_legacy/1, + fake_intermediate_cert/0, + fake_intermediate_cert/1 ]). %% Apply export --export([send_recv_result/1, - tcp_send_recv_result/1, +-export([tcp_send_recv_result/1, result_ok/1, protocol_info_result/1, version_info_result/1, @@ -112,7 +122,12 @@ basic_tests() -> eccs, cipher_suites, old_cipher_suites, - cipher_suites_mix + cipher_suites_mix, + fake_root, + fake_root_no_intermediate, + fake_root_legacy, + fake_root_no_intermediate_legacy, + fake_intermediate_cert ]. options_tests() -> @@ -293,7 +308,7 @@ cipher_suites(Config) when is_list(Config) -> cipher => 'aes_128_cbc', mac => sha, prf => default_prf}, - Version = ssl_test_lib:protocol_version(Config), + Version = tls_record:highest_protocol_version([]), All = [_|_] = ssl:cipher_suites(all, Version), Default = [_|_] = ssl:cipher_suites(default, Version), Anonymous = [_|_] = ssl:cipher_suites(anonymous, Version), @@ -498,14 +513,252 @@ tls_versions_option(Config) when is_list(Config) -> end, ssl_test_lib:check_client_alert(ErrClient, protocol_version). +fake_root() -> + [{doc,"Test that we can not use a fake root signed by other key but with correct name and serial number."}]. +fake_root(Config) when is_list(Config) -> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = #{cert := Cert, + key := _Key} = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + FakeKey = ssl_test_lib:hardcode_rsa_key(1), + OTPCert = public_key:pkix_decode_cert(Cert, otp), + TBS = OTPCert#'OTPCertificate'.tbsCertificate, + FakeCert = public_key:pkix_sign(TBS, FakeKey), + + + AuthExt = #'AuthorityKeyIdentifier'{authorityCertIssuer = [{directoryName, TBS#'OTPTBSCertificate'.issuer}], + authorityCertSerialNumber = TBS#'OTPTBSCertificate'.serialNumber}, + [AuthKeyExt] = x509_test:extensions([{?'id-ce-authorityKeyIdentifier', + AuthExt, + false}]), + + #{server_config := ServerConf, + client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}, + {extensions, [AuthKeyExt]}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := FakeServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => #{cert => FakeCert, key => FakeKey}, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}, + {extensions, [AuthKeyExt]}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + + test_fake_root(Hostname, ServerNode, ClientNode, ServerConf, ClientConf, FakeCert, FakeServerConf, bad_certificate, bad_certificate). + +fake_root_no_intermediate() -> + [{doc,"Test that we can not use a fake root signed by other key but with correct name and serial number."}]. + +fake_root_no_intermediate(Config) when is_list(Config) -> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = #{cert := Cert, + key := _Key} = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + FakeKey = ssl_test_lib:hardcode_rsa_key(1), + OTPCert = public_key:pkix_decode_cert(Cert, otp), + TBS = OTPCert#'OTPCertificate'.tbsCertificate, + FakeCert = public_key:pkix_sign(TBS, FakeKey), + + AuthExt = #'AuthorityKeyIdentifier'{authorityCertIssuer = [{directoryName, TBS#'OTPTBSCertificate'.issuer}], + authorityCertSerialNumber = TBS#'OTPTBSCertificate'.serialNumber}, + [AuthKeyExt] = x509_test:extensions([{?'id-ce-authorityKeyIdentifier', + AuthExt, + false}]), + + + #{server_config := ServerConf, + client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}, + {extensions, [AuthKeyExt]}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := FakeServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => #{cert => FakeCert, key => FakeKey}, + intermediates => [], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}, + {extensions, [AuthKeyExt]}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + test_fake_root(Hostname, ServerNode, ClientNode, ServerConf, ClientConf, FakeCert, FakeServerConf, bad_certificate, bad_certificate). + +fake_root_legacy() -> + [{doc,"Test that we can not use a fake root signed by other key but with correct name and serial number."}]. +fake_root_legacy(Config) when is_list(Config) -> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = #{cert := Cert, + key := _Key} = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + FakeKey = ssl_test_lib:hardcode_rsa_key(1), + OTPCert = public_key:pkix_decode_cert(Cert, otp), + TBS = OTPCert#'OTPCertificate'.tbsCertificate, + FakeCert = public_key:pkix_sign(TBS, FakeKey), + + #{server_config := ServerConf, + client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := FakeServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => #{cert => FakeCert, key => FakeKey}, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + + test_fake_root(Hostname, ServerNode, ClientNode, ServerConf, ClientConf, FakeCert, FakeServerConf, unknown_ca, unknown_ca). + +fake_root_no_intermediate_legacy() -> + [{doc,"Test that we can not use a fake root signed by other key but with correct name and serial number."}]. +fake_root_no_intermediate_legacy(Config) when is_list(Config) -> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = #{cert := Cert, + key := _Key} = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + FakeKey = ssl_test_lib:hardcode_rsa_key(1), + OTPCert = public_key:pkix_decode_cert(Cert, otp), + TBS = OTPCert#'OTPCertificate'.tbsCertificate, + FakeCert = public_key:pkix_sign(TBS, FakeKey), + + #{server_config := ServerConf, + client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := FakeServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => #{cert => FakeCert, key => FakeKey}, + intermediates => [], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + test_fake_root(Hostname, ServerNode, ClientNode, ServerConf, ClientConf, FakeCert, FakeServerConf, unknown_ca, unknown_ca). + +fake_intermediate_cert() -> + [{doc,"Test that we can not use a fake intermediat cert claiming to be signed by a trusted ROOT but is not."}]. + +fake_intermediate_cert(Config) when is_list(Config) -> + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Ext = x509_test:extensions([{key_usage, [keyCertSign, cRLSign, digitalSignature, keyAgreement]}]), + ROOT = #{cert := Cert, + key := _Key} = public_key:pkix_test_root_cert("SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(6)}, + {extensions, Ext}]), + + OtherSROOT = #{cert := OtherSCert, + key := OtherSKey} = public_key:pkix_test_root_cert("OTHER SERVER ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(3)}, + {extensions, Ext}]), + OtherCROOT = #{cert := OtherCCert, + key := _OtherCKey} = public_key:pkix_test_root_cert("OTHER Client ROOT CA", [{key, ssl_test_lib:hardcode_rsa_key(1)}, + {extensions, Ext}]), + + #{client_config := ClientConf} = public_key:pkix_test_data(#{server_chain => + #{root => ROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(4)}]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + + #{server_config := OtherServerConf} = public_key:pkix_test_data(#{server_chain => + #{root => OtherSROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(1)}]}, + client_chain => + #{root => OtherCROOT, + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(3)}]}} + ), + OTPCert = public_key:pkix_decode_cert(Cert, otp), + TBS = OTPCert#'OTPCertificate'.tbsCertificate, + TBSExt = TBS#'OTPTBSCertificate'.extensions, + AuthExt = #'AuthorityKeyIdentifier'{authorityCertIssuer = [{directoryName, TBS#'OTPTBSCertificate'.issuer}], + authorityCertSerialNumber = TBS#'OTPTBSCertificate'.serialNumber}, + [AuthKeyExt] = x509_test:extensions([{?'id-ce-authorityKeyIdentifier', + AuthExt, + false}]), + + + CAs = proplists:get_value(cacerts, OtherServerConf), + + [ICA] = CAs -- [OtherSCert, OtherCCert], + + OTPICACert = public_key:pkix_decode_cert(ICA, otp), + ICATBS = OTPICACert#'OTPCertificate'.tbsCertificate, + + FakeICA = public_key:pkix_sign(ICATBS#'OTPTBSCertificate'{extensions = [AuthKeyExt | TBSExt]}, OtherSKey), + + ServerCert = proplists:get_value(cert, OtherServerConf), + FakeServer = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, [{cert, [ServerCert, FakeICA]} | + proplists:delete(cert, OtherServerConf)] + }]), + Port1 = ssl_test_lib:inet_port(FakeServer), + + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port1}, + {host, Hostname}, + {from, self()}, + {options, [{verify, verify_peer} | ClientConf]}]), + + ssl_test_lib:check_client_alert(Client1, bad_certificate). %%-------------------------------------------------------------------- %% callback functions ------------------------------------------------ %%-------------------------------------------------------------------- -send_recv_result(Socket) -> - ssl:send(Socket, "Hello world"), - {ok,"Hello world"} = ssl:recv(Socket, 11), - ok. tcp_send_recv_result(Socket) -> gen_tcp:send(Socket, "Hello world"), {ok,"Hello world"} = gen_tcp:recv(Socket, 11), @@ -576,3 +829,60 @@ remove_supported_versions(Available, Supported) -> Versions0 end. + +test_fake_root(Hostname, ServerNode, ClientNode, ServerConf, ClientConf, FakeCert, FakeServerConf, ResultRootIncluded, ResultRootExcluded) -> + RealServer = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerConf}]), + Port0 = ssl_test_lib:inet_port(RealServer), + Client0 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port0}, + {host, Hostname}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {from, self()}, + {options, [{verify, verify_peer} | ClientConf]}]), + + ssl_test_lib:check_result(RealServer, ok, Client0, ok), + + ssl_test_lib:close(RealServer), + ssl_test_lib:close(Client0), + + %% Fake server sends ROOT cert + FakeServer = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, FakeServerConf}]), + Port1 = ssl_test_lib:inet_port(FakeServer), + + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port1}, + {host, Hostname}, + {from, self()}, + {options, [{verify, verify_peer} | ClientConf]}]), + + ssl_test_lib:check_client_alert(Client1, ResultRootIncluded), + + + %%Fake server does not send ROOT cert + CAS0 = proplists:get_value(cacerts, FakeServerConf), + CAS1 = CAS0 -- [FakeCert], + + FakeServer1 = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, [{cacerts, CAS1} | proplists:delete(cacerts, FakeServerConf)]}]), + + Port2 = ssl_test_lib:inet_port(FakeServer1), + + Client2 = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port2}, + {host, Hostname}, + {from, self()}, + {options, [{verify, verify_peer} | ClientConf]}]), + + ssl_test_lib:check_client_alert(Client2, ResultRootExcluded), + + ssl_test_lib:close(FakeServer1). + + + + + diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl index 99911af370..22dbc3663c 100644 --- a/lib/ssl/test/ssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -124,7 +124,11 @@ aes_128_gcm_sha256/1, chacha20_poly1305_sha256/1, aes_128_ccm_sha256/1, - aes_128_ccm_8_sha256/1 + aes_128_ccm_8_sha256/1, + ecdhe_ecdsa_with_aes_128_ccm/1, + ecdhe_ecdsa_with_aes_256_ccm/1, + ecdhe_ecdsa_with_aes_128_ccm_8/1, + ecdhe_ecdsa_with_aes_256_ccm_8/1 ]). -define(TIMEOUT, {seconds, 10}). @@ -171,7 +175,11 @@ groups() -> ecdhe_ecdsa_aes_128_gcm, ecdhe_ecdsa_aes_256_cbc, ecdhe_ecdsa_aes_256_gcm, - ecdhe_ecdsa_chacha20_poly1305 + ecdhe_ecdsa_chacha20_poly1305, + ecdhe_ecdsa_with_aes_128_ccm, + ecdhe_ecdsa_with_aes_256_ccm, + ecdhe_ecdsa_with_aes_128_ccm_8, + ecdhe_ecdsa_with_aes_256_ccm_8 ]}, {rsa, [], [rsa_3des_ede_cbc, rsa_aes_128_cbc, @@ -482,6 +490,26 @@ init_per_testcase(aes_128_ccm_8_sha256, Config) -> _ -> {skip, "Missing AES_128_CCM_8_SHA256 crypto support"} end; +init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_128_ccm; + TestCase == ecdhe_ecdsa_with_aes_128_ccm_8-> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap(?TIMEOUT), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == ecdhe_ecdsa_with_aes_256_ccm; + TestCase == ecdhe_ecdsa_with_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap(?TIMEOUT), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; init_per_testcase(TestCase, Config) -> Cipher = ssl_test_lib:test_cipher(TestCase, Config), SupCiphers = proplists:get_value(ciphers, crypto:supports()), @@ -744,6 +772,18 @@ ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). + +ecdhe_ecdsa_with_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_ccm', Config). + +ecdhe_ecdsa_with_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_ccm', Config). + +ecdhe_ecdsa_with_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_ccm_8', Config). + +ecdhe_ecdsa_with_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_ccm_8', Config). %%-------------------------------------------------------------------- %% DHE_DSS -------------------------------------------------------- %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 019e22eaa8..3b0c4d8c09 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -56,17 +56,7 @@ verify_fun_fail/0, verify_fun_fail/1, verify_fun_pass/0, - verify_fun_pass/1, - crl_check_pass/0, - crl_check_pass/1, - crl_check_fail/0, - crl_check_fail/1, - crl_check_best_effort/0, - crl_check_best_effort/1, - crl_cache_check_pass/0, - crl_cache_check_pass/1, - crl_cache_check_fail/0, - crl_cache_check_fail/1 + verify_fun_pass/1 ]). %% Apply export @@ -80,19 +70,9 @@ connect_options_test/3, verify_fun_fail_test/3, verify_fun_pass_test/3, - crl_check_fail_test/3, - crl_check_best_effort_test/3, - crl_check_pass_test/3, - crl_cache_check_fail_test/3, - crl_cache_check_pass_test/3, verify_pass_always/3, verify_fail_always/3]). -%% CRL API --export([lookup/2, - select/2, - fresh_crl/2 - ]). -define(DEFAULT_TIMETRAP_SECS, 240). -define(AWAIT_SSL_NODE_UP_TIMEOUT, 30000). @@ -119,14 +99,11 @@ all() -> connect_options, use_interface, verify_fun_fail, - verify_fun_pass, - crl_check_pass, - crl_check_fail, - crl_check_best_effort, - crl_cache_check_pass, - crl_cache_check_fail]. + verify_fun_pass + ]. init_per_suite(Config0) -> + _ = end_per_suite(Config0), try crypto:start() of ok -> %% Currently no ct function avilable for is_cover! @@ -142,18 +119,17 @@ init_per_suite(Config0) -> {skip, "Crypto did not start"} end. -end_per_suite(Config) -> - application:stop(crypto), - Config. +end_per_suite(_Config) -> + application:stop(crypto). init_per_testcase(plain_verify_options = Case, Config) when is_list(Config) -> - SslFlags = setup_dist_opts([{many_verify_opts, true} | Config]), + SslFlags = setup_tls_opts(Config), Flags = case os:getenv("ERL_FLAGS") of false -> os:putenv("ERL_FLAGS", SslFlags), ""; OldFlags -> - os:putenv("ERL_FLAGS", OldFlags ++ "" ++ SslFlags), + os:putenv("ERL_FLAGS", OldFlags ++ " " ++ SslFlags), OldFlags end, common_init(Case, [{old_flags, Flags} | Config]); @@ -184,32 +160,30 @@ basic(Config) when is_list(Config) -> %%-------------------------------------------------------------------- payload() -> - [{doc,"Test that send a lot of data between the ssl distributed noes"}]. + [{doc,"Test that send a lot of data between the ssl distributed nodes"}]. payload(Config) when is_list(Config) -> gen_dist_test(payload_test, Config). %%-------------------------------------------------------------------- plain_options() -> - [{doc,"Test specifying additional options"}]. + [{doc,"Test specifying tls options not related to certificate verification"}]. plain_options(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt server_secure_renegotiate true " + TLSOpts = "-ssl_dist_opt server_secure_renegotiate true " "client_secure_renegotiate true " - "server_reuse_sessions true client_reuse_sessions true " - "client_verify verify_none server_verify verify_none " - "server_depth 1 client_depth 1 " "server_hibernate_after 500 client_hibernate_after 500", - gen_dist_test(plain_options_test, [{additional_dist_opts, DistOpts} | Config]). + gen_dist_test(plain_options_test, [{tls_only_basic_opts, TLSOpts} | Config]). %%-------------------------------------------------------------------- plain_verify_options() -> - [{doc,"Test specifying additional options"}]. + [{doc,"Test specifying tls options including certificate verification options"}]. plain_verify_options(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt server_secure_renegotiate true " + TLSOpts = "-ssl_dist_opt server_secure_renegotiate true " "client_secure_renegotiate true " + "server_hibernate_after 500 client_hibernate_after 500" "server_reuse_sessions true client_reuse_sessions true " - "server_hibernate_after 500 client_hibernate_after 500", - gen_dist_test(plain_verify_options_test, [{additional_dist_opts, DistOpts} | Config]). + "server_depth 1 client_depth 1 ", + gen_dist_test(plain_verify_options_test, [{tls_verify_opts, TLSOpts} | Config]). %%-------------------------------------------------------------------- nodelay_option() -> @@ -239,7 +213,7 @@ listen_port_options(Config) when is_list(Config) -> PortOpt1 = "-kernel inet_dist_listen_min " ++ integer_to_list(Port1) ++ " inet_dist_listen_max " ++ integer_to_list(Port1), - try start_ssl_node([{additional_dist_opts, PortOpt1} | Config]) of + try start_ssl_node([{tls_verify_opts, PortOpt1} | proplists:delete(tls_verify_opts, Config)]) of #node_handle{} -> %% If the node was able to start, it didn't take the port %% option into account. @@ -254,7 +228,7 @@ listen_port_options(Config) when is_list(Config) -> %% Try again, now specifying a high max port. PortOpt2 = "-kernel inet_dist_listen_min " ++ integer_to_list(Port1) ++ " inet_dist_listen_max 65535", - NH2 = start_ssl_node([{additional_dist_opts, PortOpt2} | Config]), + NH2 = start_ssl_node([{tls_verify_opts, PortOpt2} | proplists:delete(tls_verify_opts, Config)]), try Node2 = NH2#node_handle.nodename, @@ -300,7 +274,7 @@ use_interface(Config) when is_list(Config) -> Options = "-kernel inet_dist_use_interface " ++ IpString, %% Start a node, and get the port number it's listening on. - NH1 = start_ssl_node([{additional_dist_opts, Options} | Config]), + NH1 = start_ssl_node([{tls_verify_opts, Options} | Config]), try Node1 = NH1#node_handle.nodename, @@ -330,89 +304,24 @@ use_interface(Config) when is_list(Config) -> verify_fun_fail() -> [{doc,"Test specifying verify_fun with a function that always fails"}]. verify_fun_fail(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt " - "server_verify verify_peer server_verify_fun " + AddTLSVerifyOpts = "-ssl_dist_opt " + "server_verify_fun " "\"{ssl_dist_SUITE,verify_fail_always,{}}\" " - "client_verify verify_peer client_verify_fun " + "client_verify_fun " "\"{ssl_dist_SUITE,verify_fail_always,{}}\" ", - gen_dist_test(verify_fun_fail_test, [{additional_dist_opts, DistOpts} | Config]). + gen_dist_test(verify_fun_fail_test, [{tls_verify_opts, AddTLSVerifyOpts} | Config]). %%-------------------------------------------------------------------- verify_fun_pass() -> [{doc,"Test specifying verify_fun with a function that always succeeds"}]. verify_fun_pass(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt " - "server_verify verify_peer server_verify_fun " + AddTLSVerifyOpts = "-ssl_dist_opt " + "server_verify_fun " "\"{ssl_dist_SUITE,verify_pass_always,{}}\" " - "server_fail_if_no_peer_cert true " - "client_verify verify_peer client_verify_fun " + "client_verify_fun " "\"{ssl_dist_SUITE,verify_pass_always,{}}\" ", - gen_dist_test(verify_fun_pass_test, [{additional_dist_opts, DistOpts} | Config]). - - -%%-------------------------------------------------------------------- -crl_check_pass() -> - [{doc,"Test crl_check with non-revoked certificate"}]. -crl_check_pass(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt client_crl_check true", - NewConfig = - [{many_verify_opts, true}, {additional_dist_opts, DistOpts}] ++ Config, - gen_dist_test(crl_check_pass_test, NewConfig). - -%%-------------------------------------------------------------------- -crl_check_fail() -> - [{doc,"Test crl_check with revoked certificate"}]. -crl_check_fail(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt client_crl_check true", - NewConfig = - [{many_verify_opts, true}, - %% The server uses a revoked certificate. - {server_cert_dir, "revoked"}, - {additional_dist_opts, DistOpts}] ++ Config, - gen_dist_test(crl_check_fail_test, NewConfig). - -%%-------------------------------------------------------------------- -crl_check_best_effort() -> - [{doc,"Test specifying crl_check as best_effort"}]. -crl_check_best_effort(Config) when is_list(Config) -> - DistOpts = "-ssl_dist_opt " - "server_verify verify_peer server_crl_check best_effort", - NewConfig = - [{many_verify_opts, true}, {additional_dist_opts, DistOpts}] ++ Config, - gen_dist_test(crl_check_best_effort_test, NewConfig). - -%%-------------------------------------------------------------------- -crl_cache_check_pass() -> - [{doc,"Test specifying crl_check with custom crl_cache module"}]. -crl_cache_check_pass(Config) when is_list(Config) -> - PrivDir = ?config(priv_dir, Config), - NodeDir = filename:join([PrivDir, "Certs"]), - DistOpts = "-ssl_dist_opt " - "client_crl_check true " - "client_crl_cache " - "\"{ssl_dist_SUITE,{\\\"" ++ NodeDir ++ "\\\",[]}}\"", - NewConfig = - [{many_verify_opts, true}, {additional_dist_opts, DistOpts}] ++ Config, - gen_dist_test(crl_cache_check_pass_test, NewConfig). - -%%-------------------------------------------------------------------- -crl_cache_check_fail() -> - [{doc,"Test custom crl_cache module with revoked certificate"}]. -crl_cache_check_fail(Config) when is_list(Config) -> - PrivDir = ?config(priv_dir, Config), - NodeDir = filename:join([PrivDir, "Certs"]), - DistOpts = "-ssl_dist_opt " - "client_crl_check true " - "client_crl_cache " - "\"{ssl_dist_SUITE,{\\\"" ++ NodeDir ++ "\\\",[]}}\"", - NewConfig = - [{many_verify_opts, true}, - %% The server uses a revoked certificate. - {server_cert_dir, "revoked"}, - {additional_dist_opts, DistOpts}] ++ Config, - - gen_dist_test(crl_cache_check_fail_test, NewConfig). + gen_dist_test(verify_fun_pass_test, [{tls_verify_opts, AddTLSVerifyOpts} | Config]). %%-------------------------------------------------------------------- %%% Internal functions ----------------------------------------------- @@ -573,7 +482,7 @@ do_listen_options(Prio, Config) -> end, Options = "-kernel inet_dist_listen_options " ++ PriorityString, - gen_dist_test(listen_options_test, [{prio, Prio}, {additional_dist_opts, Options} | Config]). + gen_dist_test(listen_options_test, [{prio, Prio}, {tls_only_basic_opts, Options} | Config]). listen_options_test(NH1, NH2, Config) -> Prio = proplists:get_value(prio, Config), @@ -605,7 +514,7 @@ do_connect_options(Prio, Config) -> Options = "-kernel inet_dist_connect_options " ++ PriorityString, gen_dist_test(connect_options_test, - [{prio, Prio}, {additional_dist_opts, Options} | Config]). + [{prio, Prio}, {tls_only_basic_opts, Options} | Config]). connect_options_test(NH1, NH2, Config) -> Prio = proplists:get_value(prio, Config), @@ -662,58 +571,7 @@ verify_fun_pass_test(NH1, NH2, _) -> [{verify_pass_always_ran, true}] = apply_on_ssl_node(NH2, fun () -> ets:tab2list(verify_fun_ran) end). -crl_check_fail_test(NH1, NH2, Config) -> - Node2 = NH2#node_handle.nodename, - - PrivDir = ?config(priv_dir, Config), - cache_crls_on_ssl_nodes(PrivDir, ["erlangCA", "otpCA"], [NH1, NH2]), - - %% The server's certificate is revoked, so connection fails. - pang = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), - - [] = apply_on_ssl_node(NH1, fun () -> nodes() end), - [] = apply_on_ssl_node(NH2, fun () -> nodes() end). -crl_check_best_effort_test(NH1, NH2, _Config) -> - %% We don't have the correct CRL at hand, but since crl_check is - %% best_effort, we accept it anyway. - Node1 = NH1#node_handle.nodename, - Node2 = NH2#node_handle.nodename, - - pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), - - [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), - [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end). - -crl_check_pass_test(NH1, NH2, Config) -> - Node1 = NH1#node_handle.nodename, - Node2 = NH2#node_handle.nodename, - - PrivDir = ?config(priv_dir, Config), - cache_crls_on_ssl_nodes(PrivDir, ["erlangCA", "otpCA"], [NH1, NH2]), - - %% The server's certificate is not revoked, so connection succeeds. - pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), - - [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), - [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end). - -crl_cache_check_pass_test(NH1, NH2, _) -> - Node1 = NH1#node_handle.nodename, - Node2 = NH2#node_handle.nodename, - - pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), - - [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), - [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end). - - -crl_cache_check_fail_test(NH1, NH2, _) -> - Node2 = NH2#node_handle.nodename, - pang = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), - - [] = apply_on_ssl_node(NH1, fun () -> nodes() end), - [] = apply_on_ssl_node(NH2, fun () -> nodes() end). get_socket_priorities() -> [Priority || {ok,[{priority,Priority}]} <- @@ -723,36 +581,16 @@ inet_ports() -> [Port || Port <- erlang:ports(), element(2, erlang:port_info(Port, name)) =:= "tcp_inet"]. -%% -%% test_server side api -%% - start_ssl_node(Config) -> start_ssl_node(Config, ""). start_ssl_node(Config, XArgs) -> Name = mk_node_name(Config), - SSL = proplists:get_value(ssl_opts, Config), - SSLDistOpts = setup_dist_opts(Config), + App = proplists:get_value(app_opts, Config), + SSLOpts = setup_tls_opts(Config), start_ssl_node_name( - Name, SSL ++ " " ++ SSLDistOpts ++ XArgs). - -cache_crls_on_ssl_nodes(PrivDir, CANames, NHs) -> - [begin - File = filename:join([PrivDir, "Certs", CAName, "crl.pem"]), - {ok, PemBin} = file:read_file(File), - PemEntries = public_key:pem_decode(PemBin), - CRLs = [ CRL || {'CertificateList', CRL, not_encrypted} - <- PemEntries], - ok = apply_on_ssl_node(NH, ssl_manager, insert_crls, - ["no_distribution_point", CRLs, dist]) - end - || NH <- NHs, CAName <- CANames], - ok. + Name, App ++ " " ++ SSLOpts ++ XArgs). -%% -%% command line creation -%% mk_node_name(Config) -> N = erlang:unique_integer([positive]), @@ -763,107 +601,51 @@ mk_node_name(Config) -> ++ "_" ++ integer_to_list(N). -%% -%% Setup ssl dist info -%% - -rand_bin(N) -> - rand_bin(N, []). - -rand_bin(0, Acc) -> - Acc; -rand_bin(N, Acc) -> - rand_bin(N-1, [rand:uniform(256)-1|Acc]). - -make_randfile(Dir) -> - {ok, IoDev} = file:open(filename:join([Dir, "RAND"]), [write]), - ok = file:write(IoDev, rand_bin(1024)), - file:close(IoDev). - -append_files(FileNames, ResultFileName) -> - {ok, ResultFile} = file:open(ResultFileName, [write]), - do_append_files(FileNames, ResultFile). - -do_append_files([], RF) -> - ok = file:close(RF); -do_append_files([F|Fs], RF) -> - {ok, Data} = file:read_file(F), - ok = file:write(RF, Data), - do_append_files(Fs, RF). - setup_certs(Config) -> PrivDir = proplists:get_value(priv_dir, Config), - NodeDir = filename:join([PrivDir, "Certs"]), - RGenDir = filename:join([NodeDir, "rand_gen"]), - ok = file:make_dir(NodeDir), - ok = file:make_dir(RGenDir), - make_randfile(RGenDir), - [Hostname|_] = string:split(net_adm:localhost(), ".", all), - {ok, _} = make_certs:all(RGenDir, NodeDir, [{hostname,Hostname}]), - SDir = filename:join([NodeDir, "server"]), - SC = filename:join([SDir, "cert.pem"]), - SK = filename:join([SDir, "key.pem"]), - SKC = filename:join([SDir, "keycert.pem"]), - append_files([SK, SC], SKC), - CDir = filename:join([NodeDir, "client"]), - CC = filename:join([CDir, "cert.pem"]), - CK = filename:join([CDir, "key.pem"]), - CKC = filename:join([CDir, "keycert.pem"]), - append_files([CK, CC], CKC). - -setup_dist_opts(Config) -> + DerConfig = public_key:pkix_test_data(#{server_chain => #{root => rsa_root_key(1), + intermediates => [rsa_intermediate(2)], + peer => rsa_peer_key(3)}, + client_chain => #{root => rsa_root_key(1), + intermediates => [rsa_intermediate(5)], + peer => rsa_peer_key(6)}}), + ClientBase = filename:join([PrivDir, "rsa"]), + SeverBase = filename:join([PrivDir, "rsa"]), + + _ = x509_test:gen_pem_config_files(DerConfig, ClientBase, SeverBase). + +setup_tls_opts(Config) -> PrivDir = proplists:get_value(priv_dir, Config), - DataDir = proplists:get_value(data_dir, Config), - Dhfile = filename:join([DataDir, "dHParam.pem"]), - NodeDir = filename:join([PrivDir, "Certs"]), - SDir = filename:join([NodeDir, proplists:get_value(server_cert_dir, Config, "server")]), - CDir = filename:join([NodeDir, proplists:get_value(client_cert_dir, Config, "client")]), - SC = filename:join([SDir, "cert.pem"]), - SK = filename:join([SDir, "key.pem"]), - SKC = filename:join([SDir, "keycert.pem"]), - SCA = filename:join([CDir, "cacerts.pem"]), - CC = filename:join([CDir, "cert.pem"]), - CK = filename:join([CDir, "key.pem"]), - CKC = filename:join([CDir, "keycert.pem"]), - CCA = filename:join([SDir, "cacerts.pem"]), - - DistOpts = case proplists:get_value(many_verify_opts, Config, false) of - false -> - "-proto_dist inet_tls " - ++ "-ssl_dist_opt server_certfile " ++ SKC ++ " " - ++ "-ssl_dist_opt client_certfile " ++ CKC ++ " "; - true -> - case os:type() of - {win32, _} -> - "-proto_dist inet_tls " - ++ "-ssl_dist_opt server_certfile " ++ SKC ++ " " - ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " - ++ "-ssl_dist_opt server_verify verify_peer " - ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " - ++ "-ssl_dist_opt server_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " - ++ "-ssl_dist_opt server_dhfile " ++ Dhfile ++ " " - ++ "-ssl_dist_opt client_certfile " ++ CKC ++ " " - ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " - ++ "-ssl_dist_opt client_verify verify_peer " - ++ "-ssl_dist_opt client_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA "; - _ -> - "-proto_dist inet_tls " - ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " - ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " - ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " - ++ "-ssl_dist_opt server_verify verify_peer " - ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " - ++ "-ssl_dist_opt server_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " - ++ "-ssl_dist_opt server_dhfile " ++ Dhfile ++ " " - ++ "-ssl_dist_opt client_certfile " ++ CC ++ " " - ++ "-ssl_dist_opt client_keyfile " ++ CK ++ " " - ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " - ++ "-ssl_dist_opt client_verify verify_peer " - ++ "-ssl_dist_opt client_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " - end - end, - MoreOpts = proplists:get_value(additional_dist_opts, Config, []), - DistOpts ++ MoreOpts. + SC = filename:join([PrivDir, "rsa_server_cert.pem"]), + SK = filename:join([PrivDir, "rsa_server_key.pem"]), + SCA = filename:join([PrivDir, "rsa_server_cacerts.pem"]), + CC = filename:join([PrivDir, "rsa_client_cert.pem"]), + CK = filename:join([PrivDir, "rsa_client_key.pem"]), + CCA = filename:join([PrivDir, "rsa_client_cacerts.pem"]), + + case proplists:get_value(tls_only_basic_opts, Config, []) of + [_|_] = BasicOpts -> %% No verify but server still need to have cert + "-proto_dist inet_tls " ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " + ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " ++ BasicOpts; + [] -> %% Verify + case proplists:get_value(tls_verify_opts, Config, []) of + [_|_] -> + BasicVerifyOpts = "-proto_dist inet_tls " + ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " + ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " + ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " + ++ "-ssl_dist_opt server_verify verify_peer " + ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " + ++ "-ssl_dist_opt client_certfile " ++ CC ++ " " + ++ "-ssl_dist_opt client_keyfile " ++ CK ++ " " + ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " + ++ "-ssl_dist_opt client_verify verify_peer ", + BasicVerifyOpts ++ proplists:get_value(tls_verify_opts, Config, []); + _ -> %% No verify, no extra opts + "-proto_dist inet_tls " ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " + ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " + end + end. %% %% Start scripts etc... @@ -914,20 +696,16 @@ add_ssl_opts_config(Config) -> SSL_VSN]), ok = file:close(RelFile), ok = systools:make_script(Script, []), - [{ssl_opts, "-boot " ++ Script} | Config] + [{app_opts, "-boot " ++ Script} | Config] catch _:_ -> - [{ssl_opts, "-pa \"" ++ filename:dirname(code:which(ssl))++"\""} + [{app_opts, "-pa \"" ++ filename:dirname(code:which(ssl))++"\""} | add_comment_config( "Bootscript wasn't used since the test wasn't run on an " "installed OTP system.", Config)] end. -%% -%% Add common comments to config -%% - add_comment_config(Comment, []) -> [{comment, Comment}]; add_comment_config(Comment, [{comment, OldComment} | Cs]) -> @@ -935,9 +713,6 @@ add_comment_config(Comment, [{comment, OldComment} | Cs]) -> add_comment_config(Comment, [C|Cs]) -> [C|add_comment_config(Comment, Cs)]. -%% -%% Call when test case success -%% success(Config) -> case lists:keysearch(comment, 1, Config) of @@ -965,6 +740,7 @@ verify_fail_always(_Certificate, _Event, _State) -> Parent = self(), spawn( fun() -> + catch ets:delete(verify_fun_ran), ets:new(verify_fun_ran, [public, named_table]), ets:insert(verify_fun_ran, {verify_fail_always_ran, true}), Parent ! go_ahead, @@ -979,6 +755,7 @@ verify_pass_always(_Certificate, _Event, State) -> Parent = self(), spawn( fun() -> + catch ets:delete(verify_fun_ran), ets:new(verify_fun_ran, [public, named_table]), ets:insert(verify_fun_ran, {verify_pass_always_ran, true}), Parent ! go_ahead, @@ -987,28 +764,6 @@ verify_pass_always(_Certificate, _Event, State) -> receive go_ahead -> ok end, {valid, State}. -%% ssl_crl_cache_api callbacks -lookup(_DistributionPoint, _DbHandle) -> - not_available. - -select({rdnSequence, NameParts}, {NodeDir, _}) -> - %% Extract the CN from the issuer name... - [CN] = [CN || - [#'AttributeTypeAndValue'{ - type = ?'id-at-commonName', - value = <<_, _, CN/binary>>}] <- NameParts], - %% ...and use that as the directory name to find the CRL. - error_logger:info_report([{found_cn, CN}]), - CRLFile = filename:join([NodeDir, CN, "crl.pem"]), - {ok, PemBin} = file:read_file(CRLFile), - PemEntries = public_key:pem_decode(PemBin), - CRLs = [ CRL || {'CertificateList', CRL, not_encrypted} - <- PemEntries], - CRLs. - -fresh_crl(_DistributionPoint, CRL) -> - CRL. - localhost_ip(InetVer) -> {ok, Addr} = inet:getaddr(net_adm:localhost(), InetVer), Addr. @@ -1026,3 +781,16 @@ localhost_ipstr(InetVer) -> inet_ver() -> inet. + +rsa_root_key(N) -> + %% As rsa keygen is not guaranteed to be fast + [{key, ssl_test_lib:hardcode_rsa_key(N)}]. + +rsa_peer_key(N) -> + %% As rsa keygen is not guaranteed to be fast + [{key, ssl_test_lib:hardcode_rsa_key(N)}]. + +rsa_intermediate(N) -> + [{key, ssl_test_lib:hardcode_rsa_key(N)}]. + + diff --git a/lib/ssl/test/ssl_dist_test_lib.erl b/lib/ssl/test/ssl_dist_test_lib.erl index bf010e6ad4..fc6646690d 100644 --- a/lib/ssl/test/ssl_dist_test_lib.erl +++ b/lib/ssl/test/ssl_dist_test_lib.erl @@ -104,7 +104,7 @@ start_ssl_node(Name, Args) -> case open_port({spawn, CmdLine}, []) of Port when is_port(Port) -> unlink(Port), - erlang:port_close(Port), + catch erlang:port_close(Port), case await_ssl_node_up(Name, LSock) of #node_handle{} = NodeHandle -> ?t:format("Ssl node ~s started.~n", [Name]), diff --git a/lib/ssl/test/ssl_eqc_SUITE.erl b/lib/ssl/test/ssl_eqc_SUITE.erl index 15f8782d8a..3c9a1d0ab0 100644 --- a/lib/ssl/test/ssl_eqc_SUITE.erl +++ b/lib/ssl/test/ssl_eqc_SUITE.erl @@ -34,6 +34,8 @@ -export([tls_handshake_encoding/1, tls_cipher_suite_names/1, tls_cipher_openssl_suite_names/1, + tls_anon_cipher_suite_names/1, + tls_anon_cipher_openssl_suite_names/1, tls_unorded_chains/1, tls_extraneous_chain/1, tls_extraneous_chains/1, @@ -49,6 +51,8 @@ all() -> tls_handshake_encoding, tls_cipher_suite_names, tls_cipher_openssl_suite_names, + tls_anon_cipher_suite_names, + tls_anon_cipher_openssl_suite_names, tls_unorded_chains, tls_extraneous_chain, tls_extraneous_chains, @@ -85,6 +89,15 @@ tls_cipher_openssl_suite_names(Config) when is_list(Config) -> %% manual test: proper:quickcheck(ssl_eqc_handshake:prop_tls_cipher_suite_openssl_name()). true = ct_property_test:quickcheck(ssl_eqc_cipher_format:prop_tls_cipher_suite_openssl_name(), Config). +tls_anon_cipher_suite_names(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(ssl_eqc_cipher_format:prop_tls_cipher_suite_rfc_name()). + true = ct_property_test:quickcheck(ssl_eqc_cipher_format:prop_tls_anon_cipher_suite_rfc_name(), + Config). + +tls_anon_cipher_openssl_suite_names(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(ssl_eqc_handshake:prop_tls_cipher_suite_openssl_name()). + true = ct_property_test:quickcheck(ssl_eqc_cipher_format:prop_tls_anon_cipher_suite_openssl_name(), + Config). tls_unorded_chains(Config) when is_list(Config) -> %% manual test: proper:quickcheck(ssl_eqc_chain:prop_tls_ordered_path("/tmp") diff --git a/lib/ssl/test/ssl_session_SUITE.erl b/lib/ssl/test/ssl_session_SUITE.erl index 0a614f8b8c..b11e49ad89 100644 --- a/lib/ssl/test/ssl_session_SUITE.erl +++ b/lib/ssl/test/ssl_session_SUITE.erl @@ -143,8 +143,9 @@ reuse_session() -> reuse_session(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), - - ssl_test_lib:reuse_session(ClientOpts, ServerOpts, Config). + Version = ssl_test_lib:protocol_version(Config), + ssl_test_lib:reuse_session([{versions,[Version]} | ClientOpts], + [{versions,[Version]} | ServerOpts], Config). %%-------------------------------------------------------------------- reuse_session_expired() -> [{doc,"Test sessions is not reused when it has expired"}]. diff --git a/lib/ssl/test/ssl_session_ticket_SUITE.erl b/lib/ssl/test/ssl_session_ticket_SUITE.erl index 16791e2c36..d39afaa5b7 100644 --- a/lib/ssl/test/ssl_session_ticket_SUITE.erl +++ b/lib/ssl/test/ssl_session_ticket_SUITE.erl @@ -54,7 +54,23 @@ multiple_tickets/0, multiple_tickets/1, multiple_tickets_2hash/0, - multiple_tickets_2hash/1]). + multiple_tickets_2hash/1, + early_data_client_too_much_data/0, + early_data_client_too_much_data/1, + early_data_trial_decryption/0, + early_data_trial_decryption/1, + early_data_trial_decryption_failure/0, + early_data_trial_decryption_failure/1, + early_data_decryption_failure/0, + early_data_decryption_failure/1, + early_data_disabled_small_limit/0, + early_data_disabled_small_limit/1, + early_data_enabled_small_limit/0, + early_data_enabled_small_limit/1, + early_data_basic/0, + early_data_basic/1, + early_data_basic_auth/0, + early_data_basic_auth/1]). -include("tls_handshake.hrl"). @@ -83,7 +99,15 @@ session_tests() -> [basic, hello_retry_request, multiple_tickets, - multiple_tickets_2hash]. + multiple_tickets_2hash, + early_data_client_too_much_data, + early_data_trial_decryption, + early_data_trial_decryption_failure, + early_data_decryption_failure, + early_data_disabled_small_limit, + early_data_enabled_small_limit, + early_data_basic, + early_data_basic_auth]. mixed_tests() -> [ @@ -127,6 +151,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(_TestCase, Config) -> + application:unset_env(ssl, server_session_ticket_max_early_data), Config. %%-------------------------------------------------------------------- @@ -583,6 +608,516 @@ multiple_tickets_2hash(Config) when is_list(Config) -> process_flag(trap_exit, false), ssl_test_lib:close(Server0). +early_data_trial_decryption() -> + [{doc,"Test trial decryption when server rejects early data (erlang client - erlang server)"}]. +early_data_trial_decryption(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send maximum sized early data to verify calculation of plain text size + %% in the server. + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16384)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, disabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts1}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, {options, ClientOpts2}]), + ssl_test_lib:check_result(Server0, ok, Client1, ok), + + process_flag(trap_exit, false), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + +early_data_client_too_much_data() -> + [{doc,"Client sending too much early data (erlang client - erlang server)"}]. +early_data_client_too_much_data(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send more early data than max_early_data_size to verify calculation + %% of plain text size in the server. + MaxEarlyDataSize = 10000, + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16384)}|ClientOpts1], + + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, disabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + application:set_env(ssl, server_session_ticket_max_early_data, MaxEarlyDataSize), + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + application:unset_env(ssl, server_session_ticket_max_early_data), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false, no_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + Tickets0 = ssl_test_lib:check_tickets(Client0), + ssl_test_lib:verify_session_ticket_extension(Tickets0, MaxEarlyDataSize), + %% ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [false, no_reply, no_tickets]}}, + {from, self()}, {options, [{use_ticket, Tickets0}|ClientOpts2]}]), + ssl_test_lib:check_client_alert(Client1, illegal_parameter), + process_flag(trap_exit, false), + ssl_test_lib:close(Server0). + +early_data_trial_decryption_failure() -> + [{doc,"Emulate faulty client that sends too much early data (erlang client - erlang server)"}]. +early_data_trial_decryption_failure(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send more early data than max_early_data_size to verify calculation + %% of plain text size in the server. + MaxEarlyDataSize = 10000, + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16385)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + %% up to the configured amount. If more data is received the server triggers + %% a bad_record_mac alert. + %% It is not possible to trigger this condition in normal use cases: + %% - The ssl client in auto mode has a built in protection against sending + %% too much early data. It will not send any early data. + %% - The ssl client in manual mode can only send the mount that is received + %% in the ticket used for the 0-RTT handshake. If more data is sent the + %% client will trigger an illegal_parameter alert (too_much_early_data). + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, disabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + application:set_env(ssl, server_session_ticket_max_early_data, MaxEarlyDataSize), + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false, no_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + Tickets0 = ssl_test_lib:check_tickets(Client0), + %% Simulate a faulty client by updating the max_early_data_size extension in + %% the received session ticket + Tickets1 = ssl_test_lib:update_session_ticket_extension(Tickets0, 16385), + %% ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [false, no_reply, no_tickets]}}, + {from, self()}, {options, [{use_ticket, Tickets1}|ClientOpts2]}]), + ssl_test_lib:check_server_alert(Server0, bad_record_mac), + process_flag(trap_exit, false), + application:unset_env(ssl, server_session_ticket_max_early_data), + ssl_test_lib:close(Server0). + +early_data_decryption_failure() -> + [{doc,"Emulate faulty client that sends too much early data - server early_data enabled (erlang client - erlang server)"}]. +early_data_decryption_failure(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, manual}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send more early data than max_early_data_size to verify calculation + %% of plain text size in the server. + MaxEarlyDataSize = 10000, + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16385)}|ClientOpts1], + + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, enabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + application:set_env(ssl, server_session_ticket_max_early_data, MaxEarlyDataSize), + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false, no_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + Tickets0 = ssl_test_lib:check_tickets(Client0), + %% Simulate a faulty client by updating the max_early_data_size extension in + %% the received session ticket + Tickets1 = ssl_test_lib:update_session_ticket_extension(Tickets0, 16385), + %% ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false, no_reply]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client_error([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [false, no_reply, no_tickets]}}, + {from, self()}, {options, [{use_ticket, Tickets1}|ClientOpts2]}]), + ssl_test_lib:check_server_alert(Server0, unexpected_message), + process_flag(trap_exit, false), + application:unset_env(ssl, server_session_ticket_max_early_data), + ssl_test_lib:close(Server0). + +early_data_disabled_small_limit() -> + [{doc,"Test trial decryption when server rejects early data (erlang client - erlang server)"}]. +early_data_disabled_small_limit(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send maximum sized early data to verify calculation of plain text size + %% in the server. + MaxEarlyDataSize = 5, + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 4)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, disabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + application:set_env(ssl, server_session_ticket_max_early_data, MaxEarlyDataSize), + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts1}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, {options, ClientOpts2}]), + ssl_test_lib:check_result(Server0, ok, Client1, ok), + + process_flag(trap_exit, false), + application:unset_env(ssl, server_session_ticket_max_early_data), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + +early_data_enabled_small_limit() -> + [{doc,"Test decryption when server accepts early data (erlang client - erlang server)"}]. +early_data_enabled_small_limit(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send maximum sized early data to verify calculation of plain text size + %% in the server. + MaxEarlyDataSize = 5, + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 4)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, enabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + application:set_env(ssl, server_session_ticket_max_early_data, MaxEarlyDataSize), + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts1}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_active_session_resumption, + [true]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, {options, ClientOpts2}]), + ssl_test_lib:check_result(Server0, ok, Client1, ok), + + process_flag(trap_exit, false), + application:unset_env(ssl, server_session_ticket_max_early_data), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + +early_data_basic() -> + [{doc,"Test early data when client is not authenticated (erlang client - erlang server)"}]. +early_data_basic(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send maximum sized early data to verify calculation of plain text size + %% in the server. + EarlyData = binary:copy(<<"F">>, 16384), + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16384)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, enabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts1}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_server_early_data, + [wait_reply, EarlyData]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, {options, ClientOpts2}]), + ssl_test_lib:check_result(Server0, ok, Client1, ok), + + process_flag(trap_exit, false), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + +early_data_basic_auth() -> + [{doc,"Test early data when client is authenticated (erlang client - erlang server)"}]. +early_data_basic_auth(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + + %% Configure session tickets + ClientOpts1 = [{session_tickets, auto}, {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + %% Send maximum sized early data to verify calculation of plain text size + %% in the server. + EarlyData = binary:copy(<<"F">>, 16384), + ClientOpts2 = [{early_data, binary:copy(<<"F">>, 16384)}|ClientOpts1], + + %% Disabled early data triggers trial decryption upon receiving early data + ServerOpts = [{session_tickets, ServerTicketMode}, {early_data, enabled}, + {log_level, debug}, + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + %% Store ticket from first connection + Client0 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts1}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + Server0 ! {listen, {mfa, {ssl_test_lib, + verify_server_early_data, + [wait_reply, EarlyData]}}}, + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% TODO This test should fail! + %% State transition is not implemented from wait_eoed to wait_cert! + %% Use ticket + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, + {options, + proplists:delete(keyfile, + proplists:delete(certfile, ClientOpts2))}]), + ssl_test_lib:check_result(Server0, ok, Client1, ok), + + process_flag(trap_exit, false), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client1). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 5054730835..ddaba06bca 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -25,6 +25,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). +-include_lib("ssl/src/tls_handshake_1_3.hrl"). -export([clean_start/0, clean_start/1, @@ -98,6 +99,10 @@ verify_active_session_resumption/2, verify_active_session_resumption/3, verify_active_session_resumption/4, + verify_active_session_resumption/5, + verify_server_early_data/3, + verify_session_ticket_extension/2, + update_session_ticket_extension/2, check_sane_openssl_version/1, check_ok/1, check_result/4, @@ -119,7 +124,8 @@ server_msg/2, hardcode_rsa_key/1, bigger_buffers/0, - stop/2 + stop/2, + working_openssl_client/0 ]). -export([basic_test/3, @@ -132,7 +138,8 @@ reuse_session/3, test_ciphers/3, test_cipher/2, - openssl_ciphers/0 + openssl_ciphers/0, + openssl_support_rsa_kex/0 ]). -export([tls_version/1, @@ -168,7 +175,8 @@ ecdh_dh_anonymous_suites/1, ecdsa_suites/1, der_to_pem/2, - pem_to_der/1 + pem_to_der/1, + appropriate_sha/1 ]). -export([maybe_force_ipv4/1, @@ -191,7 +199,8 @@ version_flag/1, portable_cmd/2, portable_open_port/2, - close_port/1 + close_port/1, + verify_early_data/1 ]). -record(sslsocket, { fd = nil, pid = nil}). @@ -271,7 +280,7 @@ init_per_group(GroupName, Config0) -> case is_protocol_version(GroupName) andalso sufficient_crypto_support(GroupName) of true -> Config = clean_protocol_version(Config0), - init_protocol_version(GroupName, Config); + [{version, GroupName}|init_protocol_version(GroupName, Config)]; _ -> case sufficient_crypto_support(GroupName) of true -> @@ -282,9 +291,24 @@ init_per_group(GroupName, Config0) -> end end. -init_per_group_openssl(GroupName, Config) -> +working_openssl_client() -> + case portable_cmd("openssl", ["version"]) of + %% Theses versions of OpenSSL has a client that + %% can not handle hello extensions. And will + %% fail with bad packet length if they are present + %% in ServerHello + "OpenSSL 0.9.8h" ++ _ -> + false; + "OpenSSL 0.9.8k" ++ _ -> + false; + _ -> + true + end. + +init_per_group_openssl(GroupName, Config0) -> case is_tls_version(GroupName) andalso sufficient_crypto_support(GroupName) of true -> + Config = clean_protocol_version(Config0), case openssl_tls_version_support(GroupName, Config) of true -> @@ -296,7 +320,7 @@ init_per_group_openssl(GroupName, Config) -> case sufficient_crypto_support(GroupName) of true -> ssl:start(), - Config; + Config0; false -> {skip, "Missing crypto support"} end @@ -320,7 +344,21 @@ openssl_ocsp_support() -> openssl_ciphers() -> Str = portable_cmd("openssl", ["ciphers"]), - string:split(string:strip(Str, right, $\n), ":", all). + Ciphers = string:split(string:strip(Str, right, $\n), ":", all), + case portable_cmd("openssl", ["version"]) of + "LibreSSL 3." ++ _ -> + Ciphers -- ["DES-CBC3-SHA","AES128-SHA", "AES256-SHA", "RC4-SHA", "RC4-MD5"]; + _ -> + Ciphers + end. + +openssl_support_rsa_kex() -> + case portable_cmd("openssl", ["version"]) of + "OpenSSL 1.1.1" ++ _Rest -> + false; + _ -> + true + end. %%==================================================================== %% Internal functions @@ -628,7 +666,8 @@ start_openssl_server(Mode, Args0, Config) -> Node = proplists:get_value(node, Args0, ServerNode), Port = proplists:get_value(port, Args0, 0), ResponderPort = proplists:get_value(responder_port, Config, 0), - Args = [{from, self()}, {port, Port}] ++ ServerOpts ++ Args0, + PrivDir = proplists:get_value(priv_dir, Config), + Args = [{from, self()}, {port, Port}] ++ ServerOpts ++ Args0 ++ [{priv_dir, PrivDir}], Result = spawn_link(Node, ?MODULE, init_openssl_server, [Mode, ResponderPort,lists:delete(return_port, Args)]), receive @@ -650,10 +689,12 @@ init_openssl_server(openssl, _, Options) -> Exe = "openssl", Ciphers = proplists:get_value(ciphers, Options, default_ciphers(Version)), Groups0 = proplists:get_value(groups, Options), + EarlyData = proplists:get_value(early_data, Options, undefined), + PrivDir = proplists:get_value(priv_dir, Options), CertArgs = openssl_cert_options(Options, server), AlpnArgs = openssl_alpn_options(proplists:get_value(alpn, Options, undefined)), NpnArgs = openssl_npn_options(proplists:get_value(np, Options, undefined)), - Debug = openssl_debug_options(), + Debug = openssl_debug_options(PrivDir), Args0 = case Groups0 of undefined -> @@ -665,7 +706,14 @@ init_openssl_server(openssl, _, Options) -> ciphers(Ciphers, Version), "-groups", Group, version_flag(Version)] ++ AlpnArgs ++ NpnArgs ++ CertArgs ++ Debug end, - Args = maybe_force_ipv4(Args0), + Args1 = case EarlyData of + undefined -> + Args0; + MaxSize -> + Args0 ++ ["-early_data", "-no_anti_replay", "-max_early_data", + integer_to_list(MaxSize)] + end, + Args = maybe_force_ipv4(Args1), SslPort = portable_open_port(Exe, Args), wait_for_openssl_server(Port, proplists:get_value(protocol, Options, tls)), Pid ! {started, SslPort}, @@ -893,16 +941,21 @@ client_loop(_Node, Host, Port, Pid, Transport, Options, Opts) -> end, client_loop_core(Socket, Pid, Transport); {error, econnrefused = Reason} -> - case get(retries) of - N when N < 5 -> - ct:log("~p:~p~neconnrefused retries=~p sleep ~p",[?MODULE,?LINE, N,?SLEEP]), - put(retries, N+1), - ct:sleep(?SLEEP), - run_client(Opts); - _ -> - ct:log("~p:~p~nClient faild several times: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), - Pid ! {self(), {error, Reason}} - end; + case proplists:get_value(return_error, Opts, undefined) of + econnrefused -> + Pid ! {connect_failed, Reason}; + _ -> + case get(retries) of + N when N < 5 -> + ct:log("~p:~p~neconnrefused retries=~p sleep ~p",[?MODULE,?LINE, N,?SLEEP]), + put(retries, N+1), + ct:sleep(?SLEEP), + run_client(Opts); + _ -> + ct:log("~p:~p~nClient faild several times: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), + Pid ! {self(), {error, Reason}} + end + end; {error, econnreset = Reason} -> case get(retries) of N when N < 5 -> @@ -1085,11 +1138,14 @@ check_server_alert(Pid, Alert) -> {Pid, {error, {tls_alert, {Alert, STxt}}}} -> check_server_txt(STxt), ok; + {Pid, {error, {tls_alert, {OtherAlert, STxt}}}} -> + ct:fail("Unexpected alert during negative test: ~p - ~p", [OtherAlert, STxt]); {Pid, {error, closed}} -> ok; {Pid, {ok, _}} -> ct:fail("Successful connection during negative test.") end. + check_server_alert(Server, Client, Alert) -> receive {Server, {error, {tls_alert, {Alert, STxt}}}} -> @@ -1098,14 +1154,19 @@ check_server_alert(Server, Client, Alert) -> {Server, {ok, _}} -> ct:fail("Successful connection during negative test.") end. + check_client_alert(Pid, Alert) -> receive {Pid, {error, {tls_alert, {Alert, CTxt}}}} -> check_client_txt(CTxt), ok; + {Pid, {error, {tls_alert, {OtherAlert, CTxt}}}} -> + ct:fail("Unexpected alert during negative test: ~p - ~p", [OtherAlert, CTxt]); {Pid, {ssl_error, _, {tls_alert, {Alert, CTxt}}}} -> check_client_txt(CTxt), ok; + {Pid, {ssl_error, _, {tls_alert, {OtherAlert, CTxt}}}} -> + ct:fail("Unexpected alert during negative test: ~p - ~p", [OtherAlert, CTxt]); {Pid, {error, closed}} -> ok; {Pid, {ok, _}} -> @@ -1244,8 +1305,7 @@ cert_options(Config) -> {server_bad_key, [{ssl_imp, new},{cacertfile, ServerCaCertFile}, {certfile, ServerCertFile}, {keyfile, BadKeyFile}]} | Config]. - - + make_dsa_cert(Config) -> CryptoSupport = crypto:supports(), case proplists:get_bool(dss, proplists:get_value(public_keys, CryptoSupport)) of @@ -2020,13 +2080,12 @@ start_server(openssl, ClientOpts, ServerOpts, Config) -> start_server(erlang, _, ServerOpts, Config) -> {_, ServerNode, _} = run_where(Config), KeyEx = proplists:get_value(check_keyex, Config, false), - Versions = protocol_versions(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, check_key_exchange_send_active, [KeyEx]}}, - {options, [{verify, verify_peer}, {versions, Versions} | ServerOpts]}]), + {options, [{verify, verify_peer} | ServerOpts]}]), {Server, inet_port(Server)}. sig_algs(undefined) -> @@ -2039,9 +2098,11 @@ cipher_flag('tlsv1.3') -> cipher_flag(_) -> "-cipher". -ciphers(Ciphers, Version) -> +ciphers([#{}| _] = Ciphers, Version) -> Strs = [ssl_cipher_format:suite_map_to_openssl_str(Cipher) || Cipher <- Ciphers], - ciphers_concat(Version, Strs, ""). + ciphers_concat(Version, Strs, ""); +ciphers(Ciphers, Version) -> + ciphers_concat(Version, Ciphers, ""). ciphers_concat(_, [], [":" | Acc]) -> lists:append(lists:reverse(Acc)); @@ -2076,7 +2137,23 @@ openssl_maxfag_option(Int) -> openssl_debug_options() -> ["-msg", "-debug"]. +%% +openssl_debug_options(PrivDir) -> + case is_keylogfile_supported() of + true -> + ["-msg", "-debug","-keylogfile", PrivDir ++ "keylog"]; + false -> + ["-msg", "-debug"] + end. +is_keylogfile_supported() -> + [{_,_, Bin}] = crypto:info_lib(), + case binary_to_list(Bin) of + "OpenSSL 1.1.1" ++ _ -> + true; + _ -> + false + end. start_server_with_raw_key(erlang, ServerOpts, Config) -> {_, ServerNode, _} = run_where(Config), @@ -2144,7 +2221,8 @@ openssl_cert_options(Opts, Role) -> Other end; _ -> - cert_option("-cert", Cert) ++ cert_option("-CAfile", CA) ++ + cert_option("-cert", Cert) ++ cert_option("-CAfile", CA) + ++ cert_option("-cert_chain", CA) ++ cert_option("-key", Key) ++ openssl_verify(Opts) ++ ["2"] end. @@ -2158,6 +2236,13 @@ openssl_verify(Opts) -> cert_option(_, undefined) -> []; +cert_option("-cert_chain", Value) -> + case portable_cmd("openssl", ["version"]) of + "OpenSSL 1.1.1" ++ _ -> + ["-cert_chain", Value]; + _ -> + "" + end; cert_option(Opt, Value) -> [Opt, Value]. @@ -2412,7 +2497,7 @@ init_protocol_version(Version, Config) -> [{protocol, tls} | NewConfig]. clean_protocol_version(Config) -> - proplists:delete(protocol_opts, proplists:delete(protocol, Config)). + proplists:delete(version, proplists:delete(protocol_opts, proplists:delete(protocol, Config))). sufficient_crypto_support(Version) when Version == 'tlsv1.3' -> @@ -2519,13 +2604,19 @@ send_recv_result_active_once(Socket) -> ssl:send(Socket, Data), active_once_recv_list(Socket, length(Data)). +%% This function can verify the following functionalities in clients: +%% - session resumption, sending/receiving application data, receiving session tickets +%% - verifying if client early data is accepted/rejected verify_active_session_resumption(Socket, SessionResumption) -> - verify_active_session_resumption(Socket, SessionResumption, wait_reply, no_tickets). + verify_active_session_resumption(Socket, SessionResumption, wait_reply, no_tickets, no_early_data). %% verify_active_session_resumption(Socket, SessionResumption, WaitReply) -> - verify_active_session_resumption(Socket, SessionResumption, WaitReply, no_tickets). + verify_active_session_resumption(Socket, SessionResumption, WaitReply, no_tickets, no_early_data). +%% +verify_active_session_resumption(Socket, SessionResumption, WaitReply, TicketOption) -> + verify_active_session_resumption(Socket, SessionResumption, WaitReply, TicketOption, no_early_data). %% -verify_active_session_resumption(Socket, SessionResumption, WaitForReply, TicketOption) -> +verify_active_session_resumption(Socket, SessionResumption, WaitForReply, TicketOption, EarlyData) -> case ssl:connection_information(Socket, [session_resumption]) of {ok, [{session_resumption, SessionResumption}]} -> Msg = boolean_to_log_msg(SessionResumption), @@ -2551,15 +2642,93 @@ verify_active_session_resumption(Socket, SessionResumption, WaitForReply, Ticket Else1 -> ct:fail("~p:~p~nFaulty parameter: ~p", [?MODULE, ?LINE, Else1]) end, - case TicketOption of - {tickets, N} -> - receive_tickets(N); - no_tickets -> - ok; - Else2 -> - ct:fail("~p:~p~nFaulty parameter: ~p", [?MODULE, ?LINE, Else2]) + Tickets = + case TicketOption of + {tickets, N} -> + receive_tickets(N); + no_tickets -> + ok; + Else2 -> + ct:fail("~p:~p~nFaulty parameter: ~p", [?MODULE, ?LINE, Else2]) + end, + case EarlyData of + {verify_early_data, Atom} -> + case verify_early_data(Atom) of + ok -> + Tickets; + Else -> + ct:fail("~p:~p~nFailed to verify early_data! (expected ~p, got ~p)", + [?MODULE, ?LINE, Atom, Else]) + end; + no_early_data -> + Tickets; + Else3 -> + ct:fail("~p:~p~nFaulty parameter: ~p", [?MODULE, ?LINE, Else3]) end. +verify_server_early_data(Socket, WaitForReply, EarlyData) -> + case ssl:connection_information(Socket, [session_resumption]) of + {ok, [{session_resumption, true}]} -> + Msg = boolean_to_log_msg(true), + ct:log("~p:~p~nSession resumption verified! (expected ~p, got ~p)!", + [?MODULE, ?LINE, Msg, Msg]); + {ok, [{session_resumption, Got0}]} -> + Expected = boolean_to_log_msg(true), + Got = boolean_to_log_msg(Got0), + ct:fail("~p:~p~nFailed to verify session resumption! (expected ~p, got ~p)", + [?MODULE, ?LINE, Expected, Got]); + {error, Reason} -> + ct:fail("~p:~p~nFailed to verify session resumption! Reason: ~p", + [?MODULE, ?LINE, Reason]) + end, + Data = "Hello world", + ssl:send(Socket, Data), + Reply = + case EarlyData of + no_early_data -> + Data; + _ -> + binary_to_list(EarlyData) ++ Data + end, + ct:log("Expected Reply: ~p~n", [Reply]), + case WaitForReply of + wait_reply -> + Reply = active_recv(Socket, length(Reply)); + no_reply -> + ok; + Else1 -> + ct:fail("~p:~p~nFaulty parameter: ~p", [?MODULE, ?LINE, Else1]) + end, + ok. + +verify_session_ticket_extension([Ticket0|_], MaxEarlyDataSize) -> + #{ticket := #new_session_ticket{ + extensions = #{early_data := + #early_data_indication_nst{ + indication = Size}}}} = Ticket0, + case Size of + MaxEarlyDataSize -> + ct:log("~p:~p~nmax_early_data_size verified! (expected ~p, got ~p)!", + [?MODULE, ?LINE, MaxEarlyDataSize, Size]); + Else -> + ct:log("~p:~p~nFailed to verify max_early_data_size! (expected ~p, got ~p)!", + [?MODULE, ?LINE, MaxEarlyDataSize, Else]) + end. + +update_session_ticket_extension([Ticket|_], MaxEarlyDataSize) -> + #{ticket := #new_session_ticket{ + extensions = #{early_data := + #early_data_indication_nst{ + indication = Size}}}} = Ticket, + ct:log("~p:~p~nOverwrite max_early_data_size (from ~p to ~p)!", + [?MODULE, ?LINE, Size, MaxEarlyDataSize]), + #{ticket := #new_session_ticket{ + extensions = #{early_data := Extensions0}} = NST0} = Ticket, + Extensions = #{early_data => #early_data_indication_nst{ + indication = MaxEarlyDataSize}}, + NST = NST0#new_session_ticket{extensions = Extensions}, + [Ticket#{ticket => NST}]. + boolean_to_log_msg(true) -> "OK"; boolean_to_log_msg(false) -> @@ -2572,7 +2741,7 @@ receive_tickets(0, Acc) -> Acc; receive_tickets(N, Acc) -> receive - {ssl, session_ticket, {_, Ticket}} -> + {ssl, session_ticket, Ticket} -> receive_tickets(N - 1, [Ticket|Acc]) end. @@ -2606,7 +2775,8 @@ active_recv(_Socket, N, Acc) when N < 0 -> T; active_recv(Socket, N, Acc) -> receive - {ssl, Socket, Bytes} -> + %% Filter {ssl, Socket, {early_data, Atom}} messages + {ssl, Socket, Bytes} when not is_tuple(Bytes) -> active_recv(Socket, N-data_length(Bytes), Acc ++ Bytes); {Socket, {data, Bytes0}} -> Bytes = filter_openssl_debug_data(Bytes0), @@ -2808,7 +2978,7 @@ check_sane_openssl_renegotiate(Config) -> {skip, "Known renegotiation bug in OpenSSL"}; "LibreSSL 2." ++ _ -> {skip, "Known renegotiation bug in LibreSSL"}; - "LibreSSL 3.1" ++ _ -> + "LibreSSL 3." ++ _ -> {skip, "Known renegotiation bug in LibreSSL"}; _ -> Config @@ -3602,4 +3772,12 @@ default_ciphers(Version) -> ssl:cipher_suites(default, Version) end, [Cipher || Cipher <- Ciphers, lists:member(ssl:suite_to_openssl_str(Cipher), OpenSSLCiphers)]. - + +verify_early_data(Atom) -> + receive + {ssl, _Socket, {early_data, Atom}} -> + ok; + {ssl, _Socket, {early_data, Other}} -> + Other + end. + diff --git a/lib/ssl/test/tls_1_3_record_SUITE.erl b/lib/ssl/test/tls_1_3_record_SUITE.erl index 7a61cf9411..f5f57b534b 100644 --- a/lib/ssl/test/tls_1_3_record_SUITE.erl +++ b/lib/ssl/test/tls_1_3_record_SUITE.erl @@ -90,20 +90,25 @@ encode_decode(_Config) -> client_verify_data => undefined,compression_state => undefined, mac_secret => undefined,secure_renegotiation => undefined, security_parameters => - {security_parameters, - <<19,2>>, - 0,8,2,undefined,undefined,undefined,undefined,undefined, - sha384,undefined,undefined, - {handshake_secret, - <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, - 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, - 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, - 157>>}, undefined, undefined, - undefined, - <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, - 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>, - undefined}, - sequence_number => 0,server_verify_data => undefined}, + #security_parameters{ + cipher_suite = <<19,2>>, + connection_end = 0, + bulk_cipher_algorithm = 8, + cipher_type = 2, + prf_algorithm = sha384, + master_secret = + {handshake_secret, + <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, + 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, + 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, + 157>>}, + server_random = + <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, + 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>}, + sequence_number => 0,server_verify_data => undefined, + max_early_data_size => 0, + trial_decryption => false, + early_data_limit => false}, current_write => #{beast_mitigation => one_n_minus_one, cipher_state => @@ -116,19 +121,21 @@ encode_decode(_Config) -> client_verify_data => undefined,compression_state => undefined, mac_secret => undefined,secure_renegotiation => undefined, security_parameters => - {security_parameters, - <<19,2>>, - 0,8,2,undefined,undefined,undefined,undefined,undefined, - sha384,undefined,undefined, - {handshake_secret, - <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, - 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, - 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, - 157>>}, undefined, undefined, - undefined, - <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, - 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>, - undefined}, + #security_parameters{ + cipher_suite = <<19,2>>, + connection_end = 0, + bulk_cipher_algorithm = 8, + cipher_type = 2, + prf_algorithm = sha384, + master_secret = + {handshake_secret, + <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, + 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, + 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, + 157>>}, + server_random = + <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, + 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>}, sequence_number => 0,server_verify_data => undefined},max_fragment_length => undefined}, PlainText = [11, @@ -544,7 +551,8 @@ encode_decode(_Config) -> %% TODO: remove hardcoded IV size WriteIVInfo = tls_v1:create_info(<<"iv">>, <<>>, 12), - {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, SHSTrafficSecret), + KeyLength = ssl_cipher:key_material(Cipher), + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, SHSTrafficSecret), %% {server} construct an EncryptedExtensions handshake message: %% @@ -824,7 +832,7 @@ encode_decode(_Config) -> SWIV = hexstr2bin("cf 78 2b 88 dd 83 54 9a ad f1 e9 84"), - {SWKey, SWIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, SAPTrafficSecret), + {SWKey, SWIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, SAPTrafficSecret), %% {server} derive read traffic keys for handshake data: %% @@ -849,7 +857,7 @@ encode_decode(_Config) -> SRIV = hexstr2bin("5b d3 c7 1b 83 6e 0b 76 bb 73 26 5f"), - {SRKey, SRIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, CHSTrafficSecret), + {SRKey, SRIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, CHSTrafficSecret), %% {client} calculate finished "tls13 finished": %% @@ -926,7 +934,7 @@ encode_decode(_Config) -> CWIV = hexstr2bin("5b 78 92 3d ee 08 57 90 33 e5 23 d9"), - {CWKey, CWIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, CAPTrafficSecret), + {CWKey, CWIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, KeyLength, CAPTrafficSecret), %% {client} derive secret "tls13 res master": %% diff --git a/lib/ssl/test/tls_1_3_version_SUITE.erl b/lib/ssl/test/tls_1_3_version_SUITE.erl index 3fd1683029..03fcb9afe5 100644 --- a/lib/ssl/test/tls_1_3_version_SUITE.erl +++ b/lib/ssl/test/tls_1_3_version_SUITE.erl @@ -59,25 +59,27 @@ %%-------------------------------------------------------------------- all() -> [ - {group, 'tlsv1.3'} + cert_groups() ]. groups() -> [ - {'tlsv1.3', [], cert_groups()}, - {rsa, [], tests()}, - {ecdsa, [], tests()} + {rsa, [], tls_1_3_1_2_tests() ++ legacy_tests()}, + {ecdsa, [], tls_1_3_1_2_tests()} ]. cert_groups() -> [{group, rsa}, {group, ecdsa}]. -tests() -> +tls_1_3_1_2_tests() -> [tls13_client_tls12_server, tls13_client_with_ext_tls12_server, tls12_client_tls13_server, - tls_client_tls10_server, + tls_client_tls12_server, + tls12_client_tls_server]. +legacy_tests() -> + [tls_client_tls10_server, tls_client_tls11_server, tls_client_tls12_server, tls10_client_tls_server, @@ -88,9 +90,14 @@ init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - [{client_type, erlang}, {server_type, erlang} | - Config] + case ssl_test_lib:sufficient_crypto_support('tlsv1.3') of + true -> + ssl_test_lib:clean_start(), + [{client_type, erlang}, {server_type, erlang} | + Config]; + false -> + {skip, "Insufficient crypto support for TLS-1.3"} + end catch _:_ -> {skip, "Crypto did not start"} end. @@ -99,23 +106,14 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -init_per_group(GroupName, Config) -> - case ssl_test_lib:is_protocol_version(GroupName) of - true -> - ssl_test_lib:init_per_group(GroupName, - [{client_type, erlang}, - {server_type, erlang} | Config]); - false -> - do_init_per_group(GroupName, Config) - end. - -do_init_per_group(rsa, Config0) -> +init_per_group(rsa, Config0) -> Config = ssl_test_lib:make_rsa_cert(Config0), COpts = proplists:get_value(client_rsa_opts, Config), SOpts = proplists:get_value(server_rsa_opts, Config), - [{client_cert_opts, COpts}, {server_cert_opts, SOpts} | + [{client_type, erlang}, + {server_type, erlang},{client_cert_opts, COpts}, {server_cert_opts, SOpts} | lists:delete(server_cert_opts, lists:delete(client_cert_opts, Config))]; -do_init_per_group(ecdsa, Config0) -> +init_per_group(ecdsa, Config0) -> PKAlg = crypto:supports(public_keys), case lists:member(ecdsa, PKAlg) andalso (lists:member(ecdh, PKAlg) orelse lists:member(dh, PKAlg)) of @@ -123,7 +121,8 @@ do_init_per_group(ecdsa, Config0) -> Config = ssl_test_lib:make_ecdsa_cert(Config0), COpts = proplists:get_value(client_ecdsa_opts, Config), SOpts = proplists:get_value(server_ecdsa_opts, Config), - [{client_cert_opts, COpts}, {server_cert_opts, SOpts} | + [{client_type, erlang}, + {server_type, erlang},{client_cert_opts, COpts}, {server_cert_opts, SOpts} | lists:delete(server_cert_opts, lists:delete(client_cert_opts, Config))]; false -> {skip, "Missing EC crypto support"} @@ -175,21 +174,38 @@ tls12_client_tls13_server(Config) when is_list(Config) -> tls_client_tls10_server() -> [{doc,"Test that a TLS 1.0-1.3 client can connect to a TLS 1.0 server."}]. tls_client_tls10_server(Config) when is_list(Config) -> + CCiphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, 'tlsv1.3'), + [{key_exchange, fun(srp_rsa) -> false; + (srp_anon) -> false; + (srp_dss) -> false; + (_) -> true end}]), ClientOpts = [{versions, - ['tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3']} | + ['tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3']}, + {ciphers, CCiphers} + | ssl_test_lib:ssl_options(client_cert_opts, Config)], ServerOpts = [{versions, - ['tlsv1']} | ssl_test_lib:ssl_options(server_cert_opts, Config)], + ['tlsv1']}, + {ciphers, ssl:cipher_suites(all, 'tlsv1')} + | ssl_test_lib:ssl_options(server_cert_opts, Config)], ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). tls_client_tls11_server() -> [{doc,"Test that a TLS 1.0-1.3 client can connect to a TLS 1.1 server."}]. tls_client_tls11_server(Config) when is_list(Config) -> + CCiphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, 'tlsv1.3'), + [{key_exchange, fun(srp_rsa) -> false; + (srp_anon) -> false; + (srp_dss) -> false; + (_) -> true end}]), ClientOpts = [{versions, - ['tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3']} | + ['tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3']}, + {ciphers, CCiphers} | ssl_test_lib:ssl_options(client_cert_opts, Config)], ServerOpts = [{versions, - ['tlsv1.1']} | ssl_test_lib:ssl_options(server_cert_opts, Config)], + ['tlsv1.1']}, + {ciphers, ssl:cipher_suites(all, 'tlsv1.1')} + | ssl_test_lib:ssl_options(server_cert_opts, Config)], ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). tls_client_tls12_server() -> @@ -205,20 +221,36 @@ tls_client_tls12_server(Config) when is_list(Config) -> tls10_client_tls_server() -> [{doc,"Test that a TLS 1.0 client can connect to a TLS 1.0-1.3 server."}]. tls10_client_tls_server(Config) when is_list(Config) -> + SCiphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, 'tlsv1.3'), + [{key_exchange, fun(srp_rsa) -> false; + (srp_anon) -> false; + (srp_dss) -> false; + (_) -> true end}]), ClientOpts = [{versions, - ['tlsv1']} | ssl_test_lib:ssl_options(client_cert_opts, Config)], + ['tlsv1']}, {ciphers, ssl:cipher_suites(all, 'tlsv1')} | ssl_test_lib:ssl_options(client_cert_opts, Config)], ServerOpts = [{versions, - ['tlsv1','tlsv1.1', 'tlsv1.2', 'tlsv1.3']} | + ['tlsv1','tlsv1.1', 'tlsv1.2', 'tlsv1.3']}, + {ciphers, SCiphers} + | ssl_test_lib:ssl_options(server_cert_opts, Config)], ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). tls11_client_tls_server() -> [{doc,"Test that a TLS 1.1 client can connect to a TLS 1.0-1.3 server."}]. tls11_client_tls_server(Config) when is_list(Config) -> + SCiphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, 'tlsv1.3'), + [{key_exchange, fun(srp_rsa) -> false; + (srp_anon) -> false; + (srp_dss) -> false; + (_) -> true end}]), + ClientOpts = [{versions, - ['tlsv1.1']} | ssl_test_lib:ssl_options(client_cert_opts, Config)], + ['tlsv1.1']}, {ciphers, ssl:cipher_suites(all, 'tlsv1.1')} | + ssl_test_lib:ssl_options(client_cert_opts, Config)], ServerOpts = [{versions, - ['tlsv1','tlsv1.1', 'tlsv1.2', 'tlsv1.3']} | + ['tlsv1','tlsv1.1', 'tlsv1.2', 'tlsv1.3']}, + {ciphers, SCiphers} + | ssl_test_lib:ssl_options(server_cert_opts, Config)], ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index b010a74f35..2b9e7f5e6b 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 10.2 +SSL_VSN = 10.2.3 diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 8c7e27fc5b..dd7a2c2cc1 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -1535,7 +1535,13 @@ tokens({cons,A,Head,Tail}, More) -> tokens({tuple,A,[]}, More) -> [{'{',A},{'}',A}|More]; tokens({tuple,A,[E|Es]}, More) -> - [{'{',A}|tokens(E, tokens_tuple(Es, ?anno(E), More))]. + [{'{',A}|tokens(E, tokens_tuple(Es, ?anno(E), More))]; +tokens({map,A,[]}, More) -> + [{'#',A},{'{',A},{'}',A}|More]; +tokens({map,A,[P|Ps]}, More) -> + [{'#',A},{'{',A}|tokens(P, tokens_tuple(Ps, ?anno(P), More))]; +tokens({map_field_assoc,A,K,V}, More) -> + tokens(K, [{'=>',A}|tokens(V, More)]). tokens_tail({cons,A,Head,Tail}, More) -> [{',',A}|tokens(Head, tokens_tail(Tail, More))]; diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 1eca1c29c3..1cc11d9093 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -65,16 +65,6 @@ obsolete(crypto, stream_decrypt, 2) -> {deprecated, "use crypto:crypto_update/2 instead", "OTP 24"}; obsolete(crypto, stream_encrypt, 2) -> {deprecated, "use crypto:crypto_update/2 instead", "OTP 24"}; -obsolete(erl_tidy, dir, 0) -> - {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; -obsolete(erl_tidy, dir, 1) -> - {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; -obsolete(erl_tidy, file, 1) -> - {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; -obsolete(erl_tidy, module, 1) -> - {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; -obsolete(erl_tidy, module, 2) -> - {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; obsolete(erlang, get_stacktrace, 0) -> {deprecated, "use the new try/catch syntax for retrieving the stack backtrace", "OTP 24"}; obsolete(erlang, now, 0) -> @@ -497,6 +487,8 @@ obsolete(erl_scan, attributes_info, _) -> {removed, "erl_anno:{column,line,location,text}/1 instead"}; obsolete(erl_scan, token_info, _) -> {removed, "erl_scan:{category,column,line,location,symbol,text}/1 instead"}; +obsolete(erl_tidy, _, _) -> + {deprecated, "use https://github.com/richcarl/erl_tidy", "OTP 24"}; obsolete(gen_fsm, _, _) -> {deprecated, "use the 'gen_statem' module instead"}; obsolete(igor, _, _) -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 041a89f909..b397b2fc36 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -1186,6 +1186,8 @@ record_bindings(Recs0, Bs0) -> read_records(FileOrModule, Opts0) -> Opts = lists:delete(report_warnings, Opts0), case find_file(FileOrModule) of + {beam, Beam, File} -> + read_records_from_beam(Beam, File); {files,[File]} -> read_file_records(File, Opts); {files,Files} -> @@ -1204,10 +1206,22 @@ read_records(FileOrModule, Opts0) -> find_file(Mod) when is_atom(Mod) -> case code:which(Mod) of File when is_list(File) -> - {files,[File]}; - preloaded -> - {_M,_Bin,File} = code:get_object_code(Mod), - {files,[File]}; + %% Special cases: + %% - Modules not in the code path (loaded with code:load_abs/1): + %% code:get_object_code/1 only searches in the code path + %% but code:which/1 finds all loaded modules + %% - File can also be a file in an archive, + %% beam_lib:chunks/2 cannot handle such paths but + %% erl_prim_loader:get_file/1 can + case erl_prim_loader:get_file(File) of + {ok, Beam, _} -> + {beam, Beam, File}; + error -> + {error, nofile} + end; + preloaded -> + {_M, Beam, File} = code:get_object_code(Mod), + {beam, Beam, File}; _Else -> % non_existing, interpreted, cover_compiled {error,nofile} end; @@ -1222,28 +1236,31 @@ find_file(File) -> read_file_records(File, Opts) -> case filename:extension(File) of ".beam" -> - case beam_lib:chunks(File, [abstract_code,"CInf"]) of - {ok,{_Mod,[{abstract_code,{Version,Forms}},{"CInf",CB}]}} -> - case record_attrs(Forms) of - [] when Version =:= raw_abstract_v1 -> - []; - [] -> - %% If the version is raw_X, then this test - %% is unnecessary. - try_source(File, CB); - Records -> - Records - end; - {ok,{_Mod,[{abstract_code,no_abstract_code},{"CInf",CB}]}} -> - try_source(File, CB); - Error -> - %% Could be that the "Abst" chunk is missing (pre R6). - Error - end; + read_records_from_beam(File, File); _ -> parse_file(File, Opts) end. +read_records_from_beam(Beam, File) -> + case beam_lib:chunks(Beam, [abstract_code,"CInf"]) of + {ok,{_Mod,[{abstract_code,{Version,Forms}},{"CInf",CB}]}} -> + case record_attrs(Forms) of + [] when Version =:= raw_abstract_v1 -> + []; + [] -> + %% If the version is raw_X, then this test + %% is unnecessary. + try_source(File, CB); + Records -> + Records + end; + {ok,{_Mod,[{abstract_code,no_abstract_code},{"CInf",CB}]}} -> + try_source(File, CB); + Error -> + %% Could be that the "Abst" chunk is missing (pre R6). + Error + end. + %% This is how the debugger searches for source files. See int.erl. try_source(Beam, RawCB) -> EbinDir = filename:dirname(Beam), diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl index cb1638670c..a607598136 100644 --- a/lib/stdlib/test/epp_SUITE.erl +++ b/lib/stdlib/test/epp_SUITE.erl @@ -29,7 +29,7 @@ otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1, otp_11728/1, encoding/1, extends/1, function_macro/1, test_error/1, test_warning/1, otp_14285/1, - test_if/1,source_name/1]). + test_if/1,source_name/1,otp_16978/1]). -export([epp_parse_erl_form/2]). @@ -70,7 +70,7 @@ all() -> overload_mac, otp_8388, otp_8470, otp_8562, otp_8665, otp_8911, otp_10302, otp_10820, otp_11728, encoding, extends, function_macro, test_error, test_warning, - otp_14285, test_if, source_name]. + otp_14285, test_if, source_name, otp_16978]. groups() -> [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]}, @@ -1720,19 +1720,40 @@ source_name_1(File, Expected) -> Res = epp:parse_file(File, [{source_name, Expected}]), {ok, [{attribute,_,file,{Expected,_}} | _Forms]} = Res. +otp_16978(Config) when is_list(Config) -> + %% A test of erl_parse:tokens(). + P = <<"t() -> ?a.">>, + Vs = [#{}, + #{k => 1,[[a],[{}]] => "str"}, + #{#{} => [{#{x=>#{3=>$3}}},{3.14,#{}}]}], + Ts = [{erl_parse_tokens, + P, + [{d,{a,V}}], + V} || V <- Vs], + [] = run(Config, Ts), + + ok. + check(Config, Tests) -> - eval_tests(Config, fun check_test/2, Tests). + eval_tests(Config, fun check_test/3, Tests). compile(Config, Tests) -> - eval_tests(Config, fun compile_test/2, Tests). + eval_tests(Config, fun compile_test/3, Tests). run(Config, Tests) -> - eval_tests(Config, fun run_test/2, Tests). + eval_tests(Config, fun run_test/3, Tests). eval_tests(Config, Fun, Tests) -> - F = fun({N,P,E}, BadL) -> + TestsWithOpts = + [case Test of + {N,P,E} -> + {N,P,[],E}; + {_,_,_,_} -> + Test + end || Test <- Tests], + F = fun({N,P,Opts,E}, BadL) -> %% io:format("Testing ~p~n", [P]), - Return = Fun(Config, P), + Return = Fun(Config, P, Opts), case message_compare(E, Return) of true -> case E of @@ -1748,14 +1769,14 @@ eval_tests(Config, Fun, Tests) -> fail() end end, - lists:foldl(F, [], Tests). + lists:foldl(F, [], TestsWithOpts). -check_test(Config, Test) -> +check_test(Config, Test, Opts) -> Filename = "epp_test.erl", PrivDir = proplists:get_value(priv_dir, Config), File = filename:join(PrivDir, Filename), ok = file:write_file(File, Test), - case epp:parse_file(File, [PrivDir], []) of + case epp:parse_file(File, [PrivDir], Opts) of {ok,Forms} -> Errors = [E || E={error,_} <- Forms], call_format_error([E || {error,E} <- Errors]), @@ -1764,13 +1785,13 @@ check_test(Config, Test) -> Error end. -compile_test(Config, Test0) -> +compile_test(Config, Test0, Opts0) -> Test = [<<"-module(epp_test). ">>, Test0], Filename = "epp_test.erl", PrivDir = proplists:get_value(priv_dir, Config), File = filename:join(PrivDir, Filename), ok = file:write_file(File, Test), - Opts = [export_all,nowarn_export_all,return,nowarn_unused_record,{outdir,PrivDir}], + Opts = [export_all,nowarn_export_all,return,nowarn_unused_record,{outdir,PrivDir}] ++ Opts0, case compile_file(File, Opts) of {ok, Ws} -> warnings(File, Ws); {errors, Errors}=Else -> @@ -1821,13 +1842,13 @@ epp_parse_file(File, Opts) -> unopaque_forms(Forms) -> [erl_parse:anno_to_term(Form) || Form <- Forms]. -run_test(Config, Test0) -> +run_test(Config, Test0, Opts0) -> Test = [<<"-module(epp_test). -export([t/0]). ">>, Test0], Filename = "epp_test.erl", PrivDir = proplists:get_value(priv_dir, Config), File = filename:join(PrivDir, Filename), ok = file:write_file(File, Test), - Opts = [return, {i,PrivDir},{outdir,PrivDir}], + Opts = [return, {i,PrivDir},{outdir,PrivDir}] ++ Opts0, {ok, epp_test, []} = compile:file(File, Opts), AbsFile = filename:rootname(File, ".erl"), {module, epp_test} = code:load_abs(AbsFile, epp_test), diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 4d7a2ea078..4df0a2238a 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -64,8 +64,9 @@ end_per_testcase(_Case, Config) -> OrigPath = proplists:get_value(orig_path,Config), code:set_path(OrigPath), application:unset_env(stdlib, restricted_shell), - (catch code:purge(user_default)), - (catch code:delete(user_default)), + purge_and_delete(user_default), + %% used by `records' test case + purge_and_delete(test), ok. -endif. @@ -298,8 +299,7 @@ restricted_local(Config) when is_list(Config) -> comm_err(<<"begin shell:stop_restricted() end.">>), undefined = application:get_env(stdlib, restricted_shell), - (catch code:purge(user_default)), - true = (catch code:delete(user_default)), + true = purge_and_delete(user_default), ok. @@ -428,6 +428,30 @@ records(Config) when is_list(Config) -> [{error,invalid_filename}] = scan(<<"rr({foo}).">>), [[]] = scan(<<"rr(\"not_a_file\").">>), + %% load record from archive + true = purge_and_delete(test), + + PrivDir = proplists:get_value(priv_dir, Config), + AppDir = filename:join(PrivDir, "test_app"), + ok = file:make_dir(AppDir), + AppEbinDir = filename:join(AppDir, "ebin"), + ok = file:make_dir(AppEbinDir), + + ok = file:write_file(Test, Contents), + {ok, test} = compile:file(Test, [{outdir, AppEbinDir}]), + + Ext = init:archive_extension(), + Archive = filename:join(PrivDir, "test_app" ++ Ext), + {ok, _} = zip:create(Archive, ["test_app"], [{compress, []}, {cwd, PrivDir}]), + + ArchiveEbinDir = filename:join(Archive, "test_app/ebin"), + true = code:add_path(ArchiveEbinDir), + {module, test} = code:load_file(test), + BeamInArchive = filename:join(ArchiveEbinDir, "test.beam"), + BeamInArchive = code:which(test), + + [[state]] = scan(<<"rr(test).">>), + %% using records [2] = scan(<<"rd(foo,{bar}), record_info(size, foo).">>), [true] = scan(<<"rd(foo,{bar}), is_record(#foo{}, foo).">>), @@ -3218,3 +3242,6 @@ start_node(Name, Xargs) -> global:sync(), N. +purge_and_delete(Module) -> + (catch code:purge(Module)), + (catch code:delete(Module)). diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl index fb312bb9d4..f6ae2cb960 100644 --- a/lib/syntax_tools/src/erl_tidy.erl +++ b/lib/syntax_tools/src/erl_tidy.erl @@ -48,11 +48,7 @@ %% @type filename() = file:filename(). -module(erl_tidy). --deprecated([{dir,0,"use https://github.com/richcarl/erl_tidy"}]). --deprecated([{dir,1,"use https://github.com/richcarl/erl_tidy"}]). --deprecated([{file,1,"use https://github.com/richcarl/erl_tidy"}]). --deprecated([{module,1,"use https://github.com/richcarl/erl_tidy"}]). --deprecated([{module,2,"use https://github.com/richcarl/erl_tidy"}]). +-deprecated([{'_','_',"use https://github.com/richcarl/erl_tidy"}]). -export([dir/0, dir/1, dir/2, file/1, file/2, module/1, module/2]). diff --git a/lib/tools/doc/src/fprof.xml b/lib/tools/doc/src/fprof.xml index 5d2683846f..b3ba4a200c 100644 --- a/lib/tools/doc/src/fprof.xml +++ b/lib/tools/doc/src/fprof.xml @@ -598,7 +598,7 @@ -module(foo). -export([create_file_slow/2]). -create_file_slow(Name, N) when integer(N), N >= 0 -> +create_file_slow(Name, N) when is_integer(N), N >= 0 -> {ok, FD} = file:open(Name, [raw, write, delayed_write, binary]), if N > 256 -> diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 065d3aeebb..5b98d338b0 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,22 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 3.4.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Correct the Xref analysis <c>undefined_functions</c> to + not report internally generated behaviour_info/1.</p> + <p> + Own Id: OTP-17191 Aux Id: OTP-16922, ERL-1476, GH-4192 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/doc/src/xref.xml b/lib/tools/doc/src/xref.xml index 33b8003320..72cc5c6c39 100644 --- a/lib/tools/doc/src/xref.xml +++ b/lib/tools/doc/src/xref.xml @@ -885,7 +885,9 @@ Evaluates a predefined analysis. locally used.</item> <tag><c>exports_not_used</c></tag> <item>Returns a list of exported functions that have not been - externally used.</item> + externally used. Note that in <c>modules</c> mode, + <c>M:behaviour_info/1</c> is never reported as unused. + </item> <tag><c>deprecated_function_calls</c>(*)</tag> <item>Returns a list of external calls to <seeerl marker="#deprecated_function">deprecated functions</seeerl>.</item> <tag><c>{deprecated_function_calls, DeprFlag}</c>(*)</tag> diff --git a/lib/tools/src/xref_reader.erl b/lib/tools/src/xref_reader.erl index 87c02db9eb..9857e8c150 100644 --- a/lib/tools/src/xref_reader.erl +++ b/lib/tools/src/xref_reader.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2017. All Rights Reserved. +%% Copyright Ericsson AB 2000-2020. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -53,7 +53,8 @@ %% R8: abstract_v2 %% R9C: raw_abstract_v1 -%% -> {ok, Module, {DefAt, CallAt, LC, XC, X, Attrs}, Unresolved}} | EXIT +%% -> {ok, Module, {DefAt, LCallAt, XCallAt, LC, XC, X, Attrs, Depr, OL}, +%% Unresolved}} | EXIT %% Attrs = {ALC, AXC, Bad} %% ALC, AXC and Bad are extracted from the attribute 'xref'. An experiment. module(Module, Forms, CollectBuiltins, X, DF) -> @@ -95,16 +96,19 @@ form({function, _, module_info, 0, _Clauses}, S) -> S; form({function, _, module_info, 1, _Clauses}, S) -> S; -form({function, 0 = _Line, behaviour_info, 1, _Clauses}, S) -> - S; form({function, Anno, Name, Arity, Clauses}, S) -> - MFA0 = {S#xrefr.module, Name, Arity}, - MFA = adjust_arity(S, MFA0), - S1 = S#xrefr{function = MFA}, - Line = erl_anno:line(Anno), - S2 = S1#xrefr{def_at = [{MFA,Line} | S#xrefr.def_at]}, - S3 = clauses(Clauses, S2), - S3#xrefr{function = []}; + case {Name, Arity, erl_anno:location(Anno)} of + {behaviour_info, 1, 0} -> + S; % generated + _ -> + MFA0 = {S#xrefr.module, Name, Arity}, + MFA = adjust_arity(S, MFA0), + S1 = S#xrefr{function = MFA}, + Line = erl_anno:line(Anno), + S2 = S1#xrefr{def_at = [{MFA,Line} | S#xrefr.def_at]}, + S3 = clauses(Clauses, S2), + S3#xrefr{function = []} + end; form(_, S) -> %% OTP 20. Other uninteresting forms such as {eof, _} and {warning, _}. %% Exposed because sys_pre_expand is no longer run. diff --git a/lib/tools/src/xref_utils.erl b/lib/tools/src/xref_utils.erl index eca751337b..c516f272cb 100644 --- a/lib/tools/src/xref_utils.erl +++ b/lib/tools/src/xref_utils.erl @@ -327,7 +327,7 @@ list_dirs([], _I, _Exts, C, E) -> %% Returns functions that are present in all modules. predefined_functions() -> - [{module_info,0}, {module_info,1}]. + [{module_info,0}, {module_info,1}, {behaviour_info,1}]. %% Returns true if an MFA takes functional arguments. is_funfun(erlang, apply, 2) -> true; diff --git a/lib/tools/test/instrument_SUITE.erl b/lib/tools/test/instrument_SUITE.erl index 0708251f10..41b7b11cd2 100644 --- a/lib/tools/test/instrument_SUITE.erl +++ b/lib/tools/test/instrument_SUITE.erl @@ -19,7 +19,7 @@ %% -module(instrument_SUITE). --export([all/0, suite/0]). +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). -export([allocations_enabled/1, allocations_disabled/1, allocations_ramv/1, carriers_enabled/1, carriers_disabled/1]). @@ -37,6 +37,19 @@ all() -> [allocations_enabled, allocations_disabled, allocations_ramv, carriers_enabled, carriers_disabled]. +init_per_suite(Config) -> + case test_server:is_asan() of + true -> + %% No point testing own allocators under address sanitizer. + {skip, "Address sanitizer"}; + false -> + Config + end. + +end_per_suite(_Config) -> + ok. + + -define(GENERATED_SBC_BLOCK_COUNT, 1000). -define(GENERATED_MBC_BLOCK_COUNT, ?GENERATED_SBC_BLOCK_COUNT). diff --git a/lib/tools/test/xref_SUITE.erl b/lib/tools/test/xref_SUITE.erl index b5b3ff7796..86dcb3c94d 100644 --- a/lib/tools/test/xref_SUITE.erl +++ b/lib/tools/test/xref_SUITE.erl @@ -49,7 +49,7 @@ fun_mfa_vars/1, qlc/1]). -export([analyze/1, basic/1, md/1, q/1, variables/1, unused_locals/1, - behaviour_info_t/1, fake_behaviour_info_t/1]). + behaviour/1]). -export([format_error/1, otp_7423/1, otp_7831/1, otp_10192/1, otp_13708/1, otp_14464/1, otp_14344/1]). @@ -84,7 +84,7 @@ groups() -> fun_mfa_r14, fun_mfa_vars, qlc]}, {analyses, [], - [analyze, basic, md, q, variables, unused_locals, behaviour_info_t, fake_behaviour_info_t]}, + [analyze, basic, md, q, variables, unused_locals, behaviour]}, {misc, [], [format_error, otp_7423, otp_7831, otp_10192, otp_13708, otp_14464, otp_14344]}]. @@ -2472,6 +2472,84 @@ otp_14344(Conf) when is_list(Conf) -> ok = file:delete(File1), ok = file:delete(Beam1). +%% PR-2752, ERL-1353, ERL-1476, GH-4192. +behaviour(Config) -> + ModMode = [{xref_mode, modules}], + FunMode = [], + + Test1 = [{a, <<"-module(a). + -callback a() -> ok. + ">>}], + {Undef1, UnusedExports1} = behaviour_test(Test1, Config, FunMode), + [] = Undef1, + [] = UnusedExports1, + {Undef1m, UnusedExports1m} = behaviour_test(Test1, Config, ModMode), + [] = Undef1m, + [] = UnusedExports1m, + + Test2 = [{a, <<"-module(a). + -export([behaviour_info/1]). + behaviour_info(_) -> + ok. + ">>}], + {Undef2, UnusedExports2} = behaviour_test(Test2, Config, FunMode), + [] = Undef2, + [{a,behaviour_info,1}] = UnusedExports2, + {Undef2m, UnusedExports2m} = behaviour_test(Test2, Config, ModMode), + [] = Undef2m, + %% Without abstract code it is not possible to determine if + %% M:behaviour_info/1 is generated or not. The best we can do is + %% to assume it is generated since it would otherwise always be + %% returned as unused. + [] = UnusedExports2m, + + Test3 = [{a, <<"-module(a). + -export([behaviour_info/1]). + behaviour_info(_) -> + ok. + ">>}, + {b, <<"-module(b). + -export([bar/0]). + bar() -> a:behaviour_info(callbacks). + ">>}], + {Undef3, UnusedExports3} = behaviour_test(Test3, Config, FunMode), + [] = Undef3, + [{b,bar,0}] = UnusedExports3, + {Undef3m, UnusedExports3m} = behaviour_test(Test3, Config, ModMode), + [] = Undef3m, + [{b,bar,0}] = UnusedExports3m, + ok. + +behaviour_test(Tests, Conf, Opts) -> + {ok, _} = xref:start(s, Opts), + add_modules(Tests, Conf), + case lists:keyfind(xref_mode, 1, Opts) of + {xref_mode, modules} -> + UndefinedFunctionCalls = []; + _ -> + {ok, UndefinedFunctionCalls} = + xref:analyze(s, undefined_function_calls) + end, + {ok, ExportsNotUsed} = xref:analyze(s, exports_not_used), + xref:stop(s), + {UndefinedFunctionCalls, ExportsNotUsed}. + +add_modules([], _Conf) -> + ok; +add_modules([{Mod, Test} |Tests], Conf) -> + Dir = ?copydir, + Name = atom_to_list(Mod), + File = fname(Dir, Name ++ ".erl"), + MFile = fname(Dir, Name), + Beam = fname(Dir, Name ++ ".beam"), + ok = file:write_file(File, Test), + {ok, Mod} = compile:file(File, [debug_info,{outdir,Dir}]), + {ok, Mod} = xref:add_module(s, MFile), + check_state(s), + ok = file:delete(File), + ok = file:delete(Beam), + add_modules(Tests, Conf). + %%% %%% Utilities %%% @@ -2826,24 +2904,3 @@ add_erts_code_path(KernelPath) -> [KernelPath] end end. - -behaviour_info_t(Config) -> - bi_t(_Module = bi, - _IsExportNotUsed = false, - Config). - -fake_behaviour_info_t(Config) -> - bi_t(_Module = no_bi, - _IsExportNotUsed = true, - Config). - -bi_t(Module, IsExportNotUsed, Conf) -> - LibTestDir = fname(?copydir, "lib_test"), - XRefServer = s, - {ok, Module} = compile:file(fname(LibTestDir, Module), - [debug_info, {outdir, LibTestDir}]), - {ok, _} = start(XRefServer), - {ok, Module} = xref:add_module(XRefServer, fname(LibTestDir, Module)), - {ok, MFAs} = xref:analyze(XRefServer, exports_not_used), - true = lists:member({Module, behaviour_info, 1}, MFAs) =:= IsExportNotUsed, - _ = xref:stop(XRefServer). diff --git a/lib/tools/test/xref_SUITE_data/lib_test/bi.erl b/lib/tools/test/xref_SUITE_data/lib_test/bi.erl deleted file mode 100644 index e083fa0f3c..0000000000 --- a/lib/tools/test/xref_SUITE_data/lib_test/bi.erl +++ /dev/null @@ -1,3 +0,0 @@ --module(bi). - --callback a() -> ok. diff --git a/lib/tools/test/xref_SUITE_data/lib_test/no_bi.erl b/lib/tools/test/xref_SUITE_data/lib_test/no_bi.erl deleted file mode 100644 index 5aac4a193e..0000000000 --- a/lib/tools/test/xref_SUITE_data/lib_test/no_bi.erl +++ /dev/null @@ -1,6 +0,0 @@ --module(no_bi). - --export([behaviour_info/1]). - -behaviour_info(_) -> - ok. diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 27ac75cd3f..6e4ee2fd14 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 3.4.2 +TOOLS_VSN = 3.4.3 diff --git a/lib/wx/c_src/wxe_ps_init.c b/lib/wx/c_src/wxe_ps_init.c index d82d142967..5d19502fbe 100644 --- a/lib/wx/c_src/wxe_ps_init.c +++ b/lib/wx/c_src/wxe_ps_init.c @@ -29,19 +29,8 @@ extern OSErr CPSSetProcessName (ProcessSerialNumber *psn, char *processname); -void * wxe_ps_init() +void * wxe_ps_init() { - ProcessSerialNumber psn; - // Enable GUI - if(!GetCurrentProcess(&psn)) { - TransformProcessType(&psn, kProcessTransformToForegroundApplication); -#ifdef MAC_OS_X_VERSION_10_6 - [[NSRunningApplication currentApplication] activateWithOptions: - (NSApplicationActivateAllWindows | NSApplicationActivateIgnoringOtherApps)]; -#else - SetFrontProcess(&psn); -#endif - } return (void *) 0; } @@ -66,7 +55,6 @@ void * wxe_ps_init2() { char * app_title; size_t app_icon_len = 1023; char app_icon_buf[1024]; - char * app_icon; // Setup and enable gui pool = [[NSAutoreleasePool alloc] init]; diff --git a/make/otp_subdir.mk b/make/otp_subdir.mk index 19c744955c..f9b993e048 100644 --- a/make/otp_subdir.mk +++ b/make/otp_subdir.mk @@ -20,12 +20,12 @@ # Make include file for otp .PHONY: debug opt lcnt release docs release_docs tests release_tests \ - clean depend valgrind static_lib + clean depend valgrind asan static_lib # # Targets that don't affect documentation directories # -opt debug lcnt release docs release_docs tests release_tests clean depend valgrind static_lib xmllint: +opt debug lcnt release docs release_docs tests release_tests clean depend valgrind asan static_lib xmllint: @set -e ; \ app_pwd=`pwd` ; \ if test -f vsn.mk; then \ diff --git a/make/otp_version_tickets_in_merge b/make/otp_version_tickets_in_merge index 2bbd687fa7..e69de29bb2 100644 --- a/make/otp_version_tickets_in_merge +++ b/make/otp_version_tickets_in_merge @@ -1,2 +0,0 @@ -OTP-16607 -OTP-17083 diff --git a/make/run_make.mk b/make/run_make.mk index bcbbf53f7d..087129866d 100644 --- a/make/run_make.mk +++ b/make/run_make.mk @@ -29,9 +29,9 @@ include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk -.PHONY: valgrind +.PHONY: valgrind asan -opt debug purify quantify purecov valgrind gcov gprof lcnt frmptr icount: +opt debug purify quantify purecov valgrind asan gcov gprof lcnt frmptr icount: $(make_verbose)$(MAKE) -f $(TARGET)/Makefile TYPE=$@ plain smp frag smp_frag: diff --git a/otp_versions.table b/otp_versions.table index 2170ca7847..a8d7c92f1e 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,7 @@ +OTP-23.2.5 : erts-11.1.8 ssl-10.2.3 tools-3.4.3 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7.3 ssh-4.10.7 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 wx-1.9.2 xmerl-1.3.26 : +OTP-23.2.4 : snmp-5.7.3 ssl-10.2.2 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.3 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 erts-11.1.7 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 ssh-4.10.7 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.2 wx-1.9.2 xmerl-1.3.26 : +OTP-23.2.3 : crypto-4.8.3 erts-11.1.7 snmp-5.7.2 ssh-4.10.7 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 megaco-3.19.5 mnesia-4.18.1 observer-2.9.5 odbc-2.13.2 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 ssl-10.2.1 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.2 wx-1.9.2 xmerl-1.3.26 : +OTP-23.2.2 : crypto-4.8.2 erl_interface-4.0.2 erts-11.1.6 megaco-3.19.5 odbc-2.13.2 snmp-5.7.1 ssl-10.2.1 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 mnesia-4.18.1 observer-2.9.5 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 ssh-4.10.6 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.2 wx-1.9.2 xmerl-1.3.26 : OTP-23.2.1 : erts-11.1.5 # asn1-5.0.14 common_test-1.19.1 compiler-7.6.6 crypto-4.8.1 debugger-5.0 dialyzer-4.3 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.2 erl_interface-4.0.1 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3.1 jinterface-1.11 kernel-7.2 megaco-3.19.4 mnesia-4.18.1 observer-2.9.5 odbc-2.13.1 os_mon-2.6.1 parsetools-2.2 public_key-1.9.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.7 ssh-4.10.6 ssl-10.2 stdlib-3.14 syntax_tools-2.4 tftp-1.0.2 tools-3.4.2 wx-1.9.2 xmerl-1.3.26 : OTP-23.2 : common_test-1.19.1 compiler-7.6.6 crypto-4.8.1 dialyzer-4.3 erl_docgen-1.0.2 erts-11.1.4 inets-7.3.1 kernel-7.2 megaco-3.19.4 mnesia-4.18.1 public_key-1.9.2 snmp-5.7 ssh-4.10.6 ssl-10.2 stdlib-3.14 syntax_tools-2.4 tools-3.4.2 wx-1.9.2 xmerl-1.3.26 # asn1-5.0.14 debugger-5.0 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_interface-4.0.1 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 jinterface-1.11 observer-2.9.5 odbc-2.13.1 os_mon-2.6.1 parsetools-2.2 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 tftp-1.0.2 : OTP-23.1.5 : ssh-4.10.5 # asn1-5.0.14 common_test-1.19 compiler-7.6.5 crypto-4.8 debugger-5.0 dialyzer-4.2.1 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0.1 erl_interface-4.0.1 erts-11.1.3 et-1.6.4 eunit-2.6 ftp-1.0.5 hipe-4.0.1 inets-7.3 jinterface-1.11 kernel-7.1 megaco-3.19.3 mnesia-4.18 observer-2.9.5 odbc-2.13.1 os_mon-2.6.1 parsetools-2.2 public_key-1.9.1 reltool-0.8 runtime_tools-1.15.1 sasl-4.0.1 snmp-5.6.1 ssl-10.1 stdlib-3.13.2 syntax_tools-2.3.1 tftp-1.0.2 tools-3.4.1 wx-1.9.1 xmerl-1.3.25 : @@ -11,6 +15,9 @@ OTP-23.0.3 : compiler-7.6.2 erts-11.0.3 # asn1-5.0.13 common_test-1.19 crypto-4. OTP-23.0.2 : erts-11.0.2 megaco-3.19.1 # asn1-5.0.13 common_test-1.19 compiler-7.6.1 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0.1 : compiler-7.6.1 erts-11.0.1 # asn1-5.0.13 common_test-1.19 crypto-4.7 debugger-5.0 dialyzer-4.2 diameter-2.2.3 edoc-0.12 eldap-1.2.8 erl_docgen-1.0 erl_interface-4.0 et-1.6.4 eunit-2.5 ftp-1.0.4 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 reltool-0.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tftp-1.0.2 tools-3.4 wx-1.9.1 xmerl-1.3.25 : OTP-23.0 : asn1-5.0.13 common_test-1.19 compiler-7.6 crypto-4.7 debugger-5.0 dialyzer-4.2 edoc-0.12 erl_docgen-1.0 erl_interface-4.0 erts-11.0 eunit-2.5 hipe-4.0 inets-7.2 jinterface-1.11 kernel-7.0 megaco-3.19 mnesia-4.17 observer-2.9.4 odbc-2.13 os_mon-2.5.2 parsetools-2.2 public_key-1.8 runtime_tools-1.15 sasl-4.0 snmp-5.6 ssh-4.10 ssl-10.0 stdlib-3.13 syntax_tools-2.3 tools-3.4 wx-1.9.1 xmerl-1.3.25 # diameter-2.2.3 eldap-1.2.8 et-1.6.4 ftp-1.0.4 reltool-0.8 tftp-1.0.2 : +OTP-22.3.4.16 : erts-10.7.2.8 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 crypto-4.6.5.2 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : +OTP-22.3.4.15 : crypto-4.6.5.2 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.3 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 erts-10.7.2.7 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : +OTP-22.3.4.14 : compiler-7.5.4.3 erts-10.7.2.7 # asn1-5.0.12 common_test-1.18.2 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.3 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.4 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.13 : compiler-7.5.4.2 erts-10.7.2.6 megaco-3.18.8.3 snmp-5.5.0.4 # asn1-5.0.12 common_test-1.18.2 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 ssh-4.9.1.2 ssl-9.6.2.3 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.12 : erts-10.7.2.5 ssl-9.6.2.3 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.1 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.2 mnesia-4.16.3.1 observer-2.9.3 odbc-2.12.4 os_mon-2.5.1.1 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.3 ssh-4.9.1.2 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : OTP-22.3.4.11 : erts-10.7.2.4 mnesia-4.16.3.1 os_mon-2.5.1.1 ssh-4.9.1.2 # asn1-5.0.12 common_test-1.18.2 compiler-7.5.4.1 crypto-4.6.5.1 debugger-4.2.8 dialyzer-4.1.1 diameter-2.2.3 edoc-0.11 eldap-1.2.8 erl_docgen-0.11 erl_interface-3.13.2 et-1.6.4 eunit-2.4.1 ftp-1.0.4.1 hipe-3.19.3 inets-7.1.3.3 jinterface-1.10.1 kernel-6.5.2.1 megaco-3.18.8.2 observer-2.9.3 odbc-2.12.4 parsetools-2.1.8 public_key-1.7.2 reltool-0.8 runtime_tools-1.14 sasl-3.4.2 snmp-5.5.0.3 ssl-9.6.2.2 stdlib-3.12.1 syntax_tools-2.2.1 tftp-1.0.2 tools-3.3.1 wx-1.9 xmerl-1.3.24 : @@ -56,6 +63,9 @@ OTP-22.0.3 : compiler-7.4.2 dialyzer-4.0.1 erts-10.4.2 ssl-9.3.2 stdlib-3.9.2 # OTP-22.0.2 : compiler-7.4.1 crypto-4.5.1 erts-10.4.1 stdlib-3.9.1 # asn1-5.0.9 common_test-1.17.3 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3.1 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.1 : ssl-9.3.1 # asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 stdlib-3.9 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0 : asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3 stdlib-3.9 syntax_tools-2.2 tools-3.2 wx-1.8.8 xmerl-1.3.21 # diameter-2.2.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2 parsetools-2.1.8 tftp-1.0.1 : +OTP-21.3.8.21 : erts-10.3.5.16 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3.1 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : +OTP-21.3.8.20 : erl_interface-3.11.3.1 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.3 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erts-10.3.5.15 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : +OTP-21.3.8.19 : crypto-4.4.2.3 erts-10.3.5.15 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.5 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.18 : erts-10.3.5.14 ssh-4.7.6.5 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssl-9.2.3.7 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.17 : erts-10.3.5.13 ssl-9.2.3.7 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.4 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : OTP-21.3.8.16 : erts-10.3.5.12 # asn1-5.0.8 common_test-1.17.2.1 compiler-7.3.2 crypto-4.4.2.2 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2.1 edoc-0.10 eldap-1.2.7 erl_docgen-0.9 erl_interface-3.11.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2.2 hipe-3.18.3 inets-7.0.7.2 jinterface-1.9.1 kernel-6.3.1.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.6.1 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.6.4 ssl-9.2.3.6 stdlib-3.8.2.4 syntax_tools-2.1.7.1 tftp-1.0.1 tools-3.1.0.1 wx-1.8.7 xmerl-1.3.20.1 : diff --git a/system/doc/general_info/DEPRECATIONS b/system/doc/general_info/DEPRECATIONS index 0f212348cf..358bc319af 100644 --- a/system/doc/general_info/DEPRECATIONS +++ b/system/doc/general_info/DEPRECATIONS @@ -21,11 +21,7 @@ # Added in OTP 23.2. # igor:_/_ since=23 remove=24 -erl_tidy:dir/0 since=23 remove=24 -erl_tidy:dir/1 since=23 remove=24 -erl_tidy:file/1 since=23 remove=24 -erl_tidy:module/1 since=23 remove=24 -erl_tidy:module/2 since=23 remove=24 +erl_tidy:_/_ since=23 remove=24 # # Added in OTP 23. diff --git a/system/doc/programming_examples/list_comprehensions.xml b/system/doc/programming_examples/list_comprehensions.xml index 706cb337ad..f9ce57f478 100644 --- a/system/doc/programming_examples/list_comprehensions.xml +++ b/system/doc/programming_examples/list_comprehensions.xml @@ -40,10 +40,10 @@ <c>[1,2,a,...]</c> and X is greater than 3.</p> <p>The notation <c><![CDATA[X <- [1,2,a,...]]]></c> is a generator and the expression <c>X > 3</c> is a filter.</p> - <p>An additional filter, <c>integer(X)</c>, can be added to restrict + <p>An additional filter, <c>is_integer(X)</c>, can be added to restrict the result to integers:</p> <pre> -> <input>[X || X <- [1,2,a,3,4,b,5,6], integer(X), X > 3].</input> +> <input>[X || X <- [1,2,a,3,4,b,5,6], is_integer(X), X > 3].</input> [4,5,6]</pre> <p>Generators can be combined. For example, the Cartesian product of two lists can be written as follows:</p> diff --git a/system/doc/programming_examples/records.xml b/system/doc/programming_examples/records.xml index d74ce22e4e..04c06816c5 100644 --- a/system/doc/programming_examples/records.xml +++ b/system/doc/programming_examples/records.xml @@ -222,7 +222,7 @@ print(#person{name = Name, age = Age, %% Demonstrates type testing, selector, updating. -birthday(P) when record(P, person) -> +birthday(P) when is_record(P, person) -> P#person{age = P#person.age + 1}. register_two_hackers() -> diff --git a/system/doc/reference_manual/typespec.xml b/system/doc/reference_manual/typespec.xml index a450f85742..b46cf0e8e8 100644 --- a/system/doc/reference_manual/typespec.xml +++ b/system/doc/reference_manual/typespec.xml @@ -476,22 +476,20 @@ <c>-spec</c> attribute. The general format is as follows: </p> <pre> - -spec Module:Function(ArgType1, ..., ArgTypeN) -> ReturnType.</pre> - <p> - The arity of the function must match the number of arguments, - else a compilation error occurs. - </p> + -spec Function(ArgType1, ..., ArgTypeN) -> ReturnType.</pre> <p> - This form can also be used in header files (.hrl) to declare type - information for exported functions. - Then these header files can be included in files that (implicitly or - explicitly) import these functions. + An implementation of the function with the same name + <c>Function</c> must exist in the current module, and the arity + of the function must match the number of arguments, else a + compilation error occurs. </p> <p> - Within a given module, the following shorthand suffices in most cases: + The following longer format with module name is also valid as + long as <c>Module</c> is the name of the current module. This + can be useful for documentation purposes. </p> <pre> - -spec Function(ArgType1, ..., ArgTypeN) -> ReturnType.</pre> + -spec Module:Function(ArgType1, ..., ArgTypeN) -> ReturnType.</pre> <p> Also, for documentation purposes, argument names can be given: </p> diff --git a/system/doc/system_architecture_intro/sys_arch_intro.xml b/system/doc/system_architecture_intro/sys_arch_intro.xml index e8ada6427b..f04e7a7879 100644 --- a/system/doc/system_architecture_intro/sys_arch_intro.xml +++ b/system/doc/system_architecture_intro/sys_arch_intro.xml @@ -93,7 +93,7 @@ <p>Database Management.</p> <list type="bulleted"> <item><em>QLC</em> Query language support for Mnesia DBMS.</item> - <item><em>Mnesia</em> A heavy duty real-time distributed database.</item> + <item><em>Mnesia</em> A heavy-duty real-time distributed database.</item> <item><em>ODBC</em> ODBC database interface.</item> </list> </item> |