diff options
806 files changed, 49227 insertions, 20759 deletions
diff --git a/.github/dockerfiles/Dockerfile.64-bit b/.github/dockerfiles/Dockerfile.64-bit index 25aadc5e84..f3a6e31939 100644 --- a/.github/dockerfiles/Dockerfile.64-bit +++ b/.github/dockerfiles/Dockerfile.64-bit @@ -4,7 +4,7 @@ FROM $BASE ARG MAKEFLAGS=$MAKEFLAGS ENV MAKEFLAGS=$MAKEFLAGS \ ERL_TOP=/buildroot/otp \ - PATH=/otp/bin:/buildroot/otp/bin:$PATH + PATH="/Erlang ∅⊤℞/bin":/buildroot/otp/bin:$PATH ARG ARCHIVE=./otp.tar.gz COPY $ARCHIVE /buildroot/otp.tar.gz @@ -14,35 +14,22 @@ WORKDIR /buildroot/otp/ ENV CFLAGS="-O2 -g -Werror" -## Configure, check that no application are disabled and then make -RUN ./configure --prefix=/otp && \ - if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \ - make && sudo make install +## Configure (if not cached), check that no application are disabled and then make +RUN if [ ! -f Makefile ]; then \ + touch README.md && \ + ./configure --prefix="/Erlang ∅⊤℞" && \ + if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \ + find . -type f -newer README.md | xargs tar --transform 's:^./:otp/:' -cf ../otp_cache.tar; \ + fi && \ + make && make docs DOC_TARGETS=chunks && \ + sudo make install install-docs DOC_TARGETS=chunks ## Disable -Werror as testcases do not compile with it on ENV CFLAGS="-O2 -g" -WORKDIR /buildroot/ - -## Install test tools rebar3, proper and jsx -RUN latest () { \ - local VSN=$(curl -sL "https://api.github.com/repos/$1/tags" | jq -r ".[] | .name" | grep -E '^v?[0-9]' | sort -V | tail -1); \ - curl -sL "https://github.com/$1/archive/$VSN.tar.gz" > $(basename $1).tar.gz; \ - } && \ - latest erlang/rebar3 && ls -la && \ - (tar xzf rebar3.tar.gz && cd rebar3-* && ./bootstrap && sudo cp rebar3 /usr/bin) && \ - latest proper-testing/proper && \ - (tar xzf proper.tar.gz && mv proper-* proper && cd proper && make) && \ - latest talentdeficit/jsx && \ - (tar xzf jsx.tar.gz && mv jsx-* jsx && cd jsx && rebar3 compile) - -ENV ERL_LIBS=/buildroot/proper:/buildroot/jsx - -WORKDIR /buildroot/otp/ - ## Update init.sh with correct env vars RUN echo "export MAKEFLAGS=$MAKEFLAGS" > /buildroot/env.sh && \ echo "export ERLC_USE_SERVER=$ERLC_USE_SERVER" >> /buildroot/env.sh && \ - echo "export ERL_TOP=$ERL_TOP" >> /buildroot/env.sh && \ - echo "export PATH=$PATH" >> /buildroot/env.sh && \ - echo "export ERL_LIBS=$ERL_LIBS" >> /buildroot/env.sh + echo "export ERL_TOP=\"$ERL_TOP\"" >> /buildroot/env.sh && \ + echo "export PATH=\"$PATH\"" >> /buildroot/env.sh && \ + echo "export ERL_LIBS=\"$ERL_LIBS\"" >> /buildroot/env.sh diff --git a/.github/dockerfiles/Dockerfile.ubuntu-base b/.github/dockerfiles/Dockerfile.ubuntu-base index c19537ef2a..4457d62a1f 100644 --- a/.github/dockerfiles/Dockerfile.ubuntu-base +++ b/.github/dockerfiles/Dockerfile.ubuntu-base @@ -6,47 +6,22 @@ FROM $BASE ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssl-dev unixodbc-dev libsctp-dev lksctp-tools libgmp3-dev libwxbase3.0-dev libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev" -ARG EXTRA_LIBS="erlang erlang-doc" - USER root ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +## Install build tools RUN apt-get update && apt-get -y upgrade && \ apt-get install -y build-essential m4 autoconf fop xsltproc \ - default-jdk libxml2-utils flex pkg-config \ - unixodbc odbc-postgresql postgresql \ - tzdata ssh openssh-server groff-base sudo gdb tinyproxy bind9 nsd expect vsftpd python \ - linux-tools-common linux-tools-generic linux-tools-`uname -r` curl jq \ - xvfb libgl1-mesa-dri \ - ${INSTALL_LIBS} && \ - for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \ - if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi && \ + default-jdk libxml2-utils flex pkg-config locales tzdata sudo ${INSTALL_LIBS} && \ sed -i 's@# en_US.UTF-8@en_US.UTF-8@g' /etc/locale.gen && locale-gen && \ update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0 -## EXTRA_LIBS are installed using a for loop because of bugs in the erlang-doc deb package -## Apache2 may already be installed, if so we do not want to install it again - ARG MAKEFLAGS=-j4 ENV MAKEFLAGS=$MAKEFLAGS \ ERLC_USE_SERVER=yes -## We install the latest version of the previous three releases in order to do -## backwards compatability testing of the Erlang distribution. -RUN apt-get install -y git && \ - curl -L https://raw.githubusercontent.com/kerl/kerl/master/kerl > /usr/bin/kerl && \ - chmod +x /usr/bin/kerl && \ - kerl update releases && \ - LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \ - for release in $(seq $(( LATEST - 3 )) $(( LATEST - 1 ))); do \ - VSN=$(kerl list releases | grep "^$release" | tail -1); \ - kerl build ${VSN} ${VSN} && \ - kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \ - done && \ - rm -rf ~/.kerl - ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 ARG USER=gitpod @@ -56,11 +31,54 @@ ARG uid=421 RUN echo "Europe/Stockholm" > /etc/timezone && \ ln -snf /usr/share/zoneinfo/$(cat /etc/timezone) /etc/localtime && \ + if ! grep ":${gid}:$" /etc/group; then groupadd -g ${gid} localgroup; fi && \ if [ ! -d /home/${USER} ]; then useradd -rm -d /home/${USER} -s /bin/sh -g ${gid} -G ${gid},sudo -u ${uid} ${USER}; fi && \ echo "${USER} ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \ echo "/buildroot/** r," >> /etc/apparmor.d/local/usr.sbin.named && \ echo "/tests/** r," >> /etc/apparmor.d/local/usr.sbin.named +## Java and log4j are used by fop to build documentation +COPY --chown=${USER}:${GROUP} dockerfiles/log4j.properties /home/${USER}/ +ENV JAVA_ARGS="-Dlog4j.configuration=file://home/${USER}/log4j.properties" + +ENV OTP_STRICT_INSTALL=yes + +RUN mkdir /buildroot /tests /otp && chown ${USER}:${GROUP} /buildroot /tests /otp + +## We install the latest version of the previous three releases in order to do +## backwards compatability testing of Erlang. +RUN apt-get install -y git curl && \ + curl -L https://raw.githubusercontent.com/kerl/kerl/master/kerl > /usr/bin/kerl && \ + chmod +x /usr/bin/kerl && \ + kerl update releases && \ + LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \ + echo "/usr/local/lib/erlang-${LATEST}/bin" > /home/${USER}/LATEST && \ + for release in $(seq $(( LATEST - 2 )) $(( LATEST ))); do \ + VSN=$(kerl list releases | grep "^$release" | tail -1); \ + kerl build ${VSN} ${VSN} && \ + kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \ + done && \ + rm -rf ~/.kerl + +## Install test tools +## EXTRA_LIBS are installed using a for loop because of bugs in the erlang-doc deb package +## Apache2 may already be installed, if so we do not want to install it again +ARG EXTRA_LIBS="erlang erlang-doc" +RUN apt-get install -y \ + unixodbc odbc-postgresql postgresql ssh openssh-server groff-base gdb \ + tinyproxy bind9 nsd expect vsftpd python emacs nano vim \ + linux-tools-common linux-tools-generic linux-tools-`uname -r` jq \ + xvfb libgl1-mesa-dri && \ + for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \ + if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi + +## We use tmux to test terminals +RUN apt-get install -y libevent-dev libutf8proc-dev && \ + cd /tmp && wget https://github.com/tmux/tmux/releases/download/3.2a/tmux-3.2a.tar.gz && \ + tar xvzf tmux-3.2a.tar.gz && cd tmux-3.2a && \ + ./configure --enable-static --enable-utf8proc && \ + make && make install + ## Setup progres so that the odbc test can run USER postgres @@ -79,15 +97,23 @@ ENV USER=${USER} RUN ssh-keygen -q -t rsa -N '' -f $HOME/.ssh/id_rsa && \ cp $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys -## Java and log4j are used by fop to build documentation -COPY --chown=${USER}:${GROUP} dockerfiles/log4j.properties /home/${USER}/ -ENV OTP_STRICT_INSTALL=yes \ - JAVA_ARGS="-Dlog4j.configuration=file://home/${USER}/log4j.properties" - -RUN sudo mkdir /buildroot /tests /otp && sudo chown ${USER}:${GROUP} /buildroot /tests /otp - COPY --chown=${USER}:${GROUP} dockerfiles/init.sh /buildroot/ -## TODO: Build Erlang versions N, N-1 and N-2 for compatability testing. +WORKDIR /buildroot/ + +## Install test tools rebar3, proper and jsx +RUN export PATH="$(cat /home/${USER}/LATEST)/${PATH}" && \ + latest () { \ + local VSN=$(curl -sL "https://api.github.com/repos/$1/tags" | jq -r ".[] | .name" | grep -E '^v?[0-9]' | sort -V | tail -1); \ + curl -sL "https://github.com/$1/archive/$VSN.tar.gz" > $(basename $1).tar.gz; \ + } && \ + latest erlang/rebar3 && ls -la && \ + (tar xzf rebar3.tar.gz && cd rebar3-* && ./bootstrap && sudo cp rebar3 /usr/bin) && \ + latest proper-testing/proper && \ + (tar xzf proper.tar.gz && mv proper-* proper && cd proper && make) && \ + latest talentdeficit/jsx && \ + (tar xzf jsx.tar.gz && mv jsx-* jsx && cd jsx && rebar3 compile) + +ENV ERL_LIBS=/buildroot/proper:/buildroot/jsx ENTRYPOINT ["/buildroot/init.sh"] diff --git a/.github/dockerfiles/init.sh b/.github/dockerfiles/init.sh index 8eb13abee2..af19cc097f 100755 --- a/.github/dockerfiles/init.sh +++ b/.github/dockerfiles/init.sh @@ -15,6 +15,6 @@ sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-t sudo bash -c "Xvfb :99 -ac -screen 0 1920x1080x24 -nolisten tcp" & export DISPLAY=:99 -PATH=$PATH:$(ls -1d /usr/local/lib/erlang-*/bin | tr '\n' ':') +PATH="$PATH:$(ls -1d /usr/local/lib/erlang-*/bin | tr '\n' ':')" exec /bin/bash -c "$1" diff --git a/.github/scripts/build-base-image.sh b/.github/scripts/build-base-image.sh index 543235e126..7069ef391a 100755 --- a/.github/scripts/build-base-image.sh +++ b/.github/scripts/build-base-image.sh @@ -29,16 +29,16 @@ case "${BASE_TAG}" in ;; esac -echo "::set-output name=BASE::${BASE}" -echo "::set-output name=BASE_TAG::${BASE_TAG}" -echo "::set-output name=BASE_TYPE::${BASE_TYPE}" +echo "BASE=${BASE}" >> $GITHUB_OUTPUT +echo "BASE_TAG=${BASE_TAG}" >> $GITHUB_OUTPUT +echo "BASE_TYPE=${BASE_TYPE}" >> $GITHUB_OUTPUT if [ -f "otp_docker_base.tar" ]; then docker load -i "otp_docker_base.tar" - echo "::set-output name=BASE_BUILD::loaded" + echo "BASE_BUILD=loaded" >> $GITHUB_OUTPUT elif [ -f "otp_docker_base/otp_docker_base.tar" ]; then docker load -i "otp_docker_base/otp_docker_base.tar" - echo "::set-output name=BASE_BUILD::loaded" + echo "BASE_BUILD=loaded" >> $GITHUB_OUTPUT else if [ "${BASE_USE_CACHE}" != "false" ]; then docker pull "${BASE_TAG}:${BASE_BRANCH}" @@ -58,9 +58,9 @@ else NEW_BASE_IMAGE_ID=$(docker images -q "${BASE_TAG}:latest") if [ "${BASE_IMAGE_ID}" = "${NEW_BASE_IMAGE_ID}" ]; then - echo "::set-output name=BASE_BUILD::cached" + echo "BASE_BUILD=cached" >> $GITHUB_OUTPUT else - echo "::set-output name=BASE_BUILD::re-built" + echo "BASE_BUILD=re-built" >> $GITHUB_OUTPUT docker save "${BASE_TAG}:latest" > "otp_docker_base.tar" fi fi diff --git a/.github/scripts/build-macos.sh b/.github/scripts/build-macos.sh index 82b07bac7b..73c35a6a22 100755 --- a/.github/scripts/build-macos.sh +++ b/.github/scripts/build-macos.sh @@ -1,12 +1,19 @@ #!/bin/sh -export MAKEFLAGS=-j$(getconf _NPROCESSORS_ONLN) -export ERL_TOP=`pwd` -export RELEASE_ROOT=$ERL_TOP/release +export MAKEFLAGS="-j$(getconf _NPROCESSORS_ONLN)" +export ERL_TOP="$(pwd)" export ERLC_USE_SERVER=true +export RELEASE_ROOT="$ERL_TOP/release" +BUILD_DOCS=false -./otp_build configure \ - --disable-dynamic-ssl-lib +if [ "$1" = "build_docs" ]; then + BUILD_DOCS=true + shift +fi + +./otp_build configure $* ./otp_build boot -a -./otp_build release -a $RELEASE_ROOT -make release_docs DOC_TARGETS=chunks +./otp_build release -a "$RELEASE_ROOT" +if $BUILD_DOCS; then + make release_docs DOC_TARGETS=chunks +fi diff --git a/.github/scripts/get-pr-number.es b/.github/scripts/get-pr-number.es index e925f95625..a388e6107a 100755 --- a/.github/scripts/get-pr-number.es +++ b/.github/scripts/get-pr-number.es @@ -11,19 +11,46 @@ main([Repo, HeadSha]) -> string:equal(HeadSha, Sha) end, AllOpenPrs) of {value, #{ <<"number">> := Number } } -> - io:format("::set-output name=result::~p~n", [Number]); + append_to_github_output("result=~p~n", [Number]); false -> - io:format("::set-output name=result::~ts~n", [""]) + append_to_github_output("result=~ts~n", [""]) + end. + +append_to_github_output(Fmt, Args) -> + case os:getenv("GITHUB_OUTPUT") of + false -> + io:format(standard_error, "GITHUB_OUTPUT env var missing?~n", []); + GitHubOutputFile -> + {ok, F} = file:open(GitHubOutputFile, [write, append]), + ok = io:fwrite(F, Fmt, Args), + ok = file:close(F) end. ghapi(CMD) -> - Data = cmd(CMD), - try jsx:decode(Data, [{return_maps,true}]) + decode(cmd(CMD)). + +decode(Data) -> + try jsx:decode(Data,[{return_maps, true}, return_tail]) of + {with_tail, Json, <<>>} -> + Json; + {with_tail, Json, Tail} -> + lists:concat([Json | decodeTail(Tail)]) catch E:R:ST -> io:format("Failed to decode: ~ts",[Data]), erlang:raise(E,R,ST) end. +decodeTail(Data) -> + try jsx:decode(Data,[{return_maps, true}, return_tail]) of + {with_tail, Json, <<>>} -> + [Json]; + {with_tail, Json, Tail} -> + [Json | decodeTail(Tail)] + catch E:R:ST -> + io:format(standard_error, "Failed to decode: ~ts",[Data]), + erlang:raise(E,R,ST) + end. + cmd(CMD) -> ListCmd = unicode:characters_to_list(CMD), io:format("cmd: ~ts~n",[ListCmd]), diff --git a/.github/scripts/init-pre-release.sh b/.github/scripts/init-pre-release.sh index 7a86f3052c..c7b7cab617 100755 --- a/.github/scripts/init-pre-release.sh +++ b/.github/scripts/init-pre-release.sh @@ -3,4 +3,33 @@ ## We create a tar ball that is used later by build-otp-tar ## to create the pre-built tar ball -git archive --prefix otp/ -o otp_src.tar.gz HEAD +AUTOCONF=0 +TARGET=otp_src.tar.gz + +if [ -n "$1" ]; then + TARGET="$1" +fi + +## This script is used to create archives for older releases +## so if configure does not exist in the git repo we need to +## create it. +if [ ! -f configure ]; then + ./otp_build autoconf + find . -name aclocal.m4 | xargs git add -f + find . -name configure | xargs git add -f + find . -name config.h.in | xargs git add -f + find . -name config.guess | xargs git add -f + find . -name config.sub | xargs git add -f + find . -name install-sh | xargs git add -f + if ! git config user.name; then + git config user.email "you@example.com" + git config user.name "Your Name" + fi + git commit --no-verify -m 'Add generated configure files' + AUTOCONF=1 +fi +git archive --prefix otp/ -o "$TARGET" HEAD + +if [ "$AUTOCONF" = 1 ]; then + git reset --hard HEAD~1 +fi diff --git a/.github/scripts/restore-from-prebuilt.sh b/.github/scripts/restore-from-prebuilt.sh new file mode 100755 index 0000000000..6e620bf558 --- /dev/null +++ b/.github/scripts/restore-from-prebuilt.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +set -xe + +CACHE_SOURCE_DIR="$1" +TARGET="$2" +ARCHIVE="$3" +EVENT="$4" +DELETED="$5" +CHANGES="$9" + +if [ ! -f "${CACHE_SOURCE_DIR}/otp_src.tar.gz" ] || [ "${NO_CACHE}" = "true" ]; then + cp "${ARCHIVE}" "${TARGET}" + cp "${ARCHIVE}" "${CACHE_SOURCE_DIR}/otp_src.tar.gz" + exit 0 +fi + +TMP_DIR=$(mktemp -d) +CACHE_DIR="${TMP_DIR}" +ARCHIVE_DIR="${TMP_DIR}/archive" + +mkdir "${ARCHIVE_DIR}" + +################################# +## START WORK ON THE CACHED FILES +################################# +echo "::group::{Restore cached files}" +tar -C "${CACHE_DIR}/" -xzf "${CACHE_SOURCE_DIR}/otp_src.tar.gz" + +## If configure scripts have NOT changed, we can restore configure and other C/java programs +if [ -z "${CONFIGURE}" ] || [ "${CONFIGURE}" = "false" ]; then + tar -C "${CACHE_DIR}/" -xzf "${CACHE_SOURCE_DIR}/otp_cache.tar.gz" +fi + +## If bootstrap has been changed, we do not use the cached .beam files +EXCLUDE_BOOTSTRAP="" +if [ "${BOOTSTRAP}" = "true" ]; then + find "${CACHE_DIR}/otp/lib" -name "*.beam" -exec rm -f {} \; +else + EXCLUDE_BOOTSTRAP=(--exclude "bootstrap") +fi + +## Make a copy of the cache for debugging +mkdir "${TMP_DIR}/cache" +cp -rp "${CACHE_DIR}/otp" "${TMP_DIR}/cache/" + +CACHE_DIR="${CACHE_DIR}/otp" + +echo "::group::{Delete files from PR}" +## Delete any files that this PR deletes +for delete in $DELETED; do + if [ -d "${CACHE_DIR}/${delete}" ]; then + rm -r "${CACHE_DIR}/${delete}" + elif [ -f "${CACHE_DIR}/${delete}" ]; then + rm "${CACHE_DIR}/${delete}" + else + echo "Could not find $delete to delete" + exit 1 + fi +done + +################################## +## START WORK ON THE UPDATED FILES +################################## + +echo "::group::{Extract changed files}" +if [ -n "${ARCHIVE}" ]; then + ## Extract with updated timestamp (the -m flag) so that any change will trigger a rebuild + tar -C "${ARCHIVE_DIR}/" -xzmf "${ARCHIVE}" + + ## Directory permissions in the archive and cache are for some reason different... + chmod -R g-w "${ARCHIVE_DIR}/" + + ## rlpgoD is the same as --archive, but without --times + RSYNC_ARGS=(-rlpgoD --itemize-changes --verbose --checksum --update "${EXCLUDE_BOOTSTRAP[@]}" "${ARCHIVE_DIR}/otp/" "${CACHE_DIR}/") + + CHANGES="${TMP_DIR}/changes" + PREV_CHANGES="${TMP_DIR}/prev-changes" + + touch "${PREV_CHANGES}" + + ## Below follows some rules about when we do not want to use the cache + ## The rules are run multiple times so that if any rule triggeres a delte + ## we will re-run the rules again with the new changes. + for i in $(seq 1 10); do + + echo "::group::{Run ${i} at pruning cache}" + + ## First do a dry run to see if we need to delete anything from cache + rsync --dry-run "${RSYNC_ARGS[@]}" | grep '^\(>\|c\)' > "${TMP_DIR}/changes" + cat "${TMP_DIR}/changes" + + if cmp -s "${CHANGES}" "${PREV_CHANGES}"; then + break; + fi + + ### If any parse transform is changed we recompile everything as we have + ### no idea what it may change. If the parse transform calls any other + ### modules we really should delete the cache for those as well, but + ### it is impossible for us to know which modules are used by a pt so + ### this has to be somekind of best effort. + echo "::group::{Run ${i}: parse transforms}" + PARSE_TRANSFORMS=$(grep -r '^parse_transform(' "${CACHE_DIR}/" | grep "/lib/[^/]*/src/" | awk -F ':' '{print $1}' | uniq) + for pt in $PARSE_TRANSFORMS; do + if grep "$(basename "${pt}")" "${CHANGES}"; then + echo "Deleting entire cache as a parse transform has changed" >&2 + rm -rf "${CACHE_DIR:?}/" + fi + done + + echo "::group::{Run ${i}: yecc}" + ### if yecc has changed, need to recompile all .yrl files + if grep "yecc.erl$" "${CHANGES}"; then + echo "Deleting all .yrl files as yecc has changed" >&2 + find "${CACHE_DIR}/" -name "*.yrl" -exec rm -f {} \; + fi + + echo "::group::{Run ${i}: asn1}" + ### If asn1 has changed, need to re-compile all .asn1 files + if grep lib/asn1 "${CHANGES}"; then + echo "Deleting all .asn1 files as asn1 has changed" >&2 + find "${CACHE_DIR}/" -name "*.asn1" -exec rm -f {} \; + fi + + echo "::group::{Run ${i}: docs}" + ### If any of the doc generating tools change, we need to re-compile the docs + if grep "lib/\(xmerl\|erl_docgen\|edoc\)" "${CHANGES}"; then + echo "Deleting all docs as documentation tools have changed" >&2 + rm -rf "${CACHE_DIR}"/lib/*/doc/ "${CACHE_DIR}/erts/doc/" "${CACHE_DIR}/system/" + fi + + ### Find all behaviours in OTP and check if any them as changed, we need to + ### rebuild all files that use them. + echo "::group::{Run ${i}: behaviours}" + BEHAVIOURS=$(grep -r "^-callback" "${CACHE_DIR}/" | grep "/lib/[^/]*/src/" | awk -F ':' '{print $1}' | uniq | sed 's:.*/\([^/.]*\)[.]erl$:\1:') + for behaviour in $BEHAVIOURS; do + if grep "${behaviour}[.]erl\$" "${CHANGES}"; then + echo "Deleting files using ${behaviour} has it has changed" >&2 + FILES=$(grep -r "^-behaviour(${behaviour})" "${CACHE_DIR}/" | grep "/lib/[^/]*/src/" | awk -F ':' '{print $1}') + rm -f $FILES + fi + done + + if [ "$i" = "10" ]; then + echo "Deleting entire cache as it did not stabalize in trime" >&2 + rm -rf "${CACHE_DIR:?}" + else + mv "${CHANGES}" "${PREV_CHANGES}" + fi + done + + echo "::group::{Sync changes over cached data}" + + ## Now we do the actual sync + rsync "${RSYNC_ARGS[@]}" +fi + +tar -czf "${TARGET}" -C "${TMP_DIR}" otp + +rm -rf "${TMP_DIR}" + +echo "::endgroup::" diff --git a/.github/scripts/restore-otp-image.sh b/.github/scripts/restore-otp-image.sh deleted file mode 100755 index 57621c25d5..0000000000 --- a/.github/scripts/restore-otp-image.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -BASE_BRANCH="$1" -BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.64-bit" | head -1 | tr '=' ' ' | awk '{print $3}') - -BASE_TAG="$BASE_TAG" .github/scripts/build-base-image.sh "${BASE_BRANCH}" - -cat > Dockerfile <<EOF -FROM ${BASE_TAG} -ADD otp-ubuntu-20.04.tar.gz / -WORKDIR /buildroot/otp/ -EOF -docker build -t otp . diff --git a/.github/scripts/sync-github-prs.es b/.github/scripts/sync-github-prs.es index bc033c3efe..6560ae5e02 100755 --- a/.github/scripts/sync-github-prs.es +++ b/.github/scripts/sync-github-prs.es @@ -62,7 +62,9 @@ handle_pr(_Repo, Target, string:equal(HeadSha, Sha) andalso string:equal(Status, <<"completed">>) end, maps:get(<<"workflow_runs">>, Runs)) of {value, Run} -> - Ident = integer_to_list(maps:get(<<"id">>,Run)), + Ident = integer_to_list( + erlang:phash2( + {maps:get(<<"id">>,Run), ?MODULE:module_info(md5)})), io:format("Checking for ~ts~n", [filename:join(PRDir, Ident)]), case file:read_file_info(filename:join(PRDir, Ident)) of {error, enoent} -> @@ -91,7 +93,9 @@ handle_pr(_Repo, Target, end, Artifacts), CTLogsIndex = filename:join([PRDir,"ct_logs","index.html"]), case file:read_file_info(CTLogsIndex) of - {ok, _} -> ok; + {ok, _} -> + CTSuiteFiles = filename:join([PRDir,"ct_logs","ct_run*","*.logs","run.*","suite.log"]), + lists:foreach(fun purge_suite/1, filelib:wildcard(CTSuiteFiles)); _ -> ok = filelib:ensure_dir(CTLogsIndex), ok = file:write_file(CTLogsIndex, ["No test logs found for ", Sha]) @@ -109,6 +113,47 @@ handle_pr(_Repo, Target, ok end. +%% We truncate the logs of all testcases of any suite that did not have any failures +purge_suite(SuiteFilePath) -> + {ok, SuiteFile} = file:read_file(SuiteFilePath), + SuiteDir = filename:dirname(SuiteFilePath), + Placeholder = "<html><body>github truncated successful testcase</body></html>", + case re:run(SuiteFile,"^=failed\s*\([0-9]+\)$",[multiline,{capture,all_but_first,binary}]) of + {match,[<<"0">>]} -> + io:format("Purging logs from: ~ts~n",[SuiteDir]), + ok = file:del_dir_r(filename:join(SuiteDir,"log_private")), + lists:foreach( + fun(File) -> + case filename:basename(File) of + "suite" ++ _ -> + ok; + "unexpected_io" ++_ -> + ok; + "cover.html" -> + ok; + _Else -> + file:write_file(File,Placeholder) + end + end, filelib:wildcard(filename:join(SuiteDir,"*.html"))); + _FailedTestcases -> + io:format("Purging logs from: ~ts~n",[SuiteDir]), + lists:foreach( + fun(File) -> + {ok, B} = file:read_file(File), + case re:run(B,"^=== Config value:",[multiline]) of + {match,_} -> + case re:run(B,"^=== successfully completed test case",[multiline]) of + {match, _} -> + file:write_file(File,Placeholder); + nomatch -> + ok + end; + nomatch -> + ok + end + end, filelib:wildcard(filename:join(SuiteDir,"*.html"))) + end. + ghapi(CMD) -> decode(cmd(CMD)). @@ -116,12 +161,14 @@ decode(Data) -> try jsx:decode(Data,[{return_maps, true}, return_tail]) of {with_tail, Json, <<>>} -> Json; - {with_tail, Json, Tail} -> + {with_tail, Json, Tail} when is_map(Json) -> [Key] = maps:keys(maps:remove(<<"total_count">>, Json)), #{ Key => lists:flatmap( fun(J) -> maps:get(Key, J) end, [Json | decodeTail(Tail)]) - } + }; + {with_tail, Json, Tail} when is_list(Json) -> + lists:concat([Json | decodeTail(Tail)]) catch E:R:ST -> io:format("Failed to decode: ~ts",[Data]), erlang:raise(E,R,ST) diff --git a/.github/scripts/sync-github-releases.sh b/.github/scripts/sync-github-releases.sh index 0cb2042f01..b71d5b54a4 100755 --- a/.github/scripts/sync-github-releases.sh +++ b/.github/scripts/sync-github-releases.sh @@ -215,7 +215,11 @@ if [ ${UPLOADED} = false ] && [ ${#MISSING_PREBUILD[0]} != 0 ]; then name="${MISSING_PREBUILD[0]}" stripped_name=$(_strip_name "${name}") git clone https://github.com/erlang/otp -b "${name}" otp_src - (cd otp_src && ../.github/scripts/init-pre-release.sh) + if [ -f otp_src/.github/scripts/init-pre-release.sh ]; then + (cd otp_src && ERL_TOP=$(pwd) .github/scripts/init-pre-release.sh) + else + (cd otp_src && ERL_TOP=$(pwd) ../.github/scripts/init-pre-release.sh) + fi case ${stripped_name} in 23.**) ## The 32-bit dockerfile build the doc chunks which we want diff --git a/.github/workflows/add-to-project.yaml b/.github/workflows/add-to-project.yaml new file mode 100644 index 0000000000..3c67d85add --- /dev/null +++ b/.github/workflows/add-to-project.yaml @@ -0,0 +1,26 @@ +name: Add bugs to bugs project + +on: + issues: + types: + - opened + pull_request_target: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + if: github.repository == 'erlang/otp' + steps: + - name: Generate token + id: generate_token + uses: tibdex/github-app-token@v1.5.2 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PEM }} + - uses: actions/add-to-project@v0.0.3 + with: + project-url: https://github.com/orgs/erlang/projects/13 + github-token: ${{ steps.generate_token.outputs.token }} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ce50c86a1e..bd16db2e23 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -21,6 +21,7 @@ on: pull_request: env: + ## Equivalent to github.event_name == 'pull_request' ? github.base_ref : github.ref_name BASE_BRANCH: ${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }} jobs: @@ -30,75 +31,111 @@ jobs: runs-on: ubuntu-latest outputs: BASE_BUILD: ${{ steps.base-build.outputs.BASE_BUILD }} + changes: ${{ steps.changes.outputs.changes }} + all: ${{ steps.apps.outputs.all }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Get applications + id: apps + run: | + .github/scripts/path-filters.sh > .github/scripts/path-filters.yaml + ALL_APPS=$(grep '^[a-z_]*:' .github/scripts/path-filters.yaml | sed 's/:.*$//') + ALL_APPS=$(jq -n --arg inarr "${ALL_APPS}" '$inarr | split("\n")' | tr '\n' ' ') + echo "all=${ALL_APPS}" >> $GITHUB_OUTPUT + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: .github/scripts/path-filters.yaml - name: Create initial pre-release tar - run: .github/scripts/init-pre-release.sh + run: .github/scripts/init-pre-release.sh otp_archive.tar.gz - name: Upload source tar archive - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: otp_git_archive - path: otp_src.tar.gz + path: otp_archive.tar.gz - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build BASE image - id: base-build - run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit - - name: Save BASE image - if: steps.base-build.outputs.BASE_BUILD == 're-built' - uses: actions/upload-artifact@v2 + - name: Cache BASE image + uses: actions/cache@v3 with: - name: otp_docker_base path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit + - name: Cache pre-built tar archives + id: pre-built-cache + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.base_ref }}-${{ github.event.pull_request.base.sha }} + - uses: dorny/paths-filter@v2 + id: cache + with: + filters: | + no-cache: + - '.github/**' + deleted: + - deleted: '**' + bootstrap: + - 'bootstrap/**' + configure: + - '**.ac' + - '**.in' + list-files: shell + - name: Restore from cache + env: + NO_CACHE: ${{ steps.cache.outputs.no-cache }} + BOOTSTRAP: ${{ steps.cache.outputs.bootstrap }} + CONFIGURE: ${{ steps.cache.outputs.configure }} + run: | + .github/scripts/restore-from-prebuilt.sh "`pwd`" \ + "`pwd`/.github/otp.tar.gz" \ + "`pwd`/otp_archive.tar.gz" \ + '${{ github.event_name }}' \ + '${{ steps.cache.outputs.deleted_files }}' \ + '${{ steps.changes.outputs.changes }}' + - name: Upload restored cache + uses: actions/upload-artifact@v3 + if: runner.debug == 1 + with: + name: restored-cache + path: .github/otp.tar.gz - name: Build image run: | - mv otp_src.tar.gz .github/otp.tar.gz docker build --tag otp \ --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ --file ".github/dockerfiles/Dockerfile.64-bit" \ .github/ - - name: Save Erlang/OTP image - run: | - docker run -v $PWD:/github --entrypoint "" otp \ - tar czf /github/otp-ubuntu-20.04.tar.gz /buildroot/ /otp/ - - name: Upload otp ubuntu image - uses: actions/upload-artifact@v2 - with: - name: otp-ubuntu-20.04 - path: otp-ubuntu-20.04.tar.gz - name: Build pre-built tar archives run: | docker run -v $PWD:/github --entrypoint "" otp \ - scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz - - name: Upload pre-built tar archive - uses: actions/upload-artifact@v2 - with: - name: otp_prebuilt_no_chunks - path: otp_src.tar.gz - - changed-apps: - name: Calculate changed applications - runs-on: ubuntu-latest - outputs: - changes: ${{ steps.changes.outputs.changes }} - all: ${{ steps.apps.outputs.all }} - steps: - - uses: actions/checkout@v2 - - name: Get applications - id: apps + scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /github/otp_src.tar.gz + - name: Build cache run: | - .github/scripts/path-filters.sh > .github/scripts/path-filters.yaml - ALL_APPS=$(grep '^[a-z_]*:' .github/scripts/path-filters.yaml | sed 's/:.*$//') - ALL_APPS=$(jq -n --arg inarr "${ALL_APPS}" '$inarr | split("\n")' | tr '\n' ' ') - echo "::set-output name=all::${ALL_APPS}" - - uses: dorny/paths-filter@v2 - id: changes + if [ -f otp_cache.tar.gz ]; then + gunzip otp_cache.tar.gz + else + docker run -v $PWD:/github --entrypoint "" otp \ + bash -c 'cp ../otp_cache.tar /github/' + fi + docker run -v $PWD:/github --entrypoint "" otp \ + bash -c 'set -x; C_APPS=$(ls -d ./lib/*/c_src); find Makefile ./make ./erts ./bin/`erts/autoconf/config.guess` ./lib/erl_interface ./lib/jinterface ${C_APPS} `echo "${C_APPS}" | sed -e 's:c_src$:priv:'` -type f -newer README.md \! -name "*.beam" \! -path "*/doc/*" | xargs tar --transform "s:^./:otp/:" -uvf /github/otp_cache.tar' + gzip otp_cache.tar + - name: Upload pre-built tar archive + uses: actions/upload-artifact@v3 with: - filters: .github/scripts/path-filters.yaml + name: otp_prebuilt + path: | + otp_src.tar.gz + otp_cache.tar.gz build-macos: name: Build Erlang/OTP (macOS) @@ -107,16 +144,16 @@ jobs: env: WXWIDGETS_VERSION: 3.1.5 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Download source archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: otp_prebuilt_no_chunks + name: otp_prebuilt - name: Cache wxWidgets id: wxwidgets-cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: wxWidgets key: wxWidgets-${{ env.WXWIDGETS_VERSION }}-${{ runner.os }}-12 @@ -130,7 +167,7 @@ jobs: tar -xzf ./otp_src.tar.gz export PATH=$PWD/wxWidgets/release/bin:$PATH cd otp - $GITHUB_WORKSPACE/.github/scripts/build-macos.sh + $GITHUB_WORKSPACE/.github/scripts/build-macos.sh build_docs --disable-dynamic-ssl-lib tar -czf otp_macos_$(cat OTP_VERSION)_x86-64.tar.gz -C release . - name: Test Erlang @@ -142,7 +179,7 @@ jobs: ./bin/erl -noshell -eval '{wx_ref,_,_,_} = wx:new(), io:format("wx ok~n"), halt().' - name: Upload tarball - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: otp_prebuilt_macos_x86-64 path: otp/otp_macos_*_x86-64.tar.gz @@ -155,21 +192,17 @@ jobs: runs-on: macos-12 needs: pack steps: + - uses: actions/checkout@v3 - name: Download source archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: otp_prebuilt_no_chunks + name: otp_prebuilt - name: Compile Erlang run: | tar -xzf ./otp_src.tar.gz cd otp - export ERL_TOP=`pwd` - export MAKEFLAGS="-j$(($(nproc) + 2)) -O" - export ERLC_USE_SERVER=true - ./otp_build configure --xcomp-conf=./xcomp/erl-xcomp-arm64-ios.conf --without-ssl - ./otp_build boot -a - ./otp_build release -a + $GITHUB_WORKSPACE/.github/scripts/build-macos.sh --xcomp-conf=./xcomp/erl-xcomp-arm64-ios.conf --without-ssl - name: Package .xcframework run: | @@ -180,7 +213,7 @@ jobs: xcodebuild -create-xcframework -output ./liberlang.xcframework -library liberlang.a - name: Upload framework - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ios_framework_${{ env.TARGET_ARCH }} path: otp/liberlang.xcframework @@ -209,7 +242,7 @@ jobs: move "c:\\Program Files\\OpenSSL-Win64" "c:\\OpenSSL-Win64" - name: Cache wxWidgets - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: wxWidgets key: wxWidgets-${{ env.WXWIDGETS_VERSION }}-${{ runner.os }} @@ -217,11 +250,13 @@ jobs: # actions/cache on Windows sometimes does not set cache-hit even though there was one. Setting it manually. - name: Set wxWidgets cache id: wxwidgets-cache + env: + WSLENV: GITHUB_OUTPUT/p run: | if [ -d wxWidgets ]; then - echo "::set-output name=cache-hit::true" + echo "cache-hit=true" >> $GITHUB_OUTPUT else - echo "::set-output name=cache-hit::false" + echo "cache-hit=false" >> $GITHUB_OUTPUT fi - name: Download wxWidgets @@ -249,9 +284,9 @@ jobs: nmake TARGET_CPU=amd64 BUILD=release SHARED=0 DIR_SUFFIX_CPU= -f makefile.vc - name: Download source archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: otp_prebuilt_no_chunks + name: otp_prebuilt - name: Compile Erlang run: | @@ -272,11 +307,62 @@ jobs: ./otp_build installer_win32 - name: Upload installer - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: otp_win32_installer path: otp/release/win32/otp*.exe + build-flavors: + name: Build Erlang/OTP (Types and Flavors) + runs-on: ubuntu-latest + needs: pack + if: contains(needs.pack.outputs.changes, 'emulator') + + steps: + - uses: actions/checkout@v3 + ## Download docker images + - name: Cache BASE image + uses: actions/cache@v3 + with: + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} + - name: Docker login + uses: docker/login-action@v2 + with: + registry: docker.pkg.github.com + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit + - name: Cache pre-built tar archives + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.ref_name }}-${{ github.sha }} + - name: Build image + run: | + .github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz + rm -f otp_{src,cache}.tar.gz + docker build --tag otp \ + --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ + --file ".github/dockerfiles/Dockerfile.64-bit" \ + .github/ + - name: Build Erlang/OTP flavors and types + run: | + TYPES="opt debug lcnt" + FLAVORS="emu jit" + for TYPE in ${TYPES}; do + for FLAVOR in ${FLAVORS}; do + echo "::group::{TYPE=$TYPE FLAVOR=$FLAVOR}" + docker run otp "make TYPE=$TYPE FLAVOR=$FLAVOR" + echo "::endgroup::" + done + done + build: name: Build Erlang/OTP runs-on: ubuntu-latest @@ -288,17 +374,23 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Download source archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: otp_prebuilt_no_chunks + name: otp_prebuilt - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache BASE image + uses: actions/cache@v3 + if: matrix.type == 'clang' + with: + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} - name: Build base image run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" ${{ matrix.type }} - name: Build ${{ matrix.type }} image @@ -312,65 +404,62 @@ jobs: runs-on: ubuntu-latest needs: pack steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 ## Download docker images - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Download base build - if: needs.pack.outputs.BASE_BUILD == 're-built' - uses: actions/download-artifact@v2 + - name: Cache BASE image + uses: actions/cache@v3 with: - name: otp_docker_base - - name: Download otp build - uses: actions/download-artifact@v2 - with: - name: otp-ubuntu-20.04 - - name: Restore docker image - run: .github/scripts/restore-otp-image.sh "${BASE_BRANCH}" - - ## Build pre-built tar with chunks - - name: Build doc chunks - run: | - docker build -t otp - <<EOF - FROM otp - RUN make docs DOC_TARGETS=chunks - EOF - - name: Build pre-built tar archives + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit + - name: Cache pre-built tar archives + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.ref_name }}-${{ github.sha }} + - name: Build image run: | - docker run -v $PWD:/github otp \ - "scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz" - - name: Upload pre-built tar archive - uses: actions/upload-artifact@v2 - with: - name: otp_prebuilt - path: otp_src.tar.gz + .github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz + rm -f otp_{src,cache}.tar.gz + docker build --tag otp \ + --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ + --file ".github/dockerfiles/Dockerfile.64-bit" \ + .github/ + ## Build all the documentation - name: Build documentation run: | docker build -t otp - <<EOF FROM otp - RUN make docs + RUN make release docs release_docs && sudo make install-docs EOF - name: Release docs to publish run: | docker run -v $PWD/:/github otp \ - "make release docs release_docs && make release_docs DOC_TARGETS='man html pdf' RELEASE_ROOT=/github/docs" + "make release_docs DOC_TARGETS='man html pdf' RELEASE_ROOT=/github/docs" sudo chown -R `whoami` docs cd docs tar czf ../otp_doc_man.tar.gz man rm -rf man tar czf ../otp_doc_html.tar.gz * - name: Upload html documentation archive - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: otp_doc_html path: otp_doc_html.tar.gz - name: Upload man documentation archive - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: otp_doc_man path: otp_doc_man.tar.gz @@ -378,32 +467,45 @@ jobs: - name: Run xmllint run: docker run otp "make xmllint" - name: Run html link check - run: docker run -v $PWD/:/github otp "scripts/otp_html_check /github/docs doc/index.html" + run: docker run -v $PWD/:/github otp "/github/scripts/otp_html_check /github/docs doc/index.html" static: name: Run static analysis runs-on: ubuntu-latest needs: pack steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 ## Download docker images - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Download base build - if: needs.pack.outputs.BASE_BUILD == 're-built' - uses: actions/download-artifact@v2 - with: - name: otp_docker_base - - name: Download otp build - uses: actions/download-artifact@v2 + - name: Cache BASE image + uses: actions/cache@v3 with: - name: otp-ubuntu-20.04 - - name: Restore docker image - run: .github/scripts/restore-otp-image.sh "${BASE_BRANCH}" + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit + - name: Cache pre-built tar archives + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.ref_name }}-${{ github.sha }} + - name: Build image + run: | + .github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz + rm -f otp_{src,cache}.tar.gz + docker build --tag otp \ + --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ + --file ".github/dockerfiles/Dockerfile.64-bit" \ + .github/ - name: Install clang-format run: | docker build -t otp - <<EOF @@ -415,39 +517,52 @@ jobs: run: docker run otp "make format-check" ## Run dialyzer - name: Run dialyzer - run: docker run otp 'scripts/run-dialyzer' + run: docker run -v $PWD/:/github otp '/github/scripts/run-dialyzer' test: name: Test Erlang/OTP runs-on: ubuntu-latest - needs: [pack, changed-apps] - if: needs.changed-apps.outputs.changes != '[]' + needs: pack + if: needs.pack.outputs.changes != '[]' strategy: matrix: - # type: ${{ fromJson(needs.changed-apps.outputs.all) }} - type: ${{ fromJson(needs.changed-apps.outputs.changes) }} + # type: ${{ fromJson(needs.pack.outputs.all) }} + type: ${{ fromJson(needs.pack.outputs.changes) }} # type: ["os_mon","sasl"] fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 ## Download docker images - - name: Download base build - if: needs.pack.outputs.BASE_BUILD == 're-built' - uses: actions/download-artifact@v2 + - name: Cache BASE image + uses: actions/cache@v3 with: - name: otp_docker_base - - name: Download otp build - uses: actions/download-artifact@v2 - with: - name: otp-ubuntu-20.04 + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Restore docker image - run: .github/scripts/restore-otp-image.sh "${BASE_BRANCH}" + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit + - name: Cache pre-built tar archives + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.ref_name }}-${{ github.sha }} + - name: Build image + run: | + .github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz + rm -f otp_{src,cache}.tar.gz + docker build --tag otp \ + --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ + --file ".github/dockerfiles/Dockerfile.64-bit" \ + .github/ - name: Run tests id: run-tests run: | @@ -465,6 +580,7 @@ jobs: sudo bash -c "echo 'core.%p' > /proc/sys/kernel/core_pattern" docker run --ulimit core=-1 --ulimit nofile=5000:5000 --pids-limit 512 \ -e CTRUN_TIMEOUT=90 -e SPEC_POSTFIX=gh \ + -e TEST_NEEDS_RELEASE=true -e "RELEASE_ROOT=/buildroot/otp/Erlang ∅⊤℞" \ -e EXTRA_ARGS="-ct_hooks cth_surefire [{path,\"/buildroot/otp/$DIR/make_test_dir/${{ matrix.type }}_junit.xml\"}]" \ -v "$PWD/make_test_dir:/buildroot/otp/$DIR/make_test_dir" \ otp "make TYPE=${TYPE} && make ${APP}_test TYPE=${TYPE}" @@ -479,7 +595,7 @@ jobs: sudo bash -c "chown -R `whoami` make_test_dir && chmod -R +r make_test_dir" tar czf ${{ matrix.type }}_test_results.tar.gz make_test_dir - name: Upload test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: ${{ matrix.type }}_test_results @@ -491,18 +607,39 @@ jobs: if: always() # Run even if the need has failed needs: test steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Cache BASE image + uses: actions/cache@v3 + with: + path: otp_docker_base.tar + key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }} - name: Docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.pkg.github.com username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build BASE image + run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit - name: Download test results - uses: actions/download-artifact@v2 - - run: mv otp-ubuntu-20.04/otp-ubuntu-20.04.tar.gz . - - name: Restore docker image - run: .github/scripts/restore-otp-image.sh "${BASE_BRANCH}" + uses: actions/download-artifact@v3 + - name: Cache pre-built tar archives + uses: actions/cache@v3 + with: + path: | + otp_src.tar.gz + otp_cache.tar.gz + key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + prebuilt-${{ github.ref_name }}-${{ github.sha }} + - name: Build image + run: | + .github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz + rm -f otp_{src,cache}.tar.gz + docker build --tag otp \ + --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ + --file ".github/dockerfiles/Dockerfile.64-bit" \ + .github/ - name: Merge test results run: | shopt -s nullglob @@ -532,14 +669,14 @@ jobs: -e 's:\(file="erts/\)make_test_dir/[^/]*:\1test:g' \ make_test_dir/*_junit.xml - name: Upload test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: test_results path: test_results.tar.gz - name: Upload Test Results if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Unit Test Results path: | @@ -560,22 +697,22 @@ jobs: run: | TAG=${GITHUB_REF#refs/tags/} VSN=${TAG#OTP-} - echo "::set-output name=tag::${TAG}" - echo "::set-output name=vsn::${VSN}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + echo "vsn=${VSN}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 ## Publish the pre-built archive and docs - name: Download source archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: otp_prebuilt - name: Download html docs - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: otp_doc_html - name: Download man docs - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: otp_doc_man @@ -613,7 +750,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Upload - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Event File path: ${{ github.event_path }} diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml index 4ae88c38aa..aac5bb4970 100644 --- a/.github/workflows/pr-comment.yaml +++ b/.github/workflows/pr-comment.yaml @@ -74,9 +74,9 @@ jobs: done if [ -d "Unit Test Results" ]; then - echo "::set-output name=HAS_TEST_ARTIFACTS::true" + echo "HAS_TEST_ARTIFACTS=true" >> $GITHUB_OUTPUT else - echo "::set-output name=HAS_TEST_ARTIFACTS::false" + echo "HAS_TEST_ARTIFACTS=false" >> $GITHUB_OUTPUT fi - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index c88056e46e..94664b47c8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ TAGS .idea autom4te.cache +configure.result.* *.beam *.asn1db @@ -126,15 +127,12 @@ JAVADOC-GENERATED /bootstrap/lib/edoc /bootstrap/lib/erl_docgen /bootstrap/lib/erl_interface -/bootstrap/lib/hipe -/bootstrap/lib/jinterface /bootstrap/lib/parsetools /bootstrap/lib/public_key /bootstrap/lib/runtime_tools /bootstrap/lib/sasl /bootstrap/lib/snmp /bootstrap/lib/syntax_tools -/bootstrap/lib/test_server /bootstrap/lib/wx /bootstrap/lib/xmerl @@ -235,11 +233,13 @@ JAVADOC-GENERATED /lib/compiler/test/*_no_opt_SUITE.erl /lib/compiler/test/*_no_copt_SUITE.erl +/lib/compiler/test/*_no_copt_ssa_SUITE.erl /lib/compiler/test/*_no_ssa_opt_SUITE.erl /lib/compiler/test/*_post_opt_SUITE.erl /lib/compiler/test/*_inline_SUITE.erl /lib/compiler/test/*_r23_SUITE.erl /lib/compiler/test/*_r24_SUITE.erl +/lib/compiler/test/*_r25_SUITE.erl /lib/compiler/test/*_no_module_opt_SUITE.erl /lib/compiler/test/*_no_type_opt_SUITE.erl /lib/compiler/test/*_dialyzer_SUITE.erl diff --git a/HOWTO/DEVELOPMENT.md b/HOWTO/DEVELOPMENT.md index 2220d9ab71..67f8119d17 100644 --- a/HOWTO/DEVELOPMENT.md +++ b/HOWTO/DEVELOPMENT.md @@ -1,7 +1,7 @@ # Developing Erlang/OTP The Erlang/OTP development repository is quite large and the make system -contains a lot of functionality to help when a developing. This howto +contains a lot of functionality to help when developing. This howto will try to showcase the most important features of the make system. The guide is mostly aimed towards development on a Unix platform, but @@ -550,8 +550,8 @@ in order to fetch the base image. If you want to build the base image locally you can do that like this: ```bash -docker built -t docker.pkg.github.com/erlang/otp/ubuntu-base \ - --build-arg BASE=ubuntu --build-arg USER=otptest --build-arg uid=$(id -u) \ +docker build -t docker.pkg.github.com/erlang/otp/ubuntu-base \ + --build-arg BASE=ubuntu:20.04 --build-arg USER=otptest --build-arg uid=$(id -u) \ --build-arg GROUP=uucp --build-arg gid=$(id -g) \ -f .github/dockerfiles/Dockerfile.ubuntu-base .github/ ``` diff --git a/HOWTO/INSTALL-WIN32.md b/HOWTO/INSTALL-WIN32.md index 4ad0159bcd..bd8387aaae 100644 --- a/HOWTO/INSTALL-WIN32.md +++ b/HOWTO/INSTALL-WIN32.md @@ -68,7 +68,7 @@ This is the short story though, for the experienced and impatient: <http://www.erlang.org/download.html>) and unpack with `tar` to the windows disk for example to: /mnt/c/src/ - * Install mingw-gcc, and make: `sudo apt install g++-mingw-w64 gcc-mingw-w64 make` + * Install mingw-gcc, and make: `sudo apt update && sudo apt install g++-mingw-w64 gcc-mingw-w64 make` * `$ cd UNPACK_DIR` @@ -150,7 +150,7 @@ the different tools: Install into `C:/OpenSSL-Win64` (or `C:/OpenSSL-Win32`) * wxWidgets (optional) - You need this to build wx and use gui's in debugger and observer. + You need this to build wx to use gui's in debugger and observer. We recommend v3.1.4 or later. Unpack into `c:/opt/local64/pgm/wxWidgets-3.1.4` diff --git a/HOWTO/TESTING.md b/HOWTO/TESTING.md index 66cc21f89a..c6bad4f65c 100644 --- a/HOWTO/TESTING.md +++ b/HOWTO/TESTING.md @@ -193,7 +193,7 @@ Examining the results --------------------- Open the file `$ERL_TOP/release/tests/test_server/index.html` in a web browser. Or open -`$ERL_TOP/release/tests/test_server/last_test.html` when a test suite is running to +`$ERL_TOP/release/tests/test_server/suite.log.latest.html` when a test suite is running to examine the results so far for the currently executing test suite. diff --git a/Makefile.in b/Makefile.in index cc92df3a21..72ae67524a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -379,12 +379,12 @@ endif # bootstrap is not included, it requires a pre built emulator... ifeq ($(OTP_TINY_BUILD),true) all_bootstraps: erl_interface emulator bootstrap_setup \ - tiny_secondary_bootstrap_build tiny_secondary_bootstrap_copy + tiny_secondary_bootstrap_copy else all_bootstraps: erl_interface emulator \ bootstrap_setup \ - secondary_bootstrap_build secondary_bootstrap_copy \ - tertiary_bootstrap_build tertiary_bootstrap_copy + secondary_bootstrap_copy \ + tertiary_bootstrap_copy endif # @@ -429,7 +429,7 @@ endif # Target only used when building commercial ERTS patches # --------------------------------------------------------------- -release_docs docs: doc_bootstrap_build doc_bootstrap_copy mod2app +release_docs docs: doc_bootstrap_copy mod2app ifeq ($(OTP_SMALL_BUILD),true) cd $(ERL_TOP)/lib && \ PATH=$(BOOT_PREFIX)"$${PATH}" ERL_TOP=$(ERL_TOP) \ @@ -464,7 +464,7 @@ endif $(DOCGEN)/priv/bin/validate_links.escript $(ERL_TOP) make/$(TARGET)/mod2app.xml \ lib/*/doc/xml/*.xml erts/doc/xml/*.xml system/doc/xml/*/*.xml -mod2app: doc_bootstrap_build doc_bootstrap_copy $(ERL_TOP)/make/$(TARGET)/mod2app.xml +mod2app: doc_bootstrap_copy $(ERL_TOP)/make/$(TARGET)/mod2app.xml $(ERL_TOP)/make/$(TARGET)/mod2app.xml: erts/doc/src/Makefile lib/*/doc/src/Makefile PATH=$(BOOT_PREFIX)"$${PATH}" escript $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/priv/bin/xref_mod_app.escript -topdir $(ERL_TOP) -outfile $(ERL_TOP)/make/$(TARGET)/mod2app.xml @@ -625,40 +625,47 @@ $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET)/inet_gethost: $(ERL_TOP)/bin/$(TARGET) @cp $(ERL_TOP)/bin/$(TARGET)/inet_gethost $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET)/inet_gethost @chmod 755 $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET)/inet_gethost +$(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET): + $(V_at)mkdir $@ -bootstrap_setup_target: +bootstrap_setup_target: $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET) @{ test -r $(BOOTSTRAP_ROOT)/bootstrap/target && \ test $(TARGET) = `cat $(BOOTSTRAP_ROOT)/bootstrap/target`; } || \ echo $(TARGET) > $(BOOTSTRAP_ROOT)/bootstrap/target - if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET) ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/bin/$(TARGET) ; fi -tiny_secondary_bootstrap_build: - $(make_verbose)cd lib && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt SECONDARY_BOOTSTRAP=true TINY_BUILD=true ERL_COMPILE_WARNINGS_AS_ERRORS=yes - -secondary_bootstrap_build: - $(make_verbose)cd lib && \ - ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt SECONDARY_BOOTSTRAP=true ERL_COMPILE_WARNINGS_AS_ERRORS=yes - -tiny_secondary_bootstrap_copy: - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include ; fi - $(V_at)for x in lib/parsetools/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - $(V_at)for x in lib/parsetools/include/*.hrl; do \ +# The TINY SECONDARY bootstrap +# - sasl needed to create start.boot +# - parsetools needed by anything with an yrl +TINY_SECONDARY_BOOTSTRAP=parsetools sasl +TINY_SECONDARY_BOOTSTRAP_BUILD=$(TINY_SECONDARY_BOOTSTRAP) +# The SECONDARY bootstrap +# - asn1 needed by public_key +SECONDARY_BOOTSTRAP=$(TINY_SECONDARY_BOOTSTRAP) asn1 +SECONDARY_BOOTSTRAP_BUILD=$(TINY_SECONDARY_BOOTSTRAP_BUILD) asn1/src + +# The TERTIARY bootstrap +# - wx is needed for wx_object behaviour in debugger, observer and et +# - public_key is needed for include files in ssl and ssh +# - erl_interface is needed by odbc +# - syntax_tools is needed by diameter +# - snmp is needed to compile tests +# - runtime_tools includes are needed by observer +# - xmerl includes are needed by ct, edoc, erl_docgen and wx +# - common_test is needed to compile tests +TERTIARY_BOOTSTRAP_BUILD=parsetools wx public_key erl_interface syntax_tools snmp +TERTIARY_BOOTSTRAP=$(TERTIARY_BOOTSTRAP_BUILD) runtime_tools xmerl common_test + +# The DOC bootstrap +DOC_BOOTSTRAP_BUILD=xmerl edoc erl_docgen +DOC_BOOTSTRAP=$(DOC_BOOTSTRAP_BUILD) + +$(BOOTSTRAP_ROOT)/bootstrap/lib/%/update: + $(V_at)for x in "$(dir $@)" "$(dir $@)/ebin" "$(dir $@)/include"; do \ + if [ ! -d "$$x" ]; then mkdir "$$x"; fi \ + done + $(V_at)for x in $(wildcard lib/$*/ebin/*.beam); do \ BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include/$$BN; \ + TF=$(dir $@)/ebin/$$BN; \ test -f $$TF && \ test '!' -z "`find $$x -newer $$TF -print`" && \ ${INSTALL_DATA} -p $$x $$TF; \ @@ -666,299 +673,39 @@ tiny_secondary_bootstrap_copy: ${INSTALL_DATA} -p $$x $$TF; \ true; \ done - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/include ; fi - $(V_at)for x in lib/sasl/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ + $(V_at)for x in $(wildcard lib/$*/include/*.hrl) $(wildcard lib/$*/include/*.h); do \ + BN=`basename $$x`; \ + TF=$(dir $@)/include/$$BN; \ + test -f $$TF && \ + test '!' -z "`find $$x -newer $$TF -print`" && \ + ${INSTALL_DATA} -p $$x $$TF; \ + test '!' -f $$TF && \ + ${INSTALL_DATA} -p $$x $$TF; \ + true; \ done -secondary_bootstrap_copy: - $(make_verbose) - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include ; fi - $(V_at)for x in lib/parsetools/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p lib/parsetools/ebin/*.beam $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/ebin - $(V_at)for x in lib/parsetools/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p -f lib/parsetools/include/*.hrl $(BOOTSTRAP_ROOT)/bootstrap/lib/parsetools/include - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1 ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1 ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/src ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/src ; fi - $(V_at)for x in lib/asn1/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p lib/asn1/ebin/*.beam $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/ebin - $(V_at)for x in lib/asn1/src/*.[eh]rl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/src/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p -f lib/asn1/src/*.erl lib/asn1/src/*.hrl $(BOOTSTRAP_ROOT)/bootstrap/lib/asn1/src - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/include ; fi - $(V_at)for x in lib/xmerl/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - -tertiary_bootstrap_build: +tiny_secondary_bootstrap: $(make_verbose)cd lib && \ ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt TERTIARY_BOOTSTRAP=true ERL_COMPILE_WARNINGS_AS_ERRORS=yes + $(MAKE) opt BOOTSTRAP="$(TINY_SECONDARY_BOOTSTRAP_BUILD)" ERL_COMPILE_WARNINGS_AS_ERRORS=yes -tertiary_bootstrap_copy: - $(make_verbose) - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/include ; fi - $(V_at)for x in lib/snmp/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p lib/snmp/ebin/*.beam $(BOOTSTRAP_ROOT)/bootstrap/lib/snmp/ebin - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/wx ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/wx ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/wx/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/wx/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/wx/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/wx/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/common_test ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/common_test ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/common_test/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/common_test/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/runtime_tools ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/runtime_tools ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/runtime_tools/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/runtime_tools/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/public_key ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/public_key ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/public_key/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/public_key/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_interface ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_interface ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_interface/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_interface/include ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/ ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/ ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson/otp ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson/otp ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson/otp/erlang ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson/otp/erlang ; fi - $(V_at)for x in lib/sasl/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# $(V_at)${INSTALL_DATA} -p lib/sasl/ebin/*.beam $(BOOTSTRAP_ROOT)/bootstrap/lib/sasl/ebin - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools/ebin ; fi - $(V_at)for x in lib/syntax_tools/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - $(V_at)for x in lib/wx/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/wx/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# copy wx_object to remove undef behaviour warnings - $(V_at)for x in lib/wx/ebin/wx_object.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/wx/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - -# copy test includes to be able to compile tests with bootstrap compiler - $(V_at)for x in lib/common_test/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/common_test/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - -# copy runtime_tool includes to be able to compile with include_lib - $(V_at)for x in lib/runtime_tools/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/runtime_tools/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# copy public_key includes to be able to compile with include_lib - $(V_at)for x in lib/public_key/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/public_key/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# copy erl_interface includes - $(V_at)for x in lib/erl_interface/include/*.h; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/erl_interface/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# copy jinterface priv directory - $(V_at)if test -d lib/jinterface/priv; then \ - for x in lib/jinterface/priv/OtpErlang.jar; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done; \ - for x in lib/jinterface/priv/com/ericsson/otp/erlang/*; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/jinterface/priv/com/ericsson/otp/erlang/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done; \ - fi -# $(V_at)${INSTALL_DATA} -p lib/syntax_tools/ebin/*.beam $(BOOTSTRAP_ROOT)/bootstrap/lib/syntax_tools/ebin - -doc_bootstrap_build: +secondary_bootstrap: $(make_verbose)cd lib && \ ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ - $(MAKE) opt DOC_BOOTSTRAP=true + $(MAKE) opt BOOTSTRAP="$(SECONDARY_BOOTSTRAP_BUILD)" ERL_COMPILE_WARNINGS_AS_ERRORS=yes -doc_bootstrap_copy: - $(make_verbose) -# XMERL - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/ebin ; fi - $(V_at)for x in lib/xmerl/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/xmerl/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# xmerl/include already copied in secondary bootstrap -# EDOC - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/ebin ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/include ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/include ; fi - $(V_at)for x in lib/edoc/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done - $(V_at)for x in lib/edoc/include/*.hrl; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/edoc/include/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done -# ERL_DOCGEN - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen ; fi - $(V_at)if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/ebin ; then mkdir $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/ebin ; fi - $(V_at)for x in lib/erl_docgen/ebin/*.beam; do \ - BN=`basename $$x`; \ - TF=$(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/ebin/$$BN; \ - test -f $$TF && \ - test '!' -z "`find $$x -newer $$TF -print`" && \ - ${INSTALL_DATA} -p $$x $$TF; \ - test '!' -f $$TF && \ - ${INSTALL_DATA} -p $$x $$TF; \ - true; \ - done +tiny_secondary_bootstrap_copy: tiny_secondary_bootstrap $(foreach app, $(TINY_SECONDARY_BOOTSTRAP), $(BOOTSTRAP_ROOT)/bootstrap/lib/$(app)/update) +secondary_bootstrap_copy: secondary_bootstrap $(foreach app, $(SECONDARY_BOOTSTRAP), $(BOOTSTRAP_ROOT)/bootstrap/lib/$(app)/update) + +tertiary_bootstrap: + $(make_verbose) ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ + $(MAKE) -C lib opt BOOTSTRAP="$(TERTIARY_BOOTSTRAP_BUILD)" ERL_COMPILE_WARNINGS_AS_ERRORS=yes +tertiary_bootstrap_copy: tertiary_bootstrap $(foreach app, $(TERTIARY_BOOTSTRAP), $(BOOTSTRAP_ROOT)/bootstrap/lib/$(app)/update) + +doc_bootstrap: + $(make_verbose) ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ + $(MAKE) -C lib opt BOOTSTRAP="$(DOC_BOOTSTRAP_BUILD)" +doc_bootstrap_copy: secondary_bootstrap_copy doc_bootstrap tertiary_bootstrap_copy $(foreach app, $(DOC_BOOTSTRAP), $(BOOTSTRAP_ROOT)/bootstrap/lib/$(app)/update) $(V_at)for d in priv priv/bin priv/css priv/dtd priv/dtd_html_entities priv/dtd_man_entities priv/images priv/js priv/js/flipmenu priv/xsl; do \ if test ! -d $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/$$d ; then mkdir -p $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/$$d ; fi; \ for x in lib/erl_docgen/$$d/*; do \ @@ -1044,8 +791,8 @@ primary_bootstrap: primary_bootstrap_build: primary_bootstrap_mkdirs primary_bootstrap_compiler \ primary_bootstrap_stdlib $(make_verbose)cd lib && $(MAKE) ERLC_FLAGS='-pa $(BOOTSTRAP_COMPILER)/ebin $(DETERMINISM_FLAG)' \ - BOOTSTRAP_TOP=$(BOOTSTRAP_TOP) \ - BOOTSTRAP=1 opt + BOOTSTRAP_TOP=$(BOOTSTRAP_TOP) PRIMARY_BOOTSTRAP=1 \ + BOOTSTRAP="kernel stdlib compiler" opt primary_bootstrap_compiler: $(make_verbose)cd lib/compiler && $(MAKE) \ diff --git a/OTP_VERSION b/OTP_VERSION index 44c5306e24..638adf52ce 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -25.1.2 +26.0-rc0 @@ -54,7 +54,7 @@ cd otp Checkout the branch or tag of your choice ```sh -git checkout maint-24 # current latest stable version +git checkout maint-25 # current latest stable version ``` Configure, build and install diff --git a/bootstrap/bin/no_dot_erlang.boot b/bootstrap/bin/no_dot_erlang.boot Binary files differindex 7153e46c1b..eaeeafe202 100644 --- a/bootstrap/bin/no_dot_erlang.boot +++ b/bootstrap/bin/no_dot_erlang.boot diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 7153e46c1b..eaeeafe202 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 7153e46c1b..eaeeafe202 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_a.beam b/bootstrap/lib/compiler/ebin/beam_a.beam Binary files differindex d81e1e3e3b..e41bd34848 100644 --- a/bootstrap/lib/compiler/ebin/beam_a.beam +++ b/bootstrap/lib/compiler/ebin/beam_a.beam diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 3d8d5f0375..f43e2ca76b 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_block.beam b/bootstrap/lib/compiler/ebin/beam_block.beam Binary files differindex 9858ec8413..2797fc04d3 100644 --- a/bootstrap/lib/compiler/ebin/beam_block.beam +++ b/bootstrap/lib/compiler/ebin/beam_block.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bounds.beam b/bootstrap/lib/compiler/ebin/beam_bounds.beam Binary files differindex f07ab5e91a..c1c6f57e68 100644 --- a/bootstrap/lib/compiler/ebin/beam_bounds.beam +++ b/bootstrap/lib/compiler/ebin/beam_bounds.beam diff --git a/bootstrap/lib/compiler/ebin/beam_call_types.beam b/bootstrap/lib/compiler/ebin/beam_call_types.beam Binary files differindex f6411aa457..4c47feb91d 100644 --- a/bootstrap/lib/compiler/ebin/beam_call_types.beam +++ b/bootstrap/lib/compiler/ebin/beam_call_types.beam diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex 91a536bf46..f2f790ee2d 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex c57682576f..72129f1572 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_digraph.beam b/bootstrap/lib/compiler/ebin/beam_digraph.beam Binary files differindex b522dac9c3..89b869964c 100644 --- a/bootstrap/lib/compiler/ebin/beam_digraph.beam +++ b/bootstrap/lib/compiler/ebin/beam_digraph.beam diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex 9f591e9f70..cb7b7b52e3 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_flatten.beam b/bootstrap/lib/compiler/ebin/beam_flatten.beam Binary files differindex d98664ce03..c0b8583734 100644 --- a/bootstrap/lib/compiler/ebin/beam_flatten.beam +++ b/bootstrap/lib/compiler/ebin/beam_flatten.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex 22dd3f82c3..8359309beb 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam Binary files differindex 97c218480c..9ccf6340c3 100644 --- a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_listing.beam b/bootstrap/lib/compiler/ebin/beam_listing.beam Binary files differindex f71f5d4bf5..9beb204460 100644 --- a/bootstrap/lib/compiler/ebin/beam_listing.beam +++ b/bootstrap/lib/compiler/ebin/beam_listing.beam diff --git a/bootstrap/lib/compiler/ebin/beam_opcodes.beam b/bootstrap/lib/compiler/ebin/beam_opcodes.beam Binary files differindex 1f42b157ff..0c468f1547 100644 --- a/bootstrap/lib/compiler/ebin/beam_opcodes.beam +++ b/bootstrap/lib/compiler/ebin/beam_opcodes.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa.beam b/bootstrap/lib/compiler/ebin/beam_ssa.beam Binary files differindex f489f46b98..2e715fb90f 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bc_size.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bc_size.beam Binary files differindex ba37502190..890014135d 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_bc_size.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bc_size.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bool.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bool.beam Binary files differindex e82b00aac9..2ccb2b682a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_bool.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bool.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam Binary files differindex e0d4fac178..3366071630 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam Binary files differindex 3f1ec45a9e..476e8e6ae1 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam Binary files differindex 690e96e2d6..bf1524037a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam b/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam Binary files differindex a1209d6fb5..5fd597c90c 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam Binary files differindex 606c145f4c..d54cfdbca6 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam Binary files differindex e4958e9b97..369a1caf46 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam Binary files differindex 38e78440bc..f38c4c7434 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam Binary files differindex 8fdaf19b9b..459fee8514 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_share.beam b/bootstrap/lib/compiler/ebin/beam_ssa_share.beam Binary files differindex 87861b519c..2178b9d47f 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_share.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_share.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_throw.beam b/bootstrap/lib/compiler/ebin/beam_ssa_throw.beam Binary files differindex 65eba1cc55..ef91c0b49b 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_throw.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_throw.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam Binary files differindex 24f1357882..9556ecae0f 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam diff --git a/bootstrap/lib/compiler/ebin/beam_trim.beam b/bootstrap/lib/compiler/ebin/beam_trim.beam Binary files differindex 403d14af8a..b3e5250843 100644 --- a/bootstrap/lib/compiler/ebin/beam_trim.beam +++ b/bootstrap/lib/compiler/ebin/beam_trim.beam diff --git a/bootstrap/lib/compiler/ebin/beam_types.beam b/bootstrap/lib/compiler/ebin/beam_types.beam Binary files differindex 80f17fb734..dd980f1608 100644 --- a/bootstrap/lib/compiler/ebin/beam_types.beam +++ b/bootstrap/lib/compiler/ebin/beam_types.beam diff --git a/bootstrap/lib/compiler/ebin/beam_utils.beam b/bootstrap/lib/compiler/ebin/beam_utils.beam Binary files differindex 43f43219ca..132818c143 100644 --- a/bootstrap/lib/compiler/ebin/beam_utils.beam +++ b/bootstrap/lib/compiler/ebin/beam_utils.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex 5fa6a2eb71..a088e13686 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/beam_z.beam b/bootstrap/lib/compiler/ebin/beam_z.beam Binary files differindex 0ad3c95960..5e849425dc 100644 --- a/bootstrap/lib/compiler/ebin/beam_z.beam +++ b/bootstrap/lib/compiler/ebin/beam_z.beam diff --git a/bootstrap/lib/compiler/ebin/cerl.beam b/bootstrap/lib/compiler/ebin/cerl.beam Binary files differindex c0e3654c1e..45a5b16c53 100644 --- a/bootstrap/lib/compiler/ebin/cerl.beam +++ b/bootstrap/lib/compiler/ebin/cerl.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_clauses.beam b/bootstrap/lib/compiler/ebin/cerl_clauses.beam Binary files differindex f1deadb265..5647482dd3 100644 --- a/bootstrap/lib/compiler/ebin/cerl_clauses.beam +++ b/bootstrap/lib/compiler/ebin/cerl_clauses.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex d3f8397584..650534d9e3 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex 79c464e800..b47072c6d6 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex 0057b1eea9..58996e2fb1 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.appup b/bootstrap/lib/compiler/ebin/compiler.appup index aa537986a5..2585c4a26d 100644 --- a/bootstrap/lib/compiler/ebin/compiler.appup +++ b/bootstrap/lib/compiler/ebin/compiler.appup @@ -16,7 +16,7 @@ %% limitations under the License. %% %% %CopyrightEnd% -{"8.1", +{"8.2", [{<<".*">>,[{restart_application, compiler}]}], [{<<".*">>,[{restart_application, compiler}]}] }. diff --git a/bootstrap/lib/compiler/ebin/core_lib.beam b/bootstrap/lib/compiler/ebin/core_lib.beam Binary files differindex e316333572..3a7a6682c1 100644 --- a/bootstrap/lib/compiler/ebin/core_lib.beam +++ b/bootstrap/lib/compiler/ebin/core_lib.beam diff --git a/bootstrap/lib/compiler/ebin/core_lint.beam b/bootstrap/lib/compiler/ebin/core_lint.beam Binary files differindex 11c1ad9c82..95b3393ed2 100644 --- a/bootstrap/lib/compiler/ebin/core_lint.beam +++ b/bootstrap/lib/compiler/ebin/core_lint.beam diff --git a/bootstrap/lib/compiler/ebin/core_parse.beam b/bootstrap/lib/compiler/ebin/core_parse.beam Binary files differindex 46ed9443dd..8244774d52 100644 --- a/bootstrap/lib/compiler/ebin/core_parse.beam +++ b/bootstrap/lib/compiler/ebin/core_parse.beam diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex f5b8955675..f44ae60a0e 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/core_scan.beam b/bootstrap/lib/compiler/ebin/core_scan.beam Binary files differindex 5da4515384..c77df9ce8f 100644 --- a/bootstrap/lib/compiler/ebin/core_scan.beam +++ b/bootstrap/lib/compiler/ebin/core_scan.beam diff --git a/bootstrap/lib/compiler/ebin/erl_bifs.beam b/bootstrap/lib/compiler/ebin/erl_bifs.beam Binary files differindex c858cbbfcd..232c4c0cec 100644 --- a/bootstrap/lib/compiler/ebin/erl_bifs.beam +++ b/bootstrap/lib/compiler/ebin/erl_bifs.beam diff --git a/bootstrap/lib/compiler/ebin/rec_env.beam b/bootstrap/lib/compiler/ebin/rec_env.beam Binary files differindex 2147d61279..d4317d2b0e 100644 --- a/bootstrap/lib/compiler/ebin/rec_env.beam +++ b/bootstrap/lib/compiler/ebin/rec_env.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_alias.beam b/bootstrap/lib/compiler/ebin/sys_core_alias.beam Binary files differindex c8c0c54442..b4f68a5579 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_alias.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_alias.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam Binary files differindex d51f74ccb1..2a715e7c35 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex bc2057d1c2..7f99d23be8 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam Binary files differindex 714ff86f48..04df8b4f71 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_inline.beam b/bootstrap/lib/compiler/ebin/sys_core_inline.beam Binary files differindex c142a3f22b..b8df9a8a36 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_inline.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_inline.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_prepare.beam b/bootstrap/lib/compiler/ebin/sys_core_prepare.beam Binary files differindex 0e2debbfba..4e7aabe242 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_prepare.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_prepare.beam diff --git a/bootstrap/lib/compiler/ebin/sys_messages.beam b/bootstrap/lib/compiler/ebin/sys_messages.beam Binary files differindex ad635f1d04..6dd1bb3f0e 100644 --- a/bootstrap/lib/compiler/ebin/sys_messages.beam +++ b/bootstrap/lib/compiler/ebin/sys_messages.beam diff --git a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam Binary files differindex ef7de3c3ae..6ce724a570 100644 --- a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam +++ b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex ff06839fba..1503738d88 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 36838861b5..0bfaf3e035 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam Binary files differindex cef39b84a4..d807b196f6 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam diff --git a/bootstrap/lib/kernel/ebin/application.beam b/bootstrap/lib/kernel/ebin/application.beam Binary files differindex 47e4cb5a17..a114030460 100644 --- a/bootstrap/lib/kernel/ebin/application.beam +++ b/bootstrap/lib/kernel/ebin/application.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex 5456b2be99..ec855896f8 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/application_master.beam b/bootstrap/lib/kernel/ebin/application_master.beam Binary files differindex 2efaa70849..a5c9e85fee 100644 --- a/bootstrap/lib/kernel/ebin/application_master.beam +++ b/bootstrap/lib/kernel/ebin/application_master.beam diff --git a/bootstrap/lib/kernel/ebin/application_starter.beam b/bootstrap/lib/kernel/ebin/application_starter.beam Binary files differindex 30495729c3..07e49873c9 100644 --- a/bootstrap/lib/kernel/ebin/application_starter.beam +++ b/bootstrap/lib/kernel/ebin/application_starter.beam diff --git a/bootstrap/lib/kernel/ebin/auth.beam b/bootstrap/lib/kernel/ebin/auth.beam Binary files differindex d20b71f629..7b341984b5 100644 --- a/bootstrap/lib/kernel/ebin/auth.beam +++ b/bootstrap/lib/kernel/ebin/auth.beam diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam Binary files differindex 3341d51436..17bfa6cde6 100644 --- a/bootstrap/lib/kernel/ebin/code.beam +++ b/bootstrap/lib/kernel/ebin/code.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 8ecba51c72..cecbda2361 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log.beam b/bootstrap/lib/kernel/ebin/disk_log.beam Binary files differindex 55e6550c16..ad26bc642b 100644 --- a/bootstrap/lib/kernel/ebin/disk_log.beam +++ b/bootstrap/lib/kernel/ebin/disk_log.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_1.beam b/bootstrap/lib/kernel/ebin/disk_log_1.beam Binary files differindex 16961789dd..082f521203 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_1.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_1.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_server.beam b/bootstrap/lib/kernel/ebin/disk_log_server.beam Binary files differindex cb13107464..18ee4b8de2 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_server.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_sup.beam b/bootstrap/lib/kernel/ebin/disk_log_sup.beam Binary files differindex 396c8e7b4c..001ecd70f7 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_sup.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_sup.beam diff --git a/bootstrap/lib/kernel/ebin/dist_ac.beam b/bootstrap/lib/kernel/ebin/dist_ac.beam Binary files differindex c9c404ae10..bd627326b3 100644 --- a/bootstrap/lib/kernel/ebin/dist_ac.beam +++ b/bootstrap/lib/kernel/ebin/dist_ac.beam diff --git a/bootstrap/lib/kernel/ebin/dist_util.beam b/bootstrap/lib/kernel/ebin/dist_util.beam Binary files differindex 0098d533ea..0a18fcc6ff 100644 --- a/bootstrap/lib/kernel/ebin/dist_util.beam +++ b/bootstrap/lib/kernel/ebin/dist_util.beam diff --git a/bootstrap/lib/kernel/ebin/erl_boot_server.beam b/bootstrap/lib/kernel/ebin/erl_boot_server.beam Binary files differindex 3d975d37a5..5091ee14d9 100644 --- a/bootstrap/lib/kernel/ebin/erl_boot_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_boot_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_compile_server.beam b/bootstrap/lib/kernel/ebin/erl_compile_server.beam Binary files differindex f00652d7f0..b3a57c5ffb 100644 --- a/bootstrap/lib/kernel/ebin/erl_compile_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_compile_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_ddll.beam b/bootstrap/lib/kernel/ebin/erl_ddll.beam Binary files differindex 8910f69874..e96e07f312 100644 --- a/bootstrap/lib/kernel/ebin/erl_ddll.beam +++ b/bootstrap/lib/kernel/ebin/erl_ddll.beam diff --git a/bootstrap/lib/kernel/ebin/erl_distribution.beam b/bootstrap/lib/kernel/ebin/erl_distribution.beam Binary files differindex cadf5947ad..81292271b0 100644 --- a/bootstrap/lib/kernel/ebin/erl_distribution.beam +++ b/bootstrap/lib/kernel/ebin/erl_distribution.beam diff --git a/bootstrap/lib/kernel/ebin/erl_epmd.beam b/bootstrap/lib/kernel/ebin/erl_epmd.beam Binary files differindex 9672775cb8..2bff8363b6 100644 --- a/bootstrap/lib/kernel/ebin/erl_epmd.beam +++ b/bootstrap/lib/kernel/ebin/erl_epmd.beam diff --git a/bootstrap/lib/kernel/ebin/erl_erts_errors.beam b/bootstrap/lib/kernel/ebin/erl_erts_errors.beam Binary files differindex ae8bc9a019..0e61b85ddc 100644 --- a/bootstrap/lib/kernel/ebin/erl_erts_errors.beam +++ b/bootstrap/lib/kernel/ebin/erl_erts_errors.beam diff --git a/bootstrap/lib/kernel/ebin/erl_kernel_errors.beam b/bootstrap/lib/kernel/ebin/erl_kernel_errors.beam Binary files differindex 0c6ab8f279..af96318d73 100644 --- a/bootstrap/lib/kernel/ebin/erl_kernel_errors.beam +++ b/bootstrap/lib/kernel/ebin/erl_kernel_errors.beam diff --git a/bootstrap/lib/kernel/ebin/erl_reply.beam b/bootstrap/lib/kernel/ebin/erl_reply.beam Binary files differindex 81ca50c86d..12387f9b2f 100644 --- a/bootstrap/lib/kernel/ebin/erl_reply.beam +++ b/bootstrap/lib/kernel/ebin/erl_reply.beam diff --git a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam Binary files differindex 6ed5a62960..93fcb7398f 100644 --- a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam +++ b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam diff --git a/bootstrap/lib/kernel/ebin/erpc.beam b/bootstrap/lib/kernel/ebin/erpc.beam Binary files differindex c081fa7ff5..f4d00f8b09 100644 --- a/bootstrap/lib/kernel/ebin/erpc.beam +++ b/bootstrap/lib/kernel/ebin/erpc.beam diff --git a/bootstrap/lib/kernel/ebin/error_handler.beam b/bootstrap/lib/kernel/ebin/error_handler.beam Binary files differindex 5e6481da1b..ed67292b09 100644 --- a/bootstrap/lib/kernel/ebin/error_handler.beam +++ b/bootstrap/lib/kernel/ebin/error_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_logger.beam b/bootstrap/lib/kernel/ebin/error_logger.beam Binary files differindex 2c4c9ae9e2..68c99180c8 100644 --- a/bootstrap/lib/kernel/ebin/error_logger.beam +++ b/bootstrap/lib/kernel/ebin/error_logger.beam diff --git a/bootstrap/lib/kernel/ebin/erts_debug.beam b/bootstrap/lib/kernel/ebin/erts_debug.beam Binary files differindex bba00e14cf..b7a8d8024b 100644 --- a/bootstrap/lib/kernel/ebin/erts_debug.beam +++ b/bootstrap/lib/kernel/ebin/erts_debug.beam diff --git a/bootstrap/lib/kernel/ebin/file.beam b/bootstrap/lib/kernel/ebin/file.beam Binary files differindex d69b38b480..1081d5f9b5 100644 --- a/bootstrap/lib/kernel/ebin/file.beam +++ b/bootstrap/lib/kernel/ebin/file.beam diff --git a/bootstrap/lib/kernel/ebin/file_io_server.beam b/bootstrap/lib/kernel/ebin/file_io_server.beam Binary files differindex cb8a5b7a37..d9a2cc42c6 100644 --- a/bootstrap/lib/kernel/ebin/file_io_server.beam +++ b/bootstrap/lib/kernel/ebin/file_io_server.beam diff --git a/bootstrap/lib/kernel/ebin/file_server.beam b/bootstrap/lib/kernel/ebin/file_server.beam Binary files differindex f4a6071ae3..6cde0ea009 100644 --- a/bootstrap/lib/kernel/ebin/file_server.beam +++ b/bootstrap/lib/kernel/ebin/file_server.beam diff --git a/bootstrap/lib/kernel/ebin/gen_sctp.beam b/bootstrap/lib/kernel/ebin/gen_sctp.beam Binary files differindex dfdbe10785..c5ecec0c0a 100644 --- a/bootstrap/lib/kernel/ebin/gen_sctp.beam +++ b/bootstrap/lib/kernel/ebin/gen_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_tcp.beam b/bootstrap/lib/kernel/ebin/gen_tcp.beam Binary files differindex 9ec1f4eadd..c8aa3453e0 100644 --- a/bootstrap/lib/kernel/ebin/gen_tcp.beam +++ b/bootstrap/lib/kernel/ebin/gen_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_tcp_socket.beam b/bootstrap/lib/kernel/ebin/gen_tcp_socket.beam Binary files differindex 88a50413f0..c9d726f7cc 100644 --- a/bootstrap/lib/kernel/ebin/gen_tcp_socket.beam +++ b/bootstrap/lib/kernel/ebin/gen_tcp_socket.beam diff --git a/bootstrap/lib/kernel/ebin/gen_udp.beam b/bootstrap/lib/kernel/ebin/gen_udp.beam Binary files differindex 48781ba99e..9938e78b23 100644 --- a/bootstrap/lib/kernel/ebin/gen_udp.beam +++ b/bootstrap/lib/kernel/ebin/gen_udp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_udp_socket.beam b/bootstrap/lib/kernel/ebin/gen_udp_socket.beam Binary files differindex bf54798ce9..8f2be0084f 100644 --- a/bootstrap/lib/kernel/ebin/gen_udp_socket.beam +++ b/bootstrap/lib/kernel/ebin/gen_udp_socket.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex 8bbc449610..4d0025d9d0 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/global_group.beam b/bootstrap/lib/kernel/ebin/global_group.beam Binary files differindex 23c6f9a52b..8feedb00b1 100644 --- a/bootstrap/lib/kernel/ebin/global_group.beam +++ b/bootstrap/lib/kernel/ebin/global_group.beam diff --git a/bootstrap/lib/kernel/ebin/global_search.beam b/bootstrap/lib/kernel/ebin/global_search.beam Binary files differindex d5afafae8f..a673a3b160 100644 --- a/bootstrap/lib/kernel/ebin/global_search.beam +++ b/bootstrap/lib/kernel/ebin/global_search.beam diff --git a/bootstrap/lib/kernel/ebin/group.beam b/bootstrap/lib/kernel/ebin/group.beam Binary files differindex 27539c6a71..9cd8511c1c 100644 --- a/bootstrap/lib/kernel/ebin/group.beam +++ b/bootstrap/lib/kernel/ebin/group.beam diff --git a/bootstrap/lib/kernel/ebin/group_history.beam b/bootstrap/lib/kernel/ebin/group_history.beam Binary files differindex 6bd7e769be..76dea08104 100644 --- a/bootstrap/lib/kernel/ebin/group_history.beam +++ b/bootstrap/lib/kernel/ebin/group_history.beam diff --git a/bootstrap/lib/kernel/ebin/heart.beam b/bootstrap/lib/kernel/ebin/heart.beam Binary files differindex c72c5b0282..f9363a9d68 100644 --- a/bootstrap/lib/kernel/ebin/heart.beam +++ b/bootstrap/lib/kernel/ebin/heart.beam diff --git a/bootstrap/lib/kernel/ebin/inet.beam b/bootstrap/lib/kernel/ebin/inet.beam Binary files differindex d985ee2ad6..8808fc1a02 100644 --- a/bootstrap/lib/kernel/ebin/inet.beam +++ b/bootstrap/lib/kernel/ebin/inet.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_sctp.beam b/bootstrap/lib/kernel/ebin/inet6_sctp.beam Binary files differindex 4efce5fba3..4fa972a9ff 100644 --- a/bootstrap/lib/kernel/ebin/inet6_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp.beam b/bootstrap/lib/kernel/ebin/inet6_tcp.beam Binary files differindex a96c63068e..09d2a321e3 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam Binary files differindex a18f62b192..8c491922d2 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_udp.beam b/bootstrap/lib/kernel/ebin/inet6_udp.beam Binary files differindex fd6e7f85c0..b7053fd88d 100644 --- a/bootstrap/lib/kernel/ebin/inet6_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_udp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_config.beam b/bootstrap/lib/kernel/ebin/inet_config.beam Binary files differindex 0eaaa71602..365c2ccaba 100644 --- a/bootstrap/lib/kernel/ebin/inet_config.beam +++ b/bootstrap/lib/kernel/ebin/inet_config.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex 4ce7dd6734..52ca62dbad 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex bddba494d9..625b92a668 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam Binary files differindex 751f7c14f6..7ae80b9cc2 100644 --- a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam +++ b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam diff --git a/bootstrap/lib/kernel/ebin/inet_hosts.beam b/bootstrap/lib/kernel/ebin/inet_hosts.beam Binary files differindex 5800de74dc..3c4597deae 100644 --- a/bootstrap/lib/kernel/ebin/inet_hosts.beam +++ b/bootstrap/lib/kernel/ebin/inet_hosts.beam diff --git a/bootstrap/lib/kernel/ebin/inet_parse.beam b/bootstrap/lib/kernel/ebin/inet_parse.beam Binary files differindex 397ec05343..7fb9263a6b 100644 --- a/bootstrap/lib/kernel/ebin/inet_parse.beam +++ b/bootstrap/lib/kernel/ebin/inet_parse.beam diff --git a/bootstrap/lib/kernel/ebin/inet_res.beam b/bootstrap/lib/kernel/ebin/inet_res.beam Binary files differindex f034500a1a..da139e5747 100644 --- a/bootstrap/lib/kernel/ebin/inet_res.beam +++ b/bootstrap/lib/kernel/ebin/inet_res.beam diff --git a/bootstrap/lib/kernel/ebin/inet_sctp.beam b/bootstrap/lib/kernel/ebin/inet_sctp.beam Binary files differindex 1f79fea238..cb59ed81df 100644 --- a/bootstrap/lib/kernel/ebin/inet_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp.beam b/bootstrap/lib/kernel/ebin/inet_tcp.beam Binary files differindex e46beb502c..0b910f339c 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam Binary files differindex 14b0c80cf3..b829199389 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet_udp.beam b/bootstrap/lib/kernel/ebin/inet_udp.beam Binary files differindex 0c04828317..9686efba4f 100644 --- a/bootstrap/lib/kernel/ebin/inet_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet_udp.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index 41e6a62a44..0a765b4ecc 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -109,6 +109,7 @@ inet_sctp, pg, pg2, + prim_tty, raw_file_io, raw_file_io_compressed, raw_file_io_deflate, @@ -158,6 +159,6 @@ {shell_docs_ansi,auto} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-@OTP-17934@", "stdlib-4.0", "sasl-3.0", "crypto-5.0"]} + {runtime_dependencies, ["erts-@OTP-17934@", "stdlib-@OTP-17932@", "sasl-3.0", "crypto-5.0"]} ] }. diff --git a/bootstrap/lib/kernel/ebin/kernel.appup b/bootstrap/lib/kernel/ebin/kernel.appup index cf88faffac..ccd0f7becd 100644 --- a/bootstrap/lib/kernel/ebin/kernel.appup +++ b/bootstrap/lib/kernel/ebin/kernel.appup @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2022. All Rights Reserved. +%% Copyright Ericsson AB 1999-2021. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,23 +19,16 @@ %% %% We allow upgrade from, and downgrade to all previous %% versions from the following OTP releases: -%% - OTP 22 %% - OTP 23 %% - OTP 24 +%% - OTP 25 %% %% We also allow upgrade from, and downgrade to all %% versions that have branched off from the above %% stated previous versions. %% -{"8.3.1", - [{<<"^6\\.4$">>,[restart_new_emulator]}, - {<<"^6\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^6\\.5$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^7\\.0$">>,[restart_new_emulator]}, +{"8.4.1", + [{<<"^7\\.0$">>,[restart_new_emulator]}, {<<"^7\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^7\\.1$">>,[restart_new_emulator]}, {<<"^7\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, @@ -57,15 +50,12 @@ {<<"^8\\.2$">>,[restart_new_emulator]}, {<<"^8\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^8\\.3$">>,[restart_new_emulator]}, - {<<"^8\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], - [{<<"^6\\.4$">>,[restart_new_emulator]}, - {<<"^6\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^6\\.5$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^6\\.5\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^7\\.0$">>,[restart_new_emulator]}, + {<<"^8\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^8\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^8\\.3\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^8\\.4$">>,[restart_new_emulator]}, + {<<"^8\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + [{<<"^7\\.0$">>,[restart_new_emulator]}, {<<"^7\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^7\\.1$">>,[restart_new_emulator]}, {<<"^7\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, @@ -87,4 +77,8 @@ {<<"^8\\.2$">>,[restart_new_emulator]}, {<<"^8\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^8\\.3$">>,[restart_new_emulator]}, - {<<"^8\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^8\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^8\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^8\\.3\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^8\\.4$">>,[restart_new_emulator]}, + {<<"^8\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/bootstrap/lib/kernel/ebin/kernel.beam b/bootstrap/lib/kernel/ebin/kernel.beam Binary files differindex 15eeffd50c..a652e7603b 100644 --- a/bootstrap/lib/kernel/ebin/kernel.beam +++ b/bootstrap/lib/kernel/ebin/kernel.beam diff --git a/bootstrap/lib/kernel/ebin/kernel_config.beam b/bootstrap/lib/kernel/ebin/kernel_config.beam Binary files differindex fc20a076c5..cafdb8ab65 100644 --- a/bootstrap/lib/kernel/ebin/kernel_config.beam +++ b/bootstrap/lib/kernel/ebin/kernel_config.beam diff --git a/bootstrap/lib/kernel/ebin/kernel_refc.beam b/bootstrap/lib/kernel/ebin/kernel_refc.beam Binary files differindex 85c941e15f..8eff0c04a2 100644 --- a/bootstrap/lib/kernel/ebin/kernel_refc.beam +++ b/bootstrap/lib/kernel/ebin/kernel_refc.beam diff --git a/bootstrap/lib/kernel/ebin/local_tcp.beam b/bootstrap/lib/kernel/ebin/local_tcp.beam Binary files differindex 57db9ec840..89e8ebc595 100644 --- a/bootstrap/lib/kernel/ebin/local_tcp.beam +++ b/bootstrap/lib/kernel/ebin/local_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/local_udp.beam b/bootstrap/lib/kernel/ebin/local_udp.beam Binary files differindex 67aadf1bc4..6ed0023f98 100644 --- a/bootstrap/lib/kernel/ebin/local_udp.beam +++ b/bootstrap/lib/kernel/ebin/local_udp.beam diff --git a/bootstrap/lib/kernel/ebin/logger.beam b/bootstrap/lib/kernel/ebin/logger.beam Binary files differindex 72301c71e9..81e72732a1 100644 --- a/bootstrap/lib/kernel/ebin/logger.beam +++ b/bootstrap/lib/kernel/ebin/logger.beam diff --git a/bootstrap/lib/kernel/ebin/logger_backend.beam b/bootstrap/lib/kernel/ebin/logger_backend.beam Binary files differindex 0c0a2720de..ee105779c2 100644 --- a/bootstrap/lib/kernel/ebin/logger_backend.beam +++ b/bootstrap/lib/kernel/ebin/logger_backend.beam diff --git a/bootstrap/lib/kernel/ebin/logger_config.beam b/bootstrap/lib/kernel/ebin/logger_config.beam Binary files differindex aeef730533..e0174b4af3 100644 --- a/bootstrap/lib/kernel/ebin/logger_config.beam +++ b/bootstrap/lib/kernel/ebin/logger_config.beam diff --git a/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam b/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam Binary files differindex 9b0de072dd..0f38e9d93d 100644 --- a/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_filters.beam b/bootstrap/lib/kernel/ebin/logger_filters.beam Binary files differindex d33846bd2a..63a3f30da2 100644 --- a/bootstrap/lib/kernel/ebin/logger_filters.beam +++ b/bootstrap/lib/kernel/ebin/logger_filters.beam diff --git a/bootstrap/lib/kernel/ebin/logger_formatter.beam b/bootstrap/lib/kernel/ebin/logger_formatter.beam Binary files differindex 693d6d96ae..c95768cc6e 100644 --- a/bootstrap/lib/kernel/ebin/logger_formatter.beam +++ b/bootstrap/lib/kernel/ebin/logger_formatter.beam diff --git a/bootstrap/lib/kernel/ebin/logger_h_common.beam b/bootstrap/lib/kernel/ebin/logger_h_common.beam Binary files differindex 22b9bc16b4..c691edf005 100644 --- a/bootstrap/lib/kernel/ebin/logger_h_common.beam +++ b/bootstrap/lib/kernel/ebin/logger_h_common.beam diff --git a/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam b/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam Binary files differindex 31c251072e..6607c6f7f5 100644 --- a/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam +++ b/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam diff --git a/bootstrap/lib/kernel/ebin/logger_olp.beam b/bootstrap/lib/kernel/ebin/logger_olp.beam Binary files differindex bf3dd071e4..78e61fe581 100644 --- a/bootstrap/lib/kernel/ebin/logger_olp.beam +++ b/bootstrap/lib/kernel/ebin/logger_olp.beam diff --git a/bootstrap/lib/kernel/ebin/logger_proxy.beam b/bootstrap/lib/kernel/ebin/logger_proxy.beam Binary files differindex 886518885a..4a9e22ea08 100644 --- a/bootstrap/lib/kernel/ebin/logger_proxy.beam +++ b/bootstrap/lib/kernel/ebin/logger_proxy.beam diff --git a/bootstrap/lib/kernel/ebin/logger_server.beam b/bootstrap/lib/kernel/ebin/logger_server.beam Binary files differindex 3b1f42dd85..7cc0328860 100644 --- a/bootstrap/lib/kernel/ebin/logger_server.beam +++ b/bootstrap/lib/kernel/ebin/logger_server.beam diff --git a/bootstrap/lib/kernel/ebin/logger_simple_h.beam b/bootstrap/lib/kernel/ebin/logger_simple_h.beam Binary files differindex cb5a4fb840..cdbd2d2f77 100644 --- a/bootstrap/lib/kernel/ebin/logger_simple_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_simple_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_std_h.beam b/bootstrap/lib/kernel/ebin/logger_std_h.beam Binary files differindex d2c7d0eec9..bd5c2e05d0 100644 --- a/bootstrap/lib/kernel/ebin/logger_std_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_std_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_sup.beam b/bootstrap/lib/kernel/ebin/logger_sup.beam Binary files differindex 45e372bd1e..eb5f288238 100644 --- a/bootstrap/lib/kernel/ebin/logger_sup.beam +++ b/bootstrap/lib/kernel/ebin/logger_sup.beam diff --git a/bootstrap/lib/kernel/ebin/net.beam b/bootstrap/lib/kernel/ebin/net.beam Binary files differindex 90bc08647f..b536b35ba9 100644 --- a/bootstrap/lib/kernel/ebin/net.beam +++ b/bootstrap/lib/kernel/ebin/net.beam diff --git a/bootstrap/lib/kernel/ebin/net_adm.beam b/bootstrap/lib/kernel/ebin/net_adm.beam Binary files differindex d75024222e..425071a303 100644 --- a/bootstrap/lib/kernel/ebin/net_adm.beam +++ b/bootstrap/lib/kernel/ebin/net_adm.beam diff --git a/bootstrap/lib/kernel/ebin/net_kernel.beam b/bootstrap/lib/kernel/ebin/net_kernel.beam Binary files differindex 0716e4835f..7f89c0f24d 100644 --- a/bootstrap/lib/kernel/ebin/net_kernel.beam +++ b/bootstrap/lib/kernel/ebin/net_kernel.beam diff --git a/bootstrap/lib/kernel/ebin/os.beam b/bootstrap/lib/kernel/ebin/os.beam Binary files differindex cc6d5a0122..ccb5b5e13e 100644 --- a/bootstrap/lib/kernel/ebin/os.beam +++ b/bootstrap/lib/kernel/ebin/os.beam diff --git a/bootstrap/lib/kernel/ebin/pg.beam b/bootstrap/lib/kernel/ebin/pg.beam Binary files differindex ac2242c8ff..75dd1f5000 100644 --- a/bootstrap/lib/kernel/ebin/pg.beam +++ b/bootstrap/lib/kernel/ebin/pg.beam diff --git a/bootstrap/lib/kernel/ebin/pg2.beam b/bootstrap/lib/kernel/ebin/pg2.beam Binary files differindex fdfdb68586..0aec7d1a98 100644 --- a/bootstrap/lib/kernel/ebin/pg2.beam +++ b/bootstrap/lib/kernel/ebin/pg2.beam diff --git a/bootstrap/lib/kernel/ebin/prim_tty.beam b/bootstrap/lib/kernel/ebin/prim_tty.beam Binary files differnew file mode 100644 index 0000000000..8df399df8d --- /dev/null +++ b/bootstrap/lib/kernel/ebin/prim_tty.beam diff --git a/bootstrap/lib/kernel/ebin/ram_file.beam b/bootstrap/lib/kernel/ebin/ram_file.beam Binary files differindex b228ce08ad..2955b13b79 100644 --- a/bootstrap/lib/kernel/ebin/ram_file.beam +++ b/bootstrap/lib/kernel/ebin/ram_file.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io.beam b/bootstrap/lib/kernel/ebin/raw_file_io.beam Binary files differindex 21f1fd1e8b..3bba7e2fd9 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam b/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam Binary files differindex 7b524c62e6..26778154ee 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam Binary files differindex 69037c86d3..081a3422b2 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam b/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam Binary files differindex 51b0dd4e83..2a53bb351a 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam Binary files differindex c61044d7f0..19fc64f093 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_list.beam b/bootstrap/lib/kernel/ebin/raw_file_io_list.beam Binary files differindex 4f68c9f208..efedbb73d0 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_list.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_list.beam diff --git a/bootstrap/lib/kernel/ebin/rpc.beam b/bootstrap/lib/kernel/ebin/rpc.beam Binary files differindex 65985f0b84..08f4185383 100644 --- a/bootstrap/lib/kernel/ebin/rpc.beam +++ b/bootstrap/lib/kernel/ebin/rpc.beam diff --git a/bootstrap/lib/kernel/ebin/seq_trace.beam b/bootstrap/lib/kernel/ebin/seq_trace.beam Binary files differindex 3420f59316..dfaadde1fb 100644 --- a/bootstrap/lib/kernel/ebin/seq_trace.beam +++ b/bootstrap/lib/kernel/ebin/seq_trace.beam diff --git a/bootstrap/lib/kernel/ebin/socket.beam b/bootstrap/lib/kernel/ebin/socket.beam Binary files differindex 8d9c933cb8..eaa3b8f717 100644 --- a/bootstrap/lib/kernel/ebin/socket.beam +++ b/bootstrap/lib/kernel/ebin/socket.beam diff --git a/bootstrap/lib/kernel/ebin/standard_error.beam b/bootstrap/lib/kernel/ebin/standard_error.beam Binary files differindex 6d8bb3ca9c..03190f25c8 100644 --- a/bootstrap/lib/kernel/ebin/standard_error.beam +++ b/bootstrap/lib/kernel/ebin/standard_error.beam diff --git a/bootstrap/lib/kernel/ebin/user.beam b/bootstrap/lib/kernel/ebin/user.beam Binary files differdeleted file mode 100644 index a75e13ad1f..0000000000 --- a/bootstrap/lib/kernel/ebin/user.beam +++ /dev/null diff --git a/bootstrap/lib/kernel/ebin/user_drv.beam b/bootstrap/lib/kernel/ebin/user_drv.beam Binary files differindex 94e7ecdf7e..9e51cded7a 100644 --- a/bootstrap/lib/kernel/ebin/user_drv.beam +++ b/bootstrap/lib/kernel/ebin/user_drv.beam diff --git a/bootstrap/lib/kernel/ebin/user_sup.beam b/bootstrap/lib/kernel/ebin/user_sup.beam Binary files differindex 40ee18f38d..de8f8b0ab0 100644 --- a/bootstrap/lib/kernel/ebin/user_sup.beam +++ b/bootstrap/lib/kernel/ebin/user_sup.beam diff --git a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam Binary files differindex ce175874ec..c54a828e4c 100644 --- a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam +++ b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam diff --git a/bootstrap/lib/kernel/include/dist.hrl b/bootstrap/lib/kernel/include/dist.hrl index 16320b64e9..4eef2bb3fc 100644 --- a/bootstrap/lib/kernel/include/dist.hrl +++ b/bootstrap/lib/kernel/include/dist.hrl @@ -72,6 +72,14 @@ ?DFLAG_BIG_CREATION bor ?DFLAG_HANDSHAKE_23)). +%% New mandatory flags in OTP 26 +-define(MANDATORY_DFLAGS_26, (?DFLAG_V4_NC bor + ?DFLAG_UNLINK_ID)). + +%% All mandatory flags +-define(DFLAGS_MANDATORY, (?MANDATORY_DFLAGS_25 bor + ?MANDATORY_DFLAGS_26)). + %% Also update dflag2str() in ../src/dist_util.erl %% when adding flags... diff --git a/bootstrap/lib/stdlib/ebin/array.beam b/bootstrap/lib/stdlib/ebin/array.beam Binary files differindex 524748417f..03a103d91a 100644 --- a/bootstrap/lib/stdlib/ebin/array.beam +++ b/bootstrap/lib/stdlib/ebin/array.beam diff --git a/bootstrap/lib/stdlib/ebin/base64.beam b/bootstrap/lib/stdlib/ebin/base64.beam Binary files differindex 11518d9f17..1d8586cd5b 100644 --- a/bootstrap/lib/stdlib/ebin/base64.beam +++ b/bootstrap/lib/stdlib/ebin/base64.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex 3a1295416a..00370697c2 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/binary.beam b/bootstrap/lib/stdlib/ebin/binary.beam Binary files differindex 5a66369e6f..a37278a491 100644 --- a/bootstrap/lib/stdlib/ebin/binary.beam +++ b/bootstrap/lib/stdlib/ebin/binary.beam diff --git a/bootstrap/lib/stdlib/ebin/c.beam b/bootstrap/lib/stdlib/ebin/c.beam Binary files differindex 7899bb9f36..e6bf444ca9 100644 --- a/bootstrap/lib/stdlib/ebin/c.beam +++ b/bootstrap/lib/stdlib/ebin/c.beam diff --git a/bootstrap/lib/stdlib/ebin/calendar.beam b/bootstrap/lib/stdlib/ebin/calendar.beam Binary files differindex cb9126dce9..411a7c086c 100644 --- a/bootstrap/lib/stdlib/ebin/calendar.beam +++ b/bootstrap/lib/stdlib/ebin/calendar.beam diff --git a/bootstrap/lib/stdlib/ebin/dets.beam b/bootstrap/lib/stdlib/ebin/dets.beam Binary files differindex d14c8463eb..3bff256bc1 100644 --- a/bootstrap/lib/stdlib/ebin/dets.beam +++ b/bootstrap/lib/stdlib/ebin/dets.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_server.beam b/bootstrap/lib/stdlib/ebin/dets_server.beam Binary files differindex e971702773..23a1d14b96 100644 --- a/bootstrap/lib/stdlib/ebin/dets_server.beam +++ b/bootstrap/lib/stdlib/ebin/dets_server.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_sup.beam b/bootstrap/lib/stdlib/ebin/dets_sup.beam Binary files differindex f2e6e08cc4..7313ea33ca 100644 --- a/bootstrap/lib/stdlib/ebin/dets_sup.beam +++ b/bootstrap/lib/stdlib/ebin/dets_sup.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_utils.beam b/bootstrap/lib/stdlib/ebin/dets_utils.beam Binary files differindex 952ea82f16..b4aeb9f4c6 100644 --- a/bootstrap/lib/stdlib/ebin/dets_utils.beam +++ b/bootstrap/lib/stdlib/ebin/dets_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_v9.beam b/bootstrap/lib/stdlib/ebin/dets_v9.beam Binary files differindex a98b4fbeac..eb98579e41 100644 --- a/bootstrap/lib/stdlib/ebin/dets_v9.beam +++ b/bootstrap/lib/stdlib/ebin/dets_v9.beam diff --git a/bootstrap/lib/stdlib/ebin/dict.beam b/bootstrap/lib/stdlib/ebin/dict.beam Binary files differindex e1a8cf4730..3880f721cf 100644 --- a/bootstrap/lib/stdlib/ebin/dict.beam +++ b/bootstrap/lib/stdlib/ebin/dict.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph.beam b/bootstrap/lib/stdlib/ebin/digraph.beam Binary files differindex b74ee31cb7..4c7da99050 100644 --- a/bootstrap/lib/stdlib/ebin/digraph.beam +++ b/bootstrap/lib/stdlib/ebin/digraph.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph_utils.beam b/bootstrap/lib/stdlib/ebin/digraph_utils.beam Binary files differindex 8e85e8c45b..63c1e5de20 100644 --- a/bootstrap/lib/stdlib/ebin/digraph_utils.beam +++ b/bootstrap/lib/stdlib/ebin/digraph_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin.beam b/bootstrap/lib/stdlib/ebin/edlin.beam Binary files differindex e043c8e023..f3635bd7f1 100644 --- a/bootstrap/lib/stdlib/ebin/edlin.beam +++ b/bootstrap/lib/stdlib/ebin/edlin.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_context.beam b/bootstrap/lib/stdlib/ebin/edlin_context.beam Binary files differnew file mode 100644 index 0000000000..3bef9b15a6 --- /dev/null +++ b/bootstrap/lib/stdlib/ebin/edlin_context.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_expand.beam b/bootstrap/lib/stdlib/ebin/edlin_expand.beam Binary files differindex c2c1b4d05d..8aa7fd1735 100644 --- a/bootstrap/lib/stdlib/ebin/edlin_expand.beam +++ b/bootstrap/lib/stdlib/ebin/edlin_expand.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_type_suggestion.beam b/bootstrap/lib/stdlib/ebin/edlin_type_suggestion.beam Binary files differnew file mode 100644 index 0000000000..05c05e61e7 --- /dev/null +++ b/bootstrap/lib/stdlib/ebin/edlin_type_suggestion.beam diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex 3a1b9b2c51..718b88506f 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam b/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam Binary files differindex 1a93db5293..d2f5bd7486 100644 --- a/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam +++ b/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_anno.beam b/bootstrap/lib/stdlib/ebin/erl_anno.beam Binary files differindex c454c5a045..b6a1357579 100644 --- a/bootstrap/lib/stdlib/ebin/erl_anno.beam +++ b/bootstrap/lib/stdlib/ebin/erl_anno.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_bits.beam b/bootstrap/lib/stdlib/ebin/erl_bits.beam Binary files differindex 61783aa8ac..9a57a886cd 100644 --- a/bootstrap/lib/stdlib/ebin/erl_bits.beam +++ b/bootstrap/lib/stdlib/ebin/erl_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_compile.beam b/bootstrap/lib/stdlib/ebin/erl_compile.beam Binary files differindex d099a6f6dd..3afba808bc 100644 --- a/bootstrap/lib/stdlib/ebin/erl_compile.beam +++ b/bootstrap/lib/stdlib/ebin/erl_compile.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_error.beam b/bootstrap/lib/stdlib/ebin/erl_error.beam Binary files differindex e379971ae5..e54d94c736 100644 --- a/bootstrap/lib/stdlib/ebin/erl_error.beam +++ b/bootstrap/lib/stdlib/ebin/erl_error.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex 67b6e845c4..5f2efa4b0c 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam Binary files differindex 9b1f56180c..a48dda552c 100644 --- a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam +++ b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_features.beam b/bootstrap/lib/stdlib/ebin/erl_features.beam Binary files differindex 514b284b0c..fdd5c35017 100644 --- a/bootstrap/lib/stdlib/ebin/erl_features.beam +++ b/bootstrap/lib/stdlib/ebin/erl_features.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_internal.beam b/bootstrap/lib/stdlib/ebin/erl_internal.beam Binary files differindex 3378f7da67..0098f1d24a 100644 --- a/bootstrap/lib/stdlib/ebin/erl_internal.beam +++ b/bootstrap/lib/stdlib/ebin/erl_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex 7d85aa27b1..14739c5421 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex 87f83e0785..4ef4082936 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam Binary files differindex b6a8d36cc4..4318cfe298 100644 --- a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam +++ b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_pp.beam b/bootstrap/lib/stdlib/ebin/erl_pp.beam Binary files differindex c92824793e..d361b50308 100644 --- a/bootstrap/lib/stdlib/ebin/erl_pp.beam +++ b/bootstrap/lib/stdlib/ebin/erl_pp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_scan.beam b/bootstrap/lib/stdlib/ebin/erl_scan.beam Binary files differindex 7539589cf2..4414e57a7f 100644 --- a/bootstrap/lib/stdlib/ebin/erl_scan.beam +++ b/bootstrap/lib/stdlib/ebin/erl_scan.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_stdlib_errors.beam b/bootstrap/lib/stdlib/ebin/erl_stdlib_errors.beam Binary files differindex cda90bf4a4..9ad263bc85 100644 --- a/bootstrap/lib/stdlib/ebin/erl_stdlib_errors.beam +++ b/bootstrap/lib/stdlib/ebin/erl_stdlib_errors.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_tar.beam b/bootstrap/lib/stdlib/ebin/erl_tar.beam Binary files differindex 5864c61c83..a9ca0c1755 100644 --- a/bootstrap/lib/stdlib/ebin/erl_tar.beam +++ b/bootstrap/lib/stdlib/ebin/erl_tar.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam Binary files differindex 0043d2ef08..87e48db836 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam Binary files differindex 21bd44dbad..ab2ca6dcc6 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam diff --git a/bootstrap/lib/stdlib/ebin/escript.beam b/bootstrap/lib/stdlib/ebin/escript.beam Binary files differindex 04ec6a6950..4c4178d469 100644 --- a/bootstrap/lib/stdlib/ebin/escript.beam +++ b/bootstrap/lib/stdlib/ebin/escript.beam diff --git a/bootstrap/lib/stdlib/ebin/ets.beam b/bootstrap/lib/stdlib/ebin/ets.beam Binary files differindex c48177baad..ed0db78520 100644 --- a/bootstrap/lib/stdlib/ebin/ets.beam +++ b/bootstrap/lib/stdlib/ebin/ets.beam diff --git a/bootstrap/lib/stdlib/ebin/eval_bits.beam b/bootstrap/lib/stdlib/ebin/eval_bits.beam Binary files differindex a91463ff94..23f19c2ef3 100644 --- a/bootstrap/lib/stdlib/ebin/eval_bits.beam +++ b/bootstrap/lib/stdlib/ebin/eval_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/file_sorter.beam b/bootstrap/lib/stdlib/ebin/file_sorter.beam Binary files differindex fd5139a87f..85c043e112 100644 --- a/bootstrap/lib/stdlib/ebin/file_sorter.beam +++ b/bootstrap/lib/stdlib/ebin/file_sorter.beam diff --git a/bootstrap/lib/stdlib/ebin/filelib.beam b/bootstrap/lib/stdlib/ebin/filelib.beam Binary files differindex 22007daa9c..76d9c1978c 100644 --- a/bootstrap/lib/stdlib/ebin/filelib.beam +++ b/bootstrap/lib/stdlib/ebin/filelib.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex 6720bb7dd1..9575010f02 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex 52d96aa43f..d59651319b 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_trees.beam b/bootstrap/lib/stdlib/ebin/gb_trees.beam Binary files differindex 5b37aa2d76..0bf850e29a 100644 --- a/bootstrap/lib/stdlib/ebin/gb_trees.beam +++ b/bootstrap/lib/stdlib/ebin/gb_trees.beam diff --git a/bootstrap/lib/stdlib/ebin/gen.beam b/bootstrap/lib/stdlib/ebin/gen.beam Binary files differindex 6966526786..b9cdddd66f 100644 --- a/bootstrap/lib/stdlib/ebin/gen.beam +++ b/bootstrap/lib/stdlib/ebin/gen.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_event.beam b/bootstrap/lib/stdlib/ebin/gen_event.beam Binary files differindex 96713a520c..32529a15a6 100644 --- a/bootstrap/lib/stdlib/ebin/gen_event.beam +++ b/bootstrap/lib/stdlib/ebin/gen_event.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_fsm.beam b/bootstrap/lib/stdlib/ebin/gen_fsm.beam Binary files differindex 5b4d9a3ebb..6b9eadb582 100644 --- a/bootstrap/lib/stdlib/ebin/gen_fsm.beam +++ b/bootstrap/lib/stdlib/ebin/gen_fsm.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_server.beam b/bootstrap/lib/stdlib/ebin/gen_server.beam Binary files differindex 7236b5869c..cfbe09341f 100644 --- a/bootstrap/lib/stdlib/ebin/gen_server.beam +++ b/bootstrap/lib/stdlib/ebin/gen_server.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_statem.beam b/bootstrap/lib/stdlib/ebin/gen_statem.beam Binary files differindex 3638309955..5af30739d4 100644 --- a/bootstrap/lib/stdlib/ebin/gen_statem.beam +++ b/bootstrap/lib/stdlib/ebin/gen_statem.beam diff --git a/bootstrap/lib/stdlib/ebin/io.beam b/bootstrap/lib/stdlib/ebin/io.beam Binary files differindex 0f5568c606..04848c9d11 100644 --- a/bootstrap/lib/stdlib/ebin/io.beam +++ b/bootstrap/lib/stdlib/ebin/io.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib.beam b/bootstrap/lib/stdlib/ebin/io_lib.beam Binary files differindex db1f8aa987..af67b77045 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_format.beam b/bootstrap/lib/stdlib/ebin/io_lib_format.beam Binary files differindex 4f12310de2..1cbf6863c6 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_format.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_format.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam Binary files differindex f25a79f330..8d40419740 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam Binary files differindex a8b6b41207..938ef98d07 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam diff --git a/bootstrap/lib/stdlib/ebin/lists.beam b/bootstrap/lib/stdlib/ebin/lists.beam Binary files differindex d321ccd6eb..2a74378d83 100644 --- a/bootstrap/lib/stdlib/ebin/lists.beam +++ b/bootstrap/lib/stdlib/ebin/lists.beam diff --git a/bootstrap/lib/stdlib/ebin/log_mf_h.beam b/bootstrap/lib/stdlib/ebin/log_mf_h.beam Binary files differindex 5a568cf644..b75fe299de 100644 --- a/bootstrap/lib/stdlib/ebin/log_mf_h.beam +++ b/bootstrap/lib/stdlib/ebin/log_mf_h.beam diff --git a/bootstrap/lib/stdlib/ebin/maps.beam b/bootstrap/lib/stdlib/ebin/maps.beam Binary files differindex 0504de1ac9..ec2abb4e3f 100644 --- a/bootstrap/lib/stdlib/ebin/maps.beam +++ b/bootstrap/lib/stdlib/ebin/maps.beam diff --git a/bootstrap/lib/stdlib/ebin/math.beam b/bootstrap/lib/stdlib/ebin/math.beam Binary files differindex e7c040cc4a..8352238539 100644 --- a/bootstrap/lib/stdlib/ebin/math.beam +++ b/bootstrap/lib/stdlib/ebin/math.beam diff --git a/bootstrap/lib/stdlib/ebin/ms_transform.beam b/bootstrap/lib/stdlib/ebin/ms_transform.beam Binary files differindex 7cd3c22c07..2611e6e597 100644 --- a/bootstrap/lib/stdlib/ebin/ms_transform.beam +++ b/bootstrap/lib/stdlib/ebin/ms_transform.beam diff --git a/bootstrap/lib/stdlib/ebin/orddict.beam b/bootstrap/lib/stdlib/ebin/orddict.beam Binary files differindex 9c642b7870..7bce6aaea8 100644 --- a/bootstrap/lib/stdlib/ebin/orddict.beam +++ b/bootstrap/lib/stdlib/ebin/orddict.beam diff --git a/bootstrap/lib/stdlib/ebin/ordsets.beam b/bootstrap/lib/stdlib/ebin/ordsets.beam Binary files differindex efa72b347e..8354c7886b 100644 --- a/bootstrap/lib/stdlib/ebin/ordsets.beam +++ b/bootstrap/lib/stdlib/ebin/ordsets.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex f2a9b45765..f2abda2dd1 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/peer.beam b/bootstrap/lib/stdlib/ebin/peer.beam Binary files differindex b35a4c56cd..3b61ad277e 100644 --- a/bootstrap/lib/stdlib/ebin/peer.beam +++ b/bootstrap/lib/stdlib/ebin/peer.beam diff --git a/bootstrap/lib/stdlib/ebin/pool.beam b/bootstrap/lib/stdlib/ebin/pool.beam Binary files differindex 566c6e90eb..a5611182f6 100644 --- a/bootstrap/lib/stdlib/ebin/pool.beam +++ b/bootstrap/lib/stdlib/ebin/pool.beam diff --git a/bootstrap/lib/stdlib/ebin/proc_lib.beam b/bootstrap/lib/stdlib/ebin/proc_lib.beam Binary files differindex 1a1f08b08f..85601d9298 100644 --- a/bootstrap/lib/stdlib/ebin/proc_lib.beam +++ b/bootstrap/lib/stdlib/ebin/proc_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/proplists.beam b/bootstrap/lib/stdlib/ebin/proplists.beam Binary files differindex aa386d7655..e10e09e960 100644 --- a/bootstrap/lib/stdlib/ebin/proplists.beam +++ b/bootstrap/lib/stdlib/ebin/proplists.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc.beam b/bootstrap/lib/stdlib/ebin/qlc.beam Binary files differindex 6663fb5c23..6e902b9fd1 100644 --- a/bootstrap/lib/stdlib/ebin/qlc.beam +++ b/bootstrap/lib/stdlib/ebin/qlc.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc_pt.beam b/bootstrap/lib/stdlib/ebin/qlc_pt.beam Binary files differindex 40cac7a2f9..13c6c712d6 100644 --- a/bootstrap/lib/stdlib/ebin/qlc_pt.beam +++ b/bootstrap/lib/stdlib/ebin/qlc_pt.beam diff --git a/bootstrap/lib/stdlib/ebin/queue.beam b/bootstrap/lib/stdlib/ebin/queue.beam Binary files differindex 0ffc140ea9..151f1fd60d 100644 --- a/bootstrap/lib/stdlib/ebin/queue.beam +++ b/bootstrap/lib/stdlib/ebin/queue.beam diff --git a/bootstrap/lib/stdlib/ebin/rand.beam b/bootstrap/lib/stdlib/ebin/rand.beam Binary files differindex a7938b8762..68f667694d 100644 --- a/bootstrap/lib/stdlib/ebin/rand.beam +++ b/bootstrap/lib/stdlib/ebin/rand.beam diff --git a/bootstrap/lib/stdlib/ebin/random.beam b/bootstrap/lib/stdlib/ebin/random.beam Binary files differindex e1ba2deacb..352af9d9f0 100644 --- a/bootstrap/lib/stdlib/ebin/random.beam +++ b/bootstrap/lib/stdlib/ebin/random.beam diff --git a/bootstrap/lib/stdlib/ebin/re.beam b/bootstrap/lib/stdlib/ebin/re.beam Binary files differindex 3e577c8768..bc1febf491 100644 --- a/bootstrap/lib/stdlib/ebin/re.beam +++ b/bootstrap/lib/stdlib/ebin/re.beam diff --git a/bootstrap/lib/stdlib/ebin/sets.beam b/bootstrap/lib/stdlib/ebin/sets.beam Binary files differindex a684a570bf..4fa9852839 100644 --- a/bootstrap/lib/stdlib/ebin/sets.beam +++ b/bootstrap/lib/stdlib/ebin/sets.beam diff --git a/bootstrap/lib/stdlib/ebin/shell.beam b/bootstrap/lib/stdlib/ebin/shell.beam Binary files differindex a6f9c3e3a4..65b1c8f928 100644 --- a/bootstrap/lib/stdlib/ebin/shell.beam +++ b/bootstrap/lib/stdlib/ebin/shell.beam diff --git a/bootstrap/lib/stdlib/ebin/shell_default.beam b/bootstrap/lib/stdlib/ebin/shell_default.beam Binary files differindex 167512d9f1..4a10f370e3 100644 --- a/bootstrap/lib/stdlib/ebin/shell_default.beam +++ b/bootstrap/lib/stdlib/ebin/shell_default.beam diff --git a/bootstrap/lib/stdlib/ebin/shell_docs.beam b/bootstrap/lib/stdlib/ebin/shell_docs.beam Binary files differindex 8fee06ffcd..6ca0be3659 100644 --- a/bootstrap/lib/stdlib/ebin/shell_docs.beam +++ b/bootstrap/lib/stdlib/ebin/shell_docs.beam diff --git a/bootstrap/lib/stdlib/ebin/slave.beam b/bootstrap/lib/stdlib/ebin/slave.beam Binary files differindex 30f8d8d154..e5082d6c52 100644 --- a/bootstrap/lib/stdlib/ebin/slave.beam +++ b/bootstrap/lib/stdlib/ebin/slave.beam diff --git a/bootstrap/lib/stdlib/ebin/sofs.beam b/bootstrap/lib/stdlib/ebin/sofs.beam Binary files differindex e99d617c91..ad05ff04ec 100644 --- a/bootstrap/lib/stdlib/ebin/sofs.beam +++ b/bootstrap/lib/stdlib/ebin/sofs.beam diff --git a/bootstrap/lib/stdlib/ebin/stdlib.app b/bootstrap/lib/stdlib/ebin/stdlib.app index ce39d8fbdb..b52aa908bf 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.app +++ b/bootstrap/lib/stdlib/ebin/stdlib.app @@ -112,6 +112,6 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-8.4","erts-@OTP-17934@","crypto-4.5", + {runtime_dependencies, ["sasl-3.0","kernel-@OTP-17932@","erts-@OTP-17934@","crypto-4.5", "compiler-5.0"]} ]}. diff --git a/bootstrap/lib/stdlib/ebin/stdlib.appup b/bootstrap/lib/stdlib/ebin/stdlib.appup index 5c7e569773..adebe16860 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.appup +++ b/bootstrap/lib/stdlib/ebin/stdlib.appup @@ -19,25 +19,16 @@ %% %% We allow upgrade from, and downgrade to all previous %% versions from the following OTP releases: -%% - OTP 22 %% - OTP 23 %% - OTP 24 +%% - OTP 25 %% %% We also allow upgrade from, and downgrade to all %% versions that have branched off from the above %% stated previous versions. %% -{"3.17.1", - [{<<"^3\\.10$">>,[restart_new_emulator]}, - {<<"^3\\.10\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.11$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.12$">>,[restart_new_emulator]}, - {<<"^3\\.12\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.12\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.13$">>,[restart_new_emulator]}, +{"4.0", + [{<<"^3\\.13$">>,[restart_new_emulator]}, {<<"^3\\.13\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.13\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.13\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,20 +45,9 @@ {<<"^3\\.16\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.17$">>,[restart_new_emulator]}, {<<"^3\\.17\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.9$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], - [{<<"^3\\.10$">>,[restart_new_emulator]}, - {<<"^3\\.10\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.11$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.11\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.12$">>,[restart_new_emulator]}, - {<<"^3\\.12\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.12\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.13$">>,[restart_new_emulator]}, + {<<"^3\\.17\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.17\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + [{<<"^3\\.13$">>,[restart_new_emulator]}, {<<"^3\\.13\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.13\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.13\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -84,7 +64,5 @@ {<<"^3\\.16\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^3\\.17$">>,[restart_new_emulator]}, {<<"^3\\.17\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.9$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, - {<<"^3\\.9\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^3\\.17\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.17\\.2(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex 330c4dd4ad..3a5addb75d 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex 8a8f890e79..8591df1888 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam Binary files differindex 007655f5cd..2327649b0a 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam diff --git a/bootstrap/lib/stdlib/ebin/sys.beam b/bootstrap/lib/stdlib/ebin/sys.beam Binary files differindex 509e06b790..947fa2a9ae 100644 --- a/bootstrap/lib/stdlib/ebin/sys.beam +++ b/bootstrap/lib/stdlib/ebin/sys.beam diff --git a/bootstrap/lib/stdlib/ebin/timer.beam b/bootstrap/lib/stdlib/ebin/timer.beam Binary files differindex 0191b0d795..feaf4c18cf 100644 --- a/bootstrap/lib/stdlib/ebin/timer.beam +++ b/bootstrap/lib/stdlib/ebin/timer.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode.beam b/bootstrap/lib/stdlib/ebin/unicode.beam Binary files differindex cac7d24cb1..9bdc9dcfef 100644 --- a/bootstrap/lib/stdlib/ebin/unicode.beam +++ b/bootstrap/lib/stdlib/ebin/unicode.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode_util.beam b/bootstrap/lib/stdlib/ebin/unicode_util.beam Binary files differindex 476c314d42..471da87256 100644 --- a/bootstrap/lib/stdlib/ebin/unicode_util.beam +++ b/bootstrap/lib/stdlib/ebin/unicode_util.beam diff --git a/bootstrap/lib/stdlib/ebin/uri_string.beam b/bootstrap/lib/stdlib/ebin/uri_string.beam Binary files differindex e8db2b5215..3c81214194 100644 --- a/bootstrap/lib/stdlib/ebin/uri_string.beam +++ b/bootstrap/lib/stdlib/ebin/uri_string.beam diff --git a/bootstrap/lib/stdlib/ebin/win32reg.beam b/bootstrap/lib/stdlib/ebin/win32reg.beam Binary files differindex 93c861c238..b6c2a409a5 100644 --- a/bootstrap/lib/stdlib/ebin/win32reg.beam +++ b/bootstrap/lib/stdlib/ebin/win32reg.beam diff --git a/bootstrap/lib/stdlib/ebin/zip.beam b/bootstrap/lib/stdlib/ebin/zip.beam Binary files differindex 01fbc51291..cad8e27b27 100644 --- a/bootstrap/lib/stdlib/ebin/zip.beam +++ b/bootstrap/lib/stdlib/ebin/zip.beam diff --git a/erts/.gitignore b/erts/.gitignore index 954e922492..e26eca8917 100644 --- a/erts/.gitignore +++ b/erts/.gitignore @@ -18,6 +18,7 @@ /emulator/test/Emakefile /emulator/test/*.beam /emulator/test/*_no_opt_SUITE.erl +/emulator/test/*_r25_SUITE.erl /emulator/pcre/pcre_exec_loop_break_cases.inc /emulator/beam/erl_db_insert_list.ycf.h diff --git a/erts/Makefile b/erts/Makefile index d6d9dee40d..a0f0dcfdb3 100644 --- a/erts/Makefile +++ b/erts/Makefile @@ -86,10 +86,8 @@ local_setup: cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl_call.exe $(ERL_TOP)/bin/erl_call.exe; \ - cp $(ERL_TOP)/bin/$(TARGET)/werl.exe $(ERL_TOP)/bin/werl.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/escript.exe $(ERL_TOP)/bin/escript.exe; \ - chmod 755 $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \ - $(ERL_TOP)/bin/werl.exe; \ + chmod 755 $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe; \ make_local_ini.sh $(ERL_TOP); \ cp $(ERL_TOP)/bin/erl.ini $(ERL_TOP)/bin/$(TARGET)/erl.ini; \ else \ diff --git a/erts/configure b/erts/configure index 815428caaf..f6a2ac1b49 100755 --- a/erts/configure +++ b/erts/configure @@ -10358,31 +10358,35 @@ printf "%s\n" "#define ETHR_WIN32_THREADS 1" >>confdefs.h else ilckd="_InterlockedDecrement" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedDecrement+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedDecrement=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10390,43 +10394,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedDecrement=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedDecrement" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedDecrement" >&6; } + if [ "${ethr_cv_have__InterlockedDecrement}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDDECREMENT 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedDecrement_rel" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedDecrement_rel+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedDecrement_rel=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10434,43 +10447,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedDecrement_rel=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedDecrement_rel" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedDecrement_rel" >&6; } + if [ "${ethr_cv_have__InterlockedDecrement_rel}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDDECREMENT_REL 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedIncrement" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedIncrement+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedIncrement=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10478,43 +10500,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedIncrement=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedIncrement" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedIncrement" >&6; } + if [ "${ethr_cv_have__InterlockedIncrement}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDINCREMENT 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedIncrement_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedIncrement_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedIncrement_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10522,43 +10553,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedIncrement_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedIncrement_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedIncrement_acq" >&6; } + if [ "${ethr_cv_have__InterlockedIncrement_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchangeAdd" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchangeAdd+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchangeAdd=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10566,43 +10606,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchangeAdd=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchangeAdd" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchangeAdd" >&6; } + if [ "${ethr_cv_have__InterlockedExchangeAdd}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGEADD 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchangeAdd_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchangeAdd_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchangeAdd_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10610,43 +10659,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchangeAdd_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchangeAdd_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchangeAdd_acq" >&6; } + if [ "${ethr_cv_have__InterlockedExchangeAdd_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedAnd" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedAnd+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedAnd=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10654,43 +10712,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedAnd=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedAnd" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedAnd" >&6; } + if [ "${ethr_cv_have__InterlockedAnd}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDAND 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedOr" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedOr+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedOr=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10698,43 +10765,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedOr=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedOr" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedOr" >&6; } + if [ "${ethr_cv_have__InterlockedOr}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDOR 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchange" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchange+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchange=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10742,43 +10818,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchange=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchange" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchange" >&6; } + if [ "${ethr_cv_have__InterlockedExchange}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGE 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedCompareExchange" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10786,44 +10871,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + ethr_have_native_atomics=yes + else + : + fi - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes ilckd="_InterlockedCompareExchange_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10831,44 +10925,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange_acq" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + ethr_have_native_atomics=yes + else + : + fi - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes ilckd="_InterlockedCompareExchange_rel" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (long) 0);";; "3") ilckd_call="${ilckd}(var, (long) 0, (long) 0);";; "4") ilckd_call="${ilckd}(var, (long) 0, (long) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange_rel+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange_rel=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile long *var; - volatile long arr[2]; + volatile long *var; + volatile long arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10876,45 +10979,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange_rel=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && -printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL 1" >>confdefs.h +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange_rel" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange_rel" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange_rel}" = "yes" ]; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } +printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL 1" >>confdefs.h - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ethr_have_native_atomics=yes + else + : + fi ilckd="_InterlockedDecrement64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedDecrement64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedDecrement64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10922,43 +11033,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedDecrement64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedDecrement64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedDecrement64" >&6; } + if [ "${ethr_cv_have__InterlockedDecrement64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDDECREMENT64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedDecrement64_rel" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedDecrement64_rel+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedDecrement64_rel=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -10966,43 +11086,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedDecrement64_rel=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedDecrement64_rel" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedDecrement64_rel" >&6; } + if [ "${ethr_cv_have__InterlockedDecrement64_rel}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDDECREMENT64_REL 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedIncrement64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedIncrement64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedIncrement64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11010,43 +11139,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedIncrement64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedIncrement64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedIncrement64" >&6; } + if [ "${ethr_cv_have__InterlockedIncrement64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDINCREMENT64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedIncrement64_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "1" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedIncrement64_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedIncrement64_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11054,43 +11192,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedIncrement64_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedIncrement64_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedIncrement64_acq" >&6; } + if [ "${ethr_cv_have__InterlockedIncrement64_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchangeAdd64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchangeAdd64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchangeAdd64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11098,43 +11245,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchangeAdd64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchangeAdd64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchangeAdd64" >&6; } + if [ "${ethr_cv_have__InterlockedExchangeAdd64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchangeAdd64_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchangeAdd64_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchangeAdd64_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11142,43 +11298,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchangeAdd64_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchangeAdd64_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchangeAdd64_acq" >&6; } + if [ "${ethr_cv_have__InterlockedExchangeAdd64_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedAnd64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedAnd64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedAnd64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11186,43 +11351,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedAnd64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedAnd64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedAnd64" >&6; } + if [ "${ethr_cv_have__InterlockedAnd64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDAND64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedOr64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedOr64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedOr64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11230,43 +11404,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedOr64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedOr64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedOr64" >&6; } + if [ "${ethr_cv_have__InterlockedOr64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDOR64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedExchange64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedExchange64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedExchange64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11274,43 +11457,52 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedExchange64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedExchange64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedExchange64" >&6; } + if [ "${ethr_cv_have__InterlockedExchange64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDEXCHANGE64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi ilckd="_InterlockedCompareExchange64" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange64+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange64=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11318,44 +11510,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange64=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange64" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange64" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange64}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + ethr_have_native_atomics=yes + else + : + fi - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes ilckd="_InterlockedCompareExchange64_acq" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange64_acq+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange64_acq=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11363,44 +11564,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange64_acq=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange64_acq" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange64_acq" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange64_acq}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + ethr_have_native_atomics=yes + else + : + fi - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes ilckd="_InterlockedCompareExchange64_rel" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "3" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange64_rel+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange64_rel=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11408,45 +11618,53 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange64_rel=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && -printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL 1" >>confdefs.h +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange64_rel" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange64_rel" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange64_rel}" = "yes" ]; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } +printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL 1" >>confdefs.h - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ethr_have_native_atomics=yes + else + : + fi ilckd="_InterlockedCompareExchange128" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 -printf %s "checking for ${ilckd}()... " >&6; } case "4" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, (__int64) 0);";; "3") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0);";; "4") ilckd_call="${ilckd}(var, (__int64) 0, (__int64) 0, arr);";; esac - have_interlocked_op=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ${ilckd}()" >&5 +printf %s "checking for ${ilckd}()... " >&6; } +if test ${ethr_cv_have__InterlockedCompareExchange128+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ethr_cv_have__InterlockedCompareExchange128=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> int main (void) { - volatile __int64 *var; - volatile __int64 arr[2]; + volatile __int64 *var; + volatile __int64 arr[2]; - $ilckd_call - return 0; + $ilckd_call + return 0; ; return 0; @@ -11454,15 +11672,20 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - have_interlocked_op=yes + ethr_cv_have__InterlockedCompareExchange128=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - test $have_interlocked_op = yes && +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ethr_cv_have__InterlockedCompareExchange128" >&5 +printf "%s\n" "$ethr_cv_have__InterlockedCompareExchange128" >&6; } + if [ "${ethr_cv_have__InterlockedCompareExchange128}" = "yes" ]; then + printf "%s\n" "#define ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128 1" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_interlocked_op" >&5 -printf "%s\n" "$have_interlocked_op" >&6; } + else + : + fi fi if test "$ethr_have_native_atomics" = "yes"; then @@ -16938,9 +17161,9 @@ then : try_dlpi_lib=$erl_xcomp_sysroot/lib if test x"$ac_cv_sizeof_void_p" = x"8"; then if test -d $erl_xcomp_sysroot/lib64; then - try_dlpi_lib= $erl_xcomp_sysroot/lib64 + try_dlpi_lib=$erl_xcomp_sysroot/lib64 elif test -d $erl_xcomp_sysroot/lib/64; then - try_dlpi_lib= $erl_xcomp_sysroot/lib/64 + try_dlpi_lib=$erl_xcomp_sysroot/lib/64 fi fi if test ! -f "$try_dlpi_lib/libdlpi.so" && \ @@ -25378,23 +25601,6 @@ printf "%s\n" "#define ERTS_EMU_CMDLINE_FLAGS \"$STATIC_CFLAGS $CFLAGS $DEBUG_CF -erts=${erl_top}/erts - -erts_dirs=" - $erts/obj $erts/obj.debug - - $erts/obj/$host - $erts/obj.debug/$host - -" -for d in ${erl_top}/bin ${erl_top}/bin/$host $erts_dirs ; -do - if test ! -d $d; then - mkdir -p 1>/dev/null 2>&1 $d - fi -done - - diff --git a/erts/configure.ac b/erts/configure.ac index ab2ee78acd..9ea98c25b0 100644 --- a/erts/configure.ac +++ b/erts/configure.ac @@ -1710,9 +1710,9 @@ AS_IF([test x"$ac_cv_lib_dlpi_dlpi_open" = x"no"], try_dlpi_lib=$erl_xcomp_sysroot/lib if test x"$ac_cv_sizeof_void_p" = x"8"; then if test -d $erl_xcomp_sysroot/lib64; then - try_dlpi_lib= $erl_xcomp_sysroot/lib64 + try_dlpi_lib=$erl_xcomp_sysroot/lib64 elif test -d $erl_xcomp_sysroot/lib/64; then - try_dlpi_lib= $erl_xcomp_sysroot/lib/64 + try_dlpi_lib=$erl_xcomp_sysroot/lib/64 fi fi if test ! -f "$try_dlpi_lib/libdlpi.so" && \ @@ -3468,26 +3468,6 @@ AC_DEFINE_UNQUOTED(ERTS_EMU_CMDLINE_FLAGS, AC_SUBST(STATIC_CFLAGS) -dnl ---------------------------------------------------------------------- -dnl Directories needed for the build -dnl ---------------------------------------------------------------------- - -erts=${erl_top}/erts - -erts_dirs=" - $erts/obj $erts/obj.debug - - $erts/obj/$host - $erts/obj.debug/$host - -" -for d in ${erl_top}/bin ${erl_top}/bin/$host $erts_dirs ; -do - if test ! -d $d; then - mkdir -p 1>/dev/null 2>&1 $d - fi -done - dnl --------------------------------------------------------------------- dnl Autoheader macro for adding code at top and bottom of config.h.in dnl --------------------------------------------------------------------- diff --git a/erts/doc/src/erl_cmd.xml b/erts/doc/src/erl_cmd.xml index 9f764d2b4e..b56caee569 100644 --- a/erts/doc/src/erl_cmd.xml +++ b/erts/doc/src/erl_cmd.xml @@ -540,12 +540,12 @@ $ <input>erl \ to implement an Alternative Carrier for the Erlang Distribution</seeguide>.</p> </item> - <tag><c><![CDATA[-noinput]]></c></tag> + <tag><marker id="noinput"/><c><![CDATA[-noinput]]></c></tag> <item> <p>Ensures that the Erlang runtime system never tries to read any input. Implies <c><![CDATA[-noshell]]></c>.</p> </item> - <tag><c><![CDATA[-noshell]]></c></tag> + <tag><marker id="noshell"/><c><![CDATA[-noshell]]></c></tag> <item> <p>Starts an Erlang runtime system with no shell. This flag makes it possible to have the Erlang runtime system as a @@ -1025,6 +1025,18 @@ $ <input>erl \ section in the BeamAsm internal documentation. </p> </item> + <tag><marker id="+JMsingle"/><c>+JMsingle true|false</c></tag> + <item> + <p>Enables or disables the use of single-mapped RWX memory for JIT + code. The default is to map JIT:ed machine code into two + regions sharing the same physical pages, where one region is + executable but not writable, and the other writable but not + executable. As some tools, such as QEMU user mode emulation, + cannot deal with the dual mapping, this flags allows it to be + disabled. This flag is automatically enabled by the + <seecom marker="#+JPperf"><c>+JPperf</c></seecom> flag. + </p> + </item> <tag><c><![CDATA[+L]]></c></tag> <item> <p>Prevents loading information about source filenames and line diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 0034534e1c..2236be5a63 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -1025,11 +1025,8 @@ DiB == gen_digest(ChA, ICA)? </item> <tag><marker id="DFLAG_UNLINK_ID"/><c>-define(DFLAG_UNLINK_ID, 16#2000000).</c></tag> <item> - <p>Use the <seeguide marker="#new_link_protocol">new link protocol</seeguide>.</p> - <note><p>This flag will become mandatory in OTP 26.</p></note> - <p>Unless both nodes have set the <c>DFLAG_UNLINK_ID</c> flag, the - <seeguide marker="#old_link_protocol">old link protocol</seeguide> - will be used as a fallback.</p> + <p>Use the <seeguide marker="#link_protocol">new link protocol</seeguide>.</p> + <note><p>This flag is mandatory as of OTP 26.</p></note> </item> <tag><c>-define(DFLAG_MANDATORY_25_DIGEST, (1 bsl 36)).</c></tag> <item> @@ -1061,8 +1058,7 @@ DiB == gen_digest(ChA, ICA)? <seeguide marker="erl_ext_dist#V4_PORT_EXT"><c>V4_PORT_EXT</c></seeguide>, and in the reference case up to 5 32-bit ID words are now accepted in <seeguide marker="erl_ext_dist#NEWER_REFERENCE_EXT"><c>NEWER_REFERENCE_EXT</c></seeguide>. - Introduced in OTP 24.</p> - <note><p>This flag will become mandatory in OTP 26.</p></note> + This flag was introduced in OTP 24 and became mandatory in OTP 26.</p> </item> <tag><marker id="DFLAG_ALIAS"/><c>-define(DFLAG_ALIAS, (1 bsl 35)).</c></tag> <item> @@ -1201,17 +1197,12 @@ DiB == gen_digest(ChA, ICA)? <p><c>{3, FromPid, ToPid, Reason}</c></p> <p>This signal is sent when a link has been broken</p> </item> - <tag><marker id="UNLINK"/><c>UNLINK</c> (deprecated)</tag> + <tag><marker id="UNLINK"/><c>UNLINK</c> (obsolete)</tag> <item> <p><c>{4, FromPid, ToPid}</c></p> - <p>This signal is sent by <c>FromPid</c> in order to remove - a link between <c>FromPid</c> and <c>ToPid</c>, when using the - <seeguide marker="#old_link_protocol">old link - protocol</seeguide>.</p> - <warning><p>This signal has been deprecated and will not - be supported in OTP 26. For more information see the - documentation of the - <seeguide marker="#new_link_protocol">new link protocol</seeguide>. + <warning><p>This signal is obsolete and not supported as of + OTP 26. For more information see the documentation of the + <seeguide marker="#link_protocol">link protocol</seeguide>. </p></warning> </item> <tag><c>NODE_LINK</c></tag> @@ -1501,11 +1492,9 @@ DiB == gen_digest(ChA, ICA)? among all not yet acknowledged <c>UNLINK_ID</c> signals from <c>FromPid</c> to <c>ToPid</c>.</p> <p> - This signal is only passed when the - <seeguide marker="#new_link_protocol">new link protocol</seeguide> - has been negotiated using the - <seeguide marker="erl_dist_protocol#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> - <seeguide marker="erl_dist_protocol#dflags">distribution flag</seeguide>. + This signal is part of the + <seeguide marker="#link_protocol">new link protocol</seeguide> + which became mandatory as of OTP 26. </p> </item> <tag><marker id="UNLINK_ID_ACK"/><c>UNLINK_ID_ACK</c></tag> @@ -1520,11 +1509,9 @@ DiB == gen_digest(ChA, ICA)? <c>ToPid</c> identifies the sender of the <c>UNLINK_ID</c> signal.</p> <p> - This signal is only passed when the - <seeguide marker="#new_link_protocol">new link protocol</seeguide> - has been negotiated using the - <seeguide marker="erl_dist_protocol#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> - <seeguide marker="erl_dist_protocol#dflags">distribution flag</seeguide>. + This signal is part of the + <seeguide marker="#link_protocol">new link protocol</seeguide> + which became mandatory as of OTP 26. </p> </item> </taglist> @@ -1555,200 +1542,173 @@ DiB == gen_digest(ChA, ICA)? </section> <section> <marker id="link_protocol"/> + <!-- The following markers kept in order not to break links + from the outside world... --> + <marker id="new_link_protocol"/> + <marker id="old_link_protocol"/> <title>Link Protocol</title> - <section> - <marker id="new_link_protocol"/> - <title>New Link Protocol</title> - - <p> - The new link protocol will be used when both nodes flag that - they understand it using the - <seeguide marker="#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> - <seeguide marker="#dflags">distribution flag</seeguide>. If - one of the nodes does not understand the new link protocol, the - <seeguide marker="#old_link_protocol">old link protocol</seeguide> - will be used as a fallback. - </p> - - <p> - The new link protocol introduces two new signals, - <seeguide marker="#UNLINK_ID"><c>UNLINK_ID</c></seeguide> and - <seeguide marker="#UNLINK_ID"><c>UNLINK_ID_ACK</c></seeguide>, - which replace the old - <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide> - signal. The old <seeguide marker="#LINK"><c>LINK</c></seeguide> - signal is still sent in order to set up a link, but handled - differently upon reception. - </p> - - <p> - In order to set up a link, a <c>LINK</c> signal is sent, from - the process initiating the operation, to the process that it - wants to link to. In order to remove a link, an - <c>UNLINK_ID</c> signal is sent, from the process initiating - the operation, to the linked process. The receiver of an - <c>UNLINK_ID</c> signal responds with an <c>UNLINK_ID_ACK</c> - signal. Upon reception of an <c>UNLINK_ID</c> signal, the - corresponding <c>UNLINK_ID_ACK</c> signal <em>must</em> be - sent before any other signals are sent to the sender of the - <c>UNLINK_ID</c> signal. Together with - <seeguide marker="system/reference_manual:processes#signal-delivery">the - signal ordering guarantee</seeguide> of Erlang this makes it - possible for the sender of the <c>UNLINK_ID</c> signal to know - the order of other signals which is essential for the protocol. - The <c>UNLINK_ID_ACK</c> signal should contain the same - <c>Id</c> as the <c>Id</c> contained in the <c>UNLINK_ID</c> - signal being acknowledged. - </p> + <p> + The new link protocol introduced in OTP 23.3 became mandatory + as of OTP 26. As of OTP 26, OTP nodes will therefor refuse to + connect to nodes that do not indicate that they support the + new link protocol using the + <seeguide marker="#DFLAG_UNLINK_ID"><c>DFLAG_UNLINK_ID</c></seeguide> + <seeguide marker="#dflags">distribution flag</seeguide>. + </p> - <p> - Processes also need to maintain process local information about - links. The state of this process local information is changed - when the signals above are sent and received. This process - local information also determines if a signal should be sent - when a process calls - <seemfa marker="erlang#link/1"><c>link/1</c></seemfa> or - <seemfa marker="erlang#unlink/1"><c>unlink/1</c></seemfa>. - A <c>LINK</c> signal is only sent if there does not currently - exist an active link between the processes according to the - process local information and an <c>UNLINK_ID</c> signal is - only sent if there currently exists an active link between the - processes according to the process local information. - </p> + <p> + The new link protocol introduced two new signals, + <seeguide marker="#UNLINK_ID"><c>UNLINK_ID</c></seeguide> and + <seeguide marker="#UNLINK_ID"><c>UNLINK_ID_ACK</c></seeguide>, + which replaced the old + <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide> + signal. The old <seeguide marker="#LINK"><c>LINK</c></seeguide> + signal is still sent in order to set up a link, but handled + differently upon reception. + </p> - <p> - The process local information about a link contains: - </p> - <taglist> - <tag>Pid</tag> - <item> - Process identifier of the linked process. - </item> - <tag>Active Flag</tag> - <item> - If set, the link is active and the process will react on - <seeguide marker="system/reference_manual:processes#receiving_exit_signals">incoming - exit signals</seeguide> issued due to the link. If not set, - the link is inactive and incoming exit signals, issued due - to the link, will be ignored. That is, the processes are - considered as <em>not</em> linked. - </item> - <tag>Unlink Id</tag> - <item> - Identifier of an outstanding unlink operation. That is, - an unlink operation that has not yet been acknowledged. - This information is only used when the active flag is not - set. - </item> - </taglist> + <p> + In order to set up a link, a <c>LINK</c> signal is sent, from + the process initiating the operation, to the process that it + wants to link to. In order to remove a link, an + <c>UNLINK_ID</c> signal is sent, from the process initiating + the operation, to the linked process. The receiver of an + <c>UNLINK_ID</c> signal responds with an <c>UNLINK_ID_ACK</c> + signal. Upon reception of an <c>UNLINK_ID</c> signal, the + corresponding <c>UNLINK_ID_ACK</c> signal <em>must</em> be + sent before any other signals are sent to the sender of the + <c>UNLINK_ID</c> signal. Together with + <seeguide marker="system/reference_manual:processes#signal-delivery">the + signal ordering guarantee</seeguide> of Erlang this makes it + possible for the sender of the <c>UNLINK_ID</c> signal to know + the order of other signals which is essential for the protocol. + The <c>UNLINK_ID_ACK</c> signal should contain the same + <c>Id</c> as the <c>Id</c> contained in the <c>UNLINK_ID</c> + signal being acknowledged. + </p> - <p> - A process is only considered linked to another process - if it has process local information about the link - containing the process identifier of the other process and - with the active flag set. - </p> + <p> + Processes also need to maintain process local information about + links. The state of this process local information is changed + when the signals above are sent and received. This process + local information also determines if a signal should be sent + when a process calls + <seemfa marker="erlang#link/1"><c>link/1</c></seemfa> or + <seemfa marker="erlang#unlink/1"><c>unlink/1</c></seemfa>. + A <c>LINK</c> signal is only sent if there does not currently + exist an active link between the processes according to the + process local information and an <c>UNLINK_ID</c> signal is + only sent if there currently exists an active link between the + processes according to the process local information. + </p> - <p> - The process local information about a link is updated as - follows: - </p> - <taglist> - <tag>A <c>LINK</c> signal is sent</tag> - <item> - Link information is created if not already existing. The - active flag is set, and unlink id is cleared. That is, - if we had an outstanding unlink operation we will ignore - the result of that operation and enable the link. - </item> - <tag>A <c>LINK</c> signal is received</tag> - <item> - If no link information already exists, it is created, the - active flag is set and unlink id is cleared. If the link - information already exists, the signal is silently ignored, - regardless of whether the active flag is set or not. - That is, if we have an outstanding unlink operation we will - <em>not</em> activate the link. In this scenario, the sender - of the <c>LINK</c> signal has not yet sent an - <c>UNLINK_ID_ACK</c> signal corresponding to our - <c>UNLINK_ID</c> signal which means that it will receive - our <c>UNLINK_ID</c> signal after it sent its - <c>LINK</c> signal. This in turn means that both processes - in the end will agree that there is no link between them. - </item> - <tag>An <c>UNLINK_ID</c> signal is sent</tag> - <item> - Link information already exists and the active flag is set - (otherwise the signal would not be sent). The active flag - is unset, and the unlink id of the signal is saved in the - link information. - </item> - <tag>An <c>UNLINK_ID</c> signal is received</tag> - <item> - If the active flag is set, information about the link - is removed. If the active flag is not set (that is, we have - an outstanding unlink operation), the information about the - link is left unchanged. - </item> - <tag>An <c>UNLINK_ID_ACK</c> signal is sent</tag> - <item> - This is done when an <c>UNLINK_ID</c> signal is received and - causes no further changes of the link information. - </item> - <tag>An <c>UNLINK_ID_ACK</c> signal is received</tag> - <item> - If information about the link exists, the active flag is not - set, and the unlink id in the link information equals the - <c>Id</c> in the signal, the link information is removed; - otherwise, the signal is ignored. - </item> - </taglist> + <p> + The process local information about a link contains: + </p> + <taglist> + <tag>Pid</tag> + <item> + Process identifier of the linked process. + </item> + <tag>Active Flag</tag> + <item> + If set, the link is active and the process will react on + <seeguide marker="system/reference_manual:processes#receiving_exit_signals">incoming + exit signals</seeguide> issued due to the link. If not set, + the link is inactive and incoming exit signals, issued due + to the link, will be ignored. That is, the processes are + considered as <em>not</em> linked. + </item> + <tag>Unlink Id</tag> + <item> + Identifier of an outstanding unlink operation. That is, + an unlink operation that has not yet been acknowledged. + This information is only used when the active flag is not + set. + </item> + </taglist> - <p> - When a process receives an exit signal due to a link, the - process will first react to the exit signal if the link - is active and then remove the process local information about - the link. - </p> - <p> - In case the connection is lost between two nodes, exit signals - with exit reason <c>noconnection</c> are sent to all processes - with links over the connection. This will cause all process - local information about links over the connection to be - removed. - </p> - <p> - Exactly the same link protocol is also used internally on an - Erlang node. The signals however have different formats since - they do not have to be sent over the wire. - </p> - </section> + <p> + A process is only considered linked to another process + if it has process local information about the link + containing the process identifier of the other process and + with the active flag set. + </p> - <section> - <marker id="old_link_protocol"/> - <title>Old Link Protocol</title> + <p> + The process local information about a link is updated as + follows: + </p> + <taglist> + <tag>A <c>LINK</c> signal is sent</tag> + <item> + Link information is created if not already existing. The + active flag is set, and unlink id is cleared. That is, + if we had an outstanding unlink operation we will ignore + the result of that operation and enable the link. + </item> + <tag>A <c>LINK</c> signal is received</tag> + <item> + If no link information already exists, it is created, the + active flag is set and unlink id is cleared. If the link + information already exists, the signal is silently ignored, + regardless of whether the active flag is set or not. + That is, if we have an outstanding unlink operation we will + <em>not</em> activate the link. In this scenario, the sender + of the <c>LINK</c> signal has not yet sent an + <c>UNLINK_ID_ACK</c> signal corresponding to our + <c>UNLINK_ID</c> signal which means that it will receive + our <c>UNLINK_ID</c> signal after it sent its + <c>LINK</c> signal. This in turn means that both processes + in the end will agree that there is no link between them. + </item> + <tag>An <c>UNLINK_ID</c> signal is sent</tag> + <item> + Link information already exists and the active flag is set + (otherwise the signal would not be sent). The active flag + is unset, and the unlink id of the signal is saved in the + link information. + </item> + <tag>An <c>UNLINK_ID</c> signal is received</tag> + <item> + If the active flag is set, information about the link + is removed. If the active flag is not set (that is, we have + an outstanding unlink operation), the information about the + link is left unchanged. + </item> + <tag>An <c>UNLINK_ID_ACK</c> signal is sent</tag> + <item> + This is done when an <c>UNLINK_ID</c> signal is received and + causes no further changes of the link information. + </item> + <tag>An <c>UNLINK_ID_ACK</c> signal is received</tag> + <item> + If information about the link exists, the active flag is not + set, and the unlink id in the link information equals the + <c>Id</c> in the signal, the link information is removed; + otherwise, the signal is ignored. + </item> + </taglist> - <p> - The old link protocol utilize two signals - <seeguide marker="#LINK"><c>LINK</c></seeguide>, and - <seeguide marker="#UNLINK"><c>UNLINK</c></seeguide>. The - <c>LINK</c> signal informs the other process that a link - should be set up, and the <c>UNLINK</c> signal informs the - other process that a link should be removed. This protocol - is however a bit too naive. If both processes operate on the - link simultaneously, the link may end up in an inconsistent - state where one process thinks it is linked while the other - thinks it is not linked. - </p> - <p> - This protocol is deprecated and support for it will be removed - in OTP 26. Until then, it will be used as fallback when - communicating with old nodes that do not understand the - <seeguide marker="#new_link_protocol">new link - protocol</seeguide>. - </p> - </section> + <p> + When a process receives an exit signal due to a link, the + process will first react to the exit signal if the link + is active and then remove the process local information about + the link. + </p> + <p> + In case the connection is lost between two nodes, exit signals + with exit reason <c>noconnection</c> are sent to all processes + with links over the connection. This will cause all process + local information about links over the connection to be + removed. + </p> + <p> + Exactly the same link protocol is also used internally on an + Erlang node. The signals however have different formats since + they do not have to be sent over the wire. + </p> </section> </section> diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index b4b245187d..7cf4ae3aa1 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -655,6 +655,7 @@ encoded using <c>NEW_PORT_EXT</c>, even external ports received as <seeguide marker="#PORT_EXT"><c>PORT_EXT</c></seeguide> from older nodes. </p> + </section> <section> @@ -683,8 +684,10 @@ works just like in <seeguide marker="#NEW_PID_EXT"><c>NEW_PID_EXT</c></seeguide>. Port operations are not allowed across node boundaries. </p> - <p><c>V4_PORT_EXT</c> was introduced in OTP 24, but only to be decoded - and echoed back. Not encoded for local ports. + <p>In OTP 26 distribution flag + <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> + as well as <c>V4_PORT_EXT</c> became mandatory accepting full + 64-bit ports to be decoded and echoed back. </p> </section> @@ -742,14 +745,10 @@ <seeguide marker="#utf8_atoms">encoded as an atom</seeguide>.</p> </item> <tag><c>ID</c></tag> - <item><p>A 32-bit big endian unsigned integer. If distribution flag - <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> - is not set, only 15 bits may be used and the rest must be 0.</p> + <item><p>A 32-bit big endian unsigned integer.</p> </item> <tag><c>Serial</c></tag> - <item><p>A 32-bit big endian unsigned integer. If distribution flag - <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> - is not set, only 13 bits may be used and the rest must be 0.</p> + <item><p>A 32-bit big endian unsigned integer.</p> </item> <tag><c>Creation</c></tag> <item><p>A 32-bit big endian unsigned integer. All identifiers @@ -768,9 +767,10 @@ even external pids received as <seeguide marker="#PID_EXT"><c>PID_EXT</c></seeguide> from older nodes. </p> - <p>In OTP 24 distribution flag + <p>In OTP 26 distribution flag <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> - was introduced, accepting full 64-bit pids to be decoded and echoed back. + became mandatory accepting full 64-bit pids to be decoded + and echoed back. </p> </section> @@ -1080,9 +1080,7 @@ <seeguide marker="#utf8_atoms">encoded as an atom</seeguide>.</p> </item> <tag><c>Len</c></tag> - <item><p>A 16-bit big endian unsigned integer not larger than 5 when the - <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> - has been set; otherwise not larger than 3.</p> + <item><p>A 16-bit big endian unsigned integer not larger than 5.</p> </item> <tag><c>ID</c></tag> <item><p>A sequence of <c>Len</c> big-endian unsigned integers @@ -1104,6 +1102,9 @@ <seeguide marker="#NEW_REFERENCE_EXT"><c>NEW_REFERENCE_EXT</c></seeguide> from older nodes. </p> + <p>In OTP 26 distribution flag + <seeguide marker="erl_dist_protocol#DFLAG_V4_NC"><c>DFLAG_V4_NC</c></seeguide> + became mandatory. References now can contain up to 5 <c>ID</c> words.</p> </section> <section> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 4208aa269a..cb300a976d 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -785,6 +785,41 @@ typedef struct { To compare two monitors, <seecref marker="#enif_compare_monitors"> <c>enif_compare_monitors</c></seecref> must be used.</p> </item> + <tag><marker id="ErlNifOnHaltCallback"/><c>ErlNifOnHaltCallback</c></tag> + <item> + <code type="none"> +typedef void ErlNifOnHaltCallback(void *priv_data);</code> + <p> + The function prototype of an <i>on halt</i> callback function. + </p> + <p> + An <i>on halt</i> callback can be installed using + <seecref marker="#on_halt"><c>enif_set_option()</c></seecref>. Such + an installed callback will be called when the runtime system is + halting. + </p> + </item> + <tag><marker id="ErlNifOption"/><c>ErlNifOption</c></tag> + <item> + <p> + An enumeration of the options that can be set using + <seecref marker="#enif_set_option"><c>enif_set_option()</c></seecref>. + </p> + <p>Currently valid options:</p> + <taglist> + <tag><seecref marker="#delay_halt"><c>ERL_NIF_OPT_DELAY_HALT</c></seecref></tag> + <item><p> + Enable delay of runtime system halt with flushing enabled until + all calls to NIFs in the NIF library have returned. + </p></item> + <tag><seecref marker="#on_halt"><c>ERL_NIF_OPT_ON_HALT</c></seecref></tag> + <item><p> + Install a callback that will be called when the runtime system + halts with flushing enabled. + </p></item> + </taglist> + </item> + <tag><marker id="ErlNifPid"/><c>ErlNifPid</c></tag> <item> <p>A process identifier (pid). In contrast to pid terms (instances of @@ -3413,6 +3448,146 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { </func> <func> + <name since="OTP @OTP-17771@"> + <ret>int</ret><nametext>enif_set_option(ErlNifEnv *env, ErlNifOption opt, ...)</nametext> + </name> + <fsummary> + Set an option. + </fsummary> + <desc> + <marker id="enif_set_option"/> + <p> + Set an option. On success, zero will be returned. On failure, a non + zero value will be returned. Currently the following options can be set: + </p> + <taglist> + <tag><marker id="delay_halt"/> + <c>int enif_set_option(ErlNifEnv *env, + </c><seecref marker="#ErlNifOption"><c>ERL_NIF_OPT_DELAY_HALT</c></seecref><c>)</c> + </tag> + <item> + <p> + Enable delay of runtime system halt with flushing enabled until + all calls to NIFs in the NIF library have returned. If the + <i>delay halt</i> feature has not been enabled, a halt with + flushing enabled may complete even though processes are still + executing inside NIFs in the NIF library. Note that by + <i>returning</i> we here mean the first point where the NIF + returns control back to the runtime system, and <em>not</em> the + point where a call to a NIF return a value back to the Erlang + code that called the NIF. That is, if you schedule execution of + a NIF, using <seecref marker="#enif_schedule_nif"> + <c>enif_schedule_nif()</c></seecref>, from within a NIF while + the system is halting, the scheduled NIF call will <em>not</em> + be executed even though <i>delay halt</i> has been enabled for + the NIF library. + </p> + <p> + The runtime system halts when one of the + <seemfa marker="erlang#halt/0"><c>erlang:halt()</c></seemfa> + BIFs are called. By default flushing is enabled, but can be + disabled using the + <seemfa marker="erlang#halt/2"><c>erlang:halt/2</c></seemfa> BIF. + When flushing has been disabled, the <i>delay halt</i> setting + will have no effect. That is, the runtime system will halt without + waiting for NIFs to return even if the <i>delay halt</i> setting + has been enabled. See the + <seeerl marker="erlang#halt_flush"><c>{flush, boolean()}</c></seeerl> + option of <c>erlang:halt/2</c> for more information. + </p> + <p> + The <c>ERL_NIF_OPT_DELAY_HALT</c> option can only be set during + loading of a NIF library in a call to <c>enif_set_option()</c> + inside a NIF library + <seecref marker="#load"><c>load()</c></seecref> or + <seecref marker="#upgrade"><c>upgrade()</c></seecref> call, + and will fail if set somewhere else. The <c>env</c> + argument <i>must</i> be the callback environment passed to the + <c>load()</c> or the <c>upgrade()</c> call. This option can also + only be set once. That is, the <i>delay halt</i> setting cannot + be changed once it has been enabled. The <i>delay halt</i> + setting is tied to the module instance with which the NIF library + instance has been loaded. That is, in case both a new and old + version of a module using the NIF library are loaded, they can + have the same or different <i>delay halt</i> settings. + </p> + <p> + The <i>delay halt</i> feature can be used in combination with an + <seecref marker="#on_halt"><i>on halt</i></seecref> callback. + The <i>on halt</i> callback is in this case typically used to + notify processes blocked in NIFs in the library that it is time + to return in order to let the runtime system complete the + halting. Such NIFs should be dirty NIFs, since ordinary NIFs + should never block for a long time. + </p> + </item> + <tag><marker id="on_halt"/> + <c>int enif_set_option(ErlNifEnv *env, + </c><seecref marker="#ErlNifOption"><c>ERL_NIF_OPT_ON_HALT</c></seecref><c>, + </c><seecref marker="#ErlNifOnHaltCallback"><c>ErlNifOnHaltCallback</c></seecref><c> + *on_halt)</c> + </tag> + <item> + <p> + Install a callback that will be called when the runtime system + halts with flushing enabled. + </p> + <p> + The runtime system halts when one of the + <seemfa marker="erlang#halt/0"><c>erlang:halt()</c></seemfa> BIFs + are called. By default flushing is enabled, but can be disabled + using the + <seemfa marker="erlang#halt/2"><c>erlang:halt/2</c></seemfa> BIF. + When flushing has been disabled, the runtime system will halt + without calling any <i>on halt</i> callbacks even if such are + installed. See the + <seeerl marker="erlang#halt_flush"><c>{flush, boolean()}</c></seeerl> + option of <c>erlang:halt/2</c> for more information. + </p> + <p> + The <c>ERL_NIF_OPT_ON_HALT</c> option can only be set during + loading of a NIF library in a call to <c>enif_set_option()</c> + inside a NIF library + <seecref marker="#load"><c>load()</c></seecref> or + <seecref marker="#upgrade"><c>upgrade()</c></seecref> call, + and will fail if called somewhere else. The <c>env</c> + argument <i>must</i> be the callback environment passed to the + <c>load()</c> or the <c>upgrade()</c> call. The <c>on_halt</c> + argument should be a function pointer to the callback to install. + The <i>on halt</i> callback will be tied to the module instance + with which the NIF library instance has been loaded. That is, in + case both a new and old version of a module using the NIF library + are loaded, they can both have different, none, or the same + <i>on halt</i> callbacks installed. When unloading the NIF + library during a + <seemfa marker="kernel:code#purge/1">code purge</seemfa>, an + installed <i>on halt</i> callback will be uninstalled. + The <c>ERL_NIF_OPT_ON_HALT</c> option can also only be set + once. That is, the <i>on halt</i> callback cannot be changed + or removed once it has been installed by any other means than + purging the module instance that loaded the NIF library. + </p> + <p> + When the installed <i>on halt</i> callback is called, it will be + passed a pointer to <c>priv_data</c> as argument. The + <c>priv_data</c> pointer can be set when loading the NIF library. + </p> + <p> + The <i>on halt</i> callback can be used in combination with + <seecref marker="#delay_halt"><i>delay of halt</i></seecref> until + all calls into the library have returned. The <i>on halt</i> + callback is in this case typically used to notify processes + blocked in NIFs in the library that it is time to return in order + to let the runtime system complete the halting. Such NIFs should + be dirty NIFs, since ordinary NIFs should never block for a long + time. + </p> + </item> + </taglist> + </desc> + </func> + + <func> <name since="OTP 22.0"><ret>void</ret> <nametext>enif_set_pid_undefined(ErlNifPid* pid)</nametext></name> <fsummary>Set pid as undefined.</fsummary> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 8a8a219b97..32ab7149f8 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -2925,7 +2925,7 @@ uncompiled code with the same arity are mapped to the same list by <fsummary>Halt the Erlang runtime system and indicate normal exit to the calling environment.</fsummary> <desc> - <p>The same as + <p>The same as calling <seemfa marker="#halt/2"><c>halt(0, [])</c></seemfa>. Example:</p> <pre> > <input>halt().</input> @@ -2934,10 +2934,11 @@ os_prompt%</pre> </func> <func> - <name name="halt" arity="1" since=""/> + <name name="halt" arity="1" clause_i="1" + anchor="halt_status_code_1" since=""/> <fsummary>Halt the Erlang runtime system.</fsummary> <desc> - <p>The same as <seemfa marker="#halt/2"> + <p>The same as calling <seemfa marker="#halt/2"> <c>halt(<anno>Status</anno>, [])</c></seemfa>. Example:</p> <pre> > <input>halt(17).</input> @@ -2948,44 +2949,161 @@ os_prompt%</pre> </func> <func> - <name name="halt" arity="2" since="OTP R15B01"/> + <name name="halt" arity="1" clause_i="2" + anchor="halt_abort_1" since="OTP R15B01"/> + <fsummary>Halt the Erlang runtime system by aborting.</fsummary> + <desc> + <p> + The same as calling <seeerl marker="#halt_abort_2"> + <c>halt(abort, [])</c></seeerl>. + </p> + </desc> + </func> + + <func> + <name name="halt" arity="1" clause_i="3" + anchor="halt_crash_dump_1" since=""/> + <fsummary> + Halt the Erlang runtime system and create an Erlang crash dump. + </fsummary> + <desc> + <p> + The same as calling <seeerl marker="#halt_crash_dump_2"> + <c>halt(<anno>CrashDumpSlogan</anno>, [])</c></seeerl>. + </p> + </desc> + </func> + + <func> + <name name="halt" arity="2" clause_i="1" + anchor="halt_status_code_2" since="OTP R15B01"/> <fsummary>Halt the Erlang runtime system.</fsummary> + <type name="halt_options"/> <desc> - <p><c><anno>Status</anno></c> must be a non-negative integer, a string, - or the atom <c>abort</c>. - Halts the Erlang runtime system. Has no return value. - Depending on <c><anno>Status</anno></c>, the following occurs:</p> + <p> + Halt the runtime system with status code + <c><anno>Status</anno></c>. + </p> + <note> + <p> + On many platforms, the OS supports only status codes 0-255. A too + large status code is truncated by clearing the high bits. + </p> + </note> + <p> + Currently the following options are valid: + </p> <taglist> - <tag>integer()</tag> - <item>The runtime system exits with integer value - <c><anno>Status</anno></c> - as status code to the calling environment (OS). - <note> - <p>On many platforms, the OS supports only status - codes 0-255. A too large status code is truncated by clearing - the high bits.</p> - </note> - </item> - <tag>string()</tag> - <item>An Erlang crash dump is produced with <c><anno>Status</anno></c> - as slogan. Then the runtime system exits with status code <c>1</c>. - The string will be truncated if longer than 200 characters. - <note> - <p>Before ERTS 9.1 (OTP-20.1) only code points in the range 0-255 - was accepted in the string. Now any unicode string is valid.</p> - </note> - </item> - <tag><c>abort</c></tag> - <item>The runtime system aborts producing a core dump, if that is - enabled in the OS. + <tag><marker id="halt_flush"/><c>{flush, EnableFlushing}</c></tag> + <item> + <p> + If <c>EnableFlushing</c> equals <c>true</c>, which also is the + default behavior, the runtime system will perform the following + operations before terminating: + </p> + <list> + <item><p> + Flush all outstanding output. + </p></item> + <item><p> + Send all Erlang ports exit signals and wait for them + to exit. + </p></item> + <item><p> + Wait for all async threads to complete all outstanding + async jobs. + </p></item> + <item><p> + Call all installed <seecref marker="erl_nif#on_halt">NIF + <i>on halt</i> callbacks</seecref>. + </p></item> + <item><p> + Wait for all ongoing <seecref marker="erl_nif#delay_halt">NIF + calls with the <i>delay halt</i> setting</seecref> enabled to + return. + </p></item> + <item><p> + Call all installed <c>atexit</c>/<c>on_exit</c> callbacks. + </p></item> + </list> + <p> + If <c>EnableFlushing</c> equals <c>false</c>, the runtime system + will terminate immediately without performing any of the above + listed operations. + </p> + <p> + Behavior changes compared to earlier versions: + </p> + <list> + <item> + <p> + Runtime systems prior to OTP @OTP-17771@ called all installed + <c>atexit</c>/<c>on_exit</c> callbacks also when <c>flush</c> + was disabled, but as of OTP @OTP-17771@ this is no longer the case. + </p> + </item> + </list> </item> </taglist> - <p>For integer <c><anno>Status</anno></c>, the Erlang runtime system - closes all ports and allows async threads to finish their - operations before exiting. To exit without such flushing, use - <c><anno>Option</anno></c> as <c>{flush,false}</c>.</p> - <p>For statuses <c>string()</c> and <c>abort</c>, option - <c>flush</c> is ignored and flushing is <em>not</em> done.</p> + </desc> + </func> + + <func> + <name name="halt" arity="2" clause_i="2" + anchor="halt_abort_2" since="OTP R15B01"/> + <fsummary>Halt the Erlang runtime system by aborting.</fsummary> + <type name="halt_options"/> + <desc> + <p> + Halt the Erlang runtime system by aborting and produce a core dump + if core dumping has been enabled in the environment that the + runtime system is executing in. + </p> + <note><p> + The <seeerl marker="#halt_flush"><c>{flush, boolean()}</c></seeerl> + option will be ignored, and flushing will be disabled. + </p></note> + </desc> + </func> + + <func> + <name name="halt" arity="2" clause_i="3" + anchor="halt_crash_dump_2" since="OTP R15B01"/> + <fsummary> + Halt the Erlang runtime system and create an Erlang crash dump. + </fsummary> + <type name="halt_options"/> + <desc> + <p> + Halt the Erlang runtime system and generate an + <seeguide marker="crash_dump">Erlang crash dump</seeguide>. The + string <c><anno>CrashDumpSlogan</anno></c> will be used as slogan + in the Erlang crash dump created. The slogan will be trunkated if + <c><anno>CrashDumpSlogan</anno></c> is longer than 1023 characters. + </p> + <note><p> + The <seeerl marker="#halt_flush"><c>{flush, boolean()}</c></seeerl> + option will be ignored, and flushing will be disabled. + </p></note> + <p> + Behavior changes compared to earlier versions: + </p> + <list> + <item> + <p> + Before OTP 24.2, the slogan was truncated if + <c><anno>CrashDumpSlogan</anno></c> was longer than 200 + characters. Now it will be truncated if longer than 1023 + characters. + </p> + </item> + <item> + <p> + Before OTP 20.1, only code points in the range 0-255 were + accepted in the slogan. Now any Unicode string is valid. + </p> + </item> + </list> </desc> </func> @@ -2994,12 +3112,17 @@ os_prompt%</pre> <fsummary>Head of a list.</fsummary> <desc> <p>Returns the head of <c><anno>List</anno></c>, that is, - the first element, for example:</p> + the first element.</p> + <p>It works with improper lists.</p> + <p>Examples:</p> <pre> > <input>hd([1,2,3,4,5]).</input> 1</pre> + <pre> +> <input>hd([first, second, third, so_on | improper_end]).</input> +first</pre> <p>Allowed in guard tests.</p> - <p>Failure: <c>badarg</c> if <c><anno>List</anno></c> is the empty + <p>Failure: <c>badarg</c> if <c><anno>List</anno></c> is an empty list <c>[]</c>.</p> </desc> </func> @@ -9302,7 +9425,7 @@ ok <p> Sets the default maximum heap size settings for processes. The size is specified in words. The new <c>max_heap_size</c> - effects only processes spawned efter the change has been made. + effects only processes spawned after the change has been made. <c>max_heap_size</c> can be set for individual processes using <seemfa marker="#spawn_opt/4"><c>spawn_opt/2,3,4</c></seemfa> or <seeerl marker="#process_flag_max_heap_size"> @@ -11853,7 +11976,9 @@ timestamp() -> <fsummary>Tail of a list.</fsummary> <desc> <p>Returns the tail of <c><anno>List</anno></c>, that is, - the list minus the first element, for example:</p> + the list minus the first element</p> + <p>It works with improper lists.</p> + <p>Examples:</p> <pre> > <input>tl([geesties, guilies, beasties]).</input> [guilies, beasties]</pre> @@ -11868,7 +11993,7 @@ timestamp() -> improper_end</pre> <p>Allowed in guard tests.</p> <p>Failure: <c>badarg</c> if <c><anno>List</anno></c> - is the empty list <c>[]</c>.</p> + is an empty list <c>[]</c>.</p> </desc> </func> diff --git a/erts/doc/src/erlsrv_cmd.xml b/erts/doc/src/erlsrv_cmd.xml index e8f066b21b..fe952f690e 100644 --- a/erts/doc/src/erlsrv_cmd.xml +++ b/erts/doc/src/erlsrv_cmd.xml @@ -112,8 +112,7 @@ <item> <p>The location of the Erlang emulator. The default is the <c><![CDATA[erl.exe]]></c> located in the same - directory as <c>erlsrv.exe</c>. Do not specify - <c><![CDATA[werl.exe]]></c> as this emulator, it will not work.</p> + directory as <c>erlsrv.exe</c>.</p> <p>If the system uses release handling, this is to be set to a program similar to <c><![CDATA[start_erl.exe]]></c>.</p> </item> diff --git a/erts/doc/src/time_correction.xml b/erts/doc/src/time_correction.xml index b3a36b907e..17263c26fe 100644 --- a/erts/doc/src/time_correction.xml +++ b/erts/doc/src/time_correction.xml @@ -34,29 +34,42 @@ </header> <section> - <title>New Extended Time Functionality</title> - <note><p>As from Erlang/OTP 18 (ERTS 7.0) the time functionality - has been extended. This includes a + <title>Extended Time Functionality</title> + + <p>As of Erlang/OTP 18 (ERTS 7.0) the time functionality + was extended. This includes a <seeguide marker="#The_New_Time_API">new API</seeguide> for time and <seeguide marker="#Time_Warp_Modes">time warp modes</seeguide> that change the system behavior when system time changes.</p> - <p>The <seeguide marker="#No_Time_Warp_Mode">default - time warp mode</seeguide> has the same behavior as before, and the - old API still works. Thus, you are not required to change - anything unless you want to. However, <em>you are strongly - encouraged to use the new API</em> instead of the old API based - on <seemfa marker="erlang#now/0"><c>erlang:now/0</c></seemfa>. - <c>erlang:now/0</c> is deprecated, as it is and - will be a scalability bottleneck.</p> - - <p>By using the new API, you - automatically get scalability and performance improvements. This - also enables you to use the - <seeguide marker="#Multi_Time_Warp_Mode">multi-time warp mode</seeguide> - that improves accuracy and precision of time measurements.</p> + <note> + <p> + As of Erlang/OTP 26 (ERTS 14.0) the + <seeguide marker="#Multi_Time_Warp_Mode">multi time warp + mode</seeguide> is enabled by default. This assumes that all + code executing on the system is + <seeguide marker="#Time_Warp_Safe_Code">time warp safe</seeguide>. + </p> + <p> + If you have old code in the system that is not time warp + safe, you now explicitly need to start the system in + <seeguide marker="#No_Time_Warp_Mode">no time warp + mode</seeguide> (or + <seeguide marker="#Single_Time_Warp_Mode">singe time warp + mode</seeguide> if it is partially time warp safe) in order + to avoid problems. When starting the system in no time warp + mode, the system behaves as it did prior to the introduction + of the extended time functionality introduced in OTP 18. + </p> + <p> + If you have code that is not time warp safe, you are strongly + encouraged to change this so that you can use multi time + warp mode. Compared to no time warp mode, multi time warp + mode improves scalability and performance as well as accuracy + and precision of time measurements. + </p> </note> </section> @@ -405,13 +418,9 @@ <section> <title>No Time Warp Mode</title> <p>The time offset is determined at runtime system start - and does not change later. This is the default behavior, but - not because it is the best mode (which it is not). It is - default <em>only</em> because this is how the runtime system - behaved until ERTS 7.0. - Ensure that your Erlang code that can execute during a time - warp is <seeguide marker="#Time_Warp_Safe_Code">time warp - safe</seeguide> before enabling other modes.</p> + and does not change later. This is the same behavior as was + default prior to OTP 26 (ERTS 14.0), and the only behavior + prior to OTP 18 (ERTS 7.0).</p> <p>As the time offset is not allowed to change, time correction must adjust the frequency of the Erlang @@ -558,7 +567,8 @@ better, and behave better on almost all platforms. Also, the accuracy and precision of time measurements are better. Only Erlang runtime systems executing on - ancient platforms benefit from another configuration.</p> + ancient platforms benefit from another configuration. + As of OTP 26 (ERTS 14.0) this is also the default.</p> <p>The time offset can change at any time without limitations. That is, Erlang system time can perform time warps both diff --git a/erts/doc/src/tty.xml b/erts/doc/src/tty.xml index e82e2cc807..76fba8d549 100644 --- a/erts/doc/src/tty.xml +++ b/erts/doc/src/tty.xml @@ -67,7 +67,8 @@ erl</pre> <item><c>C-a</c> means pressing the <em>Ctrl</em> key and the letter <c>a</c> simultaneously.</item> <item><c>M-f</c> means pressing the <em>Esc</em> key and the letter - <c>f</c> in sequence.</item> + <c>f</c> in sequence or pressing the <em>Alt</em> key and the letter + <c>f</c> simultaneously.</item> <item><c>Home</c> and <c>End</c> represent the keys with the same name on the keyboard.</item> <item><c>Left</c> and <c>Right</c> represent the corresponding arrow @@ -141,6 +142,10 @@ erl</pre> </row> <row> <cell align="left" valign="middle">C-l</cell> + <cell align="left" valign="middle">Clears the screen</cell> + </row> + <row> + <cell align="left" valign="middle">M-l</cell> <cell align="left" valign="middle">Redraw line</cell> </row> <row> @@ -149,11 +154,22 @@ erl</pre> buffer</cell> </row> <row> + <cell align="left" valign="middle">C-o</cell> + <cell align="left" valign="middle">Edit the current line using the editor specified in + the environment variable <c>VISUAL</c> or <c>EDITOR</c>. The environment variables can + contain arguments to the editor if needed, for example <c>VISUAL="emacs -nw"</c>. + On Windows the editor cannot be a console based editor.</cell> + </row> + <row> <cell align="left" valign="middle">C-p</cell> <cell align="left" valign="middle">Fetch previous line from the history buffer</cell> </row> <row> + <cell align="left" valign="middle">C-r</cell> + <cell align="left" valign="middle">Search the shell history</cell> + </row> + <row> <cell align="left" valign="middle">C-t</cell> <cell align="left" valign="middle">Transpose characters</cell> </row> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 979f76c4c6..d6e6158077 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -432,6 +432,7 @@ CREATE_DIRS+=$(TTF_DIR)/asmjit $(TTF_DIR)/asmjit/core $(TTF_DIR)/asmjit/$(JIT_AR endif BINDIR = $(ERL_TOP)/bin/$(TARGET) +CREATE_DIRS += $(BINDIR) ERLANG_OSTYPE = @ERLANG_OSTYPE@ @@ -1123,11 +1124,13 @@ RUN_OBJS += \ $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_global_literals.o \ $(OBJDIR)/beam_file.o \ - $(OBJDIR)/beam_types.o + $(OBJDIR)/beam_types.o \ + $(OBJDIR)/erl_term_hashing.o LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o NIF_OBJS = \ + $(OBJDIR)/prim_tty_nif.o \ $(OBJDIR)/erl_tracer_nif.o \ $(OBJDIR)/prim_buffer_nif.o \ $(OBJDIR)/prim_file_nif.o \ @@ -1138,10 +1141,8 @@ ifeq ($(TARGET),win32) DRV_OBJS = \ $(OBJDIR)/registry_drv.o \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/ram_file_drv.o \ - $(OBJDIR)/ttsl_drv.o + $(OBJDIR)/ram_file_drv.o OS_OBJS = \ - $(OBJDIR)/win_con.o \ $(OBJDIR)/dll_sys.o \ $(OBJDIR)/driver_tab.o \ $(OBJDIR)/sys_float.o \ @@ -1165,8 +1166,7 @@ OS_OBJS = \ DRV_OBJS = \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/ram_file_drv.o \ - $(OBJDIR)/ttsl_drv.o + $(OBJDIR)/ram_file_drv.o endif ifneq ($(STATIC_NIFS),no) @@ -1329,9 +1329,9 @@ NIF_COMMON_SRC=$(wildcard nifs/common/*.c) endif NIF_OSTYPE_SRC=$(wildcard nifs/$(ERLANG_OSTYPE)/*.c) ALL_SYS_SRC=$(wildcard sys/$(ERLANG_OSTYPE)/*.c) $(wildcard sys/common/*.c) -# We use $(shell ls) here instead of wildcard as $(wildcard ) resolved at +# We use $(shell find) here instead of wildcard as $(wildcard ) resolved at # loadtime of the makefile and at that time these files are not generated yet. -TARGET_SRC=$(shell ls $(TARGET)/*.c) $(shell ls $(TTF_DIR)/*.c) +TARGET_SRC=$(shell find $(TARGET) $(TTF_DIR) -maxdepth 1 -name "*.c") # I do not want the -MG flag on windows, it does not work properly for a # windows build. @@ -1434,8 +1434,9 @@ depend: else depend: $(TTF_DIR)/depend.mk $(TTF_DIR)/depend.mk: $(foreach dep, $(DEPEND_DEPS), $(TTF_DIR)/$(dep).depend.mk) - -rm $@ - for dep in $^; do cat $$dep >> $@; done + $(gen_verbose) + $(V_at)echo "" > "$@" + $(V_at)for dep in "$^"; do cat $$dep >> "$@"; done $(V_at)cd $(ERTS_LIB_DIR) && $(MAKE) depend endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 313b646cdd..04fcc6fa4b 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -257,6 +257,7 @@ atom enable_trace atom enabled atom endian atom env +atom ensure_at_least ensure_exactly atom eof atom eol atom Eq='=:=' @@ -325,6 +326,7 @@ atom get_all_trap atom get_internal_state_blocked atom get_seq_token atom get_size +atom get_tail atom get_tcw atom gather_gc_info_result atom gather_io_bytes @@ -603,6 +605,7 @@ atom return_time_trace atom return_to atom return_to_trace atom return_trace +atom reuse atom run_process atom run_queue atom run_queue_lengths @@ -645,6 +648,7 @@ atom set_tcw_fake atom short atom shutdown atom sighup +atom signed atom sigterm atom sigusr1 atom sigusr2 @@ -659,6 +663,7 @@ atom sigtstp atom sigquit atom silent atom size +atom skip atom spawn_executable atom spawn_driver atom spawn_init diff --git a/erts/emulator/beam/beam_common.c b/erts/emulator/beam/beam_common.c index 2055f60e91..6194644869 100644 --- a/erts/emulator/beam/beam_common.c +++ b/erts/emulator/beam/beam_common.c @@ -2174,7 +2174,7 @@ erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live, ASSERT(kp < (Eterm *)mp); key = *old_keys; - if ((c = CMP_TERM(key, new_key)) < 0) { + if ((c = (key == new_key) ? 0 : CMP_TERM(key, new_key)) < 0) { /* Copy old key and value */ *kp++ = key; *hp++ = *old_vals; diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c index 0c7bdfbec2..5ea8c7b48a 100644 --- a/erts/emulator/beam/beam_file.c +++ b/erts/emulator/beam/beam_file.c @@ -595,30 +595,52 @@ static void init_fallback_type_table(BeamFile *beam) { types->fallback = 1; types->entries[0].type_union = BEAM_TYPE_ANY; - types->entries[0].min = 0; - types->entries[0].max = -1; + types->entries[0].min = MAX_SMALL + 1; + types->entries[0].max = MIN_SMALL - 1; } -static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) { +static int parse_type_chunk_otp_25(BeamFile *beam, BeamReader *p_reader) { BeamFile_TypeTable *types; - BeamReader reader; - Sint32 version, count; + Sint32 count; int i; - beamreader_init(chunk->data, chunk->size, &reader); + types = &beam->types; + ASSERT(types->entries == NULL); - LoadAssert(beamreader_read_i32(&reader, &version)); - if (version != BEAM_TYPES_VERSION) { - /* Incompatible type format. */ - init_fallback_type_table(beam); - return 1; + LoadAssert(beamreader_read_i32(p_reader, &count)); + LoadAssert(CHECK_ITEM_COUNT(count, 0, sizeof(types->entries[0]))); + LoadAssert(count >= 1); + + types->entries = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + count * sizeof(types->entries[0])); + types->count = count; + types->fallback = 0; + + for (i = 0; i < count; i++) { + const byte *type_data; + + LoadAssert(beamreader_read_bytes(p_reader, 18, &type_data)); + LoadAssert(beam_types_decode_otp_25(type_data, 18, &types->entries[i])); } + /* The first entry MUST be the "any type." */ + LoadAssert(types->entries[0].type_union == BEAM_TYPE_ANY); + LoadAssert(types->entries[0].min > types->entries[0].max); + + return 1; +} + +static int parse_type_chunk_otp_26(BeamFile *beam, BeamReader *p_reader) { + BeamFile_TypeTable *types; + + Sint32 count; + int i; + types = &beam->types; ASSERT(types->entries == NULL); - LoadAssert(beamreader_read_i32(&reader, &count)); + LoadAssert(beamreader_read_i32(p_reader, &count)); LoadAssert(CHECK_ITEM_COUNT(count, 0, sizeof(types->entries[0]))); LoadAssert(count >= 1); @@ -629,9 +651,13 @@ static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) { for (i = 0; i < count; i++) { const byte *type_data; + int extra; - LoadAssert(beamreader_read_bytes(&reader, 18, &type_data)); - LoadAssert(beam_types_decode(type_data, 18, &types->entries[i])); + LoadAssert(beamreader_read_bytes(p_reader, 2, &type_data)); + extra = beam_types_decode_type_otp_26(type_data, &types->entries[i]); + LoadAssert(extra >= 0); + LoadAssert(beamreader_read_bytes(p_reader, extra, &type_data)); + beam_types_decode_extra_otp_26(type_data, &types->entries[i]); } /* The first entry MUST be the "any type." */ @@ -641,6 +667,25 @@ static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) { return 1; } +static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) { + BeamReader reader; + Sint32 version; + + beamreader_init(chunk->data, chunk->size, &reader); + + LoadAssert(beamreader_read_i32(&reader, &version)); + switch (version) { + case 1: /* OTP 25 */ + return parse_type_chunk_otp_25(beam, &reader); + case BEAM_TYPES_VERSION: /* OTP 26 */ + return parse_type_chunk_otp_26(beam, &reader); + default: + /* Incompatible type format. */ + init_fallback_type_table(beam); + return 1; + } +} + static ErlHeapFragment *new_literal_fragment(Uint size) { ErlHeapFragment *bp; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 85beba57ee..c7c0a715b5 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -47,7 +47,7 @@ Uint erts_total_code_size; static int load_code(LoaderState *stp); -#define PLEASE_RECOMPILE "please re-compile this module with an Erlang/OTP " ERLANG_OTP_RELEASE " compiler" +#define PLEASE_RECOMPILE "please re-compile this module with an Erlang/OTP " ERLANG_OTP_RELEASE " compiler or update your Erlang/OTP version" /**********************************************************************/ diff --git a/erts/emulator/beam/beam_types.c b/erts/emulator/beam/beam_types.c index f0b8c8a6dc..d9f06bfcc0 100644 --- a/erts/emulator/beam/beam_types.c +++ b/erts/emulator/beam/beam_types.c @@ -30,7 +30,55 @@ static Sint64 get_sint64(const byte *data); -int beam_types_decode(const byte *data, Uint size, BeamType *out) { +int beam_types_decode_type_otp_26(const byte *data, BeamType *out) { + int types; + int extra = 0; + + types = (Uint16)data[0] << 8 | (Uint16)data[1]; + if (types == BEAM_TYPE_NONE) { + return -1; + } + out->type_union = types; + + if (types & BEAM_TYPE_HAS_LOWER_BOUND) { + extra += 8; + } + + if (types & BEAM_TYPE_HAS_UPPER_BOUND) { + extra += 8; + } + + if (types & BEAM_TYPE_HAS_UNIT) { + extra += 1; + } + + return extra; +} + +void beam_types_decode_extra_otp_26(const byte *data, BeamType *out) { + int types = out->type_union; + out->type_union = types & BEAM_TYPE_ANY; + + out->min = MAX_SMALL + 1; + out->max = MIN_SMALL - 1; + out->size_unit = 1; + + if (types & BEAM_TYPE_HAS_LOWER_BOUND) { + out->min = get_sint64(data); + data += 8; + } + + if (types & BEAM_TYPE_HAS_UPPER_BOUND) { + out->max = get_sint64(data); + data += 8; + } + + if (types & BEAM_TYPE_HAS_UNIT) { + out->size_unit = data[0] + 1; + } +} + +int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out) { int types; if (size != 18) { @@ -49,6 +97,13 @@ int beam_types_decode(const byte *data, Uint size, BeamType *out) { data += 8; out->max = get_sint64(data); + if (out->min > out->max) { + out->min = MAX_SMALL + 1; + out->max = MIN_SMALL - 1; + } + + out->size_unit = 1; + return 1; } @@ -57,7 +112,7 @@ static Sint64 get_sint64(const byte *data) { int i; for (i = 0; i < 8; i++) { - value = value << 8 | (Sint16)data[i]; + value = value << 8 | (Sint64)data[i]; } return value; } diff --git a/erts/emulator/beam/beam_types.h b/erts/emulator/beam/beam_types.h index 000a644edb..e5697fc951 100644 --- a/erts/emulator/beam/beam_types.h +++ b/erts/emulator/beam/beam_types.h @@ -37,7 +37,7 @@ #include "sys.h" -#define BEAM_TYPES_VERSION 1 +#define BEAM_TYPES_VERSION 2 #define BEAM_TYPE_NONE (0) @@ -57,6 +57,10 @@ #define BEAM_TYPE_ANY ((1 << 13) - 1) +#define BEAM_TYPE_HAS_LOWER_BOUND (1 << 13) +#define BEAM_TYPE_HAS_UPPER_BOUND (1 << 14) +#define BEAM_TYPE_HAS_UNIT (1 << 15) + #define BEAM_TYPE_MASK_BOXED \ (BEAM_TYPE_BITSTRING | \ BEAM_TYPE_BS_MATCHSTATE | \ @@ -91,11 +95,27 @@ typedef struct { * be. When a single bit is set, the term will always be of that type. */ int type_union; - /** @brief Minimum and maximum values. Only valid if min <= max. */ + /** @brief Minimum and maximum values. Valid if at least one of + * IS_SSMALL(min) and IS_SSMALL(max) is true; otherwise the range is + * unknown. + * + * If and only if min < max, then the value is always a small within + * the range min trough max (including the endpoints). + * + * If and only if IS_SSMALL(min) && !IS_SSMALL(max) is true, then + * the value is greater than or equal to min. + * + * If and only if !IS_SSMALL(min) && IS_SSMALL(max), then the + * value is less than or equal to min. */ Sint64 min; Sint64 max; + + /** @brief Unit for bitstring size. */ + byte size_unit; } BeamType; -int beam_types_decode(const byte *data, Uint size, BeamType *out); +int beam_types_decode_type_otp_26(const byte *data, BeamType *out); +void beam_types_decode_extra_otp_26(const byte *data, BeamType *out); +int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out); #endif diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 4e6279704b..4aaf4b4649 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -23,6 +23,10 @@ #endif #include <stddef.h> /* offsetof() */ +#ifdef HAVE_SYS_IOCTL_H +#include <sys/ioctl.h> +#endif +#define WANT_NONBLOCKING #include "sys.h" #include "erl_vm.h" #include "erl_sys_driver.h" @@ -375,7 +379,6 @@ demonitor(Process *c_p, Eterm ref, Eterm *multip) } case ERTS_ML_STATE_ALIAS_ONCE: case ERTS_ML_STATE_ALIAS_DEMONITOR: - erts_pid_ref_delete(ref); /* fall through... */ default: erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), mon); @@ -4181,33 +4184,100 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) BIF_RET(res); } - -BIF_RETTYPE display_string_1(BIF_ALIST_1) +BIF_RETTYPE display_string_2(BIF_ALIST_2) { Process* p = BIF_P; - Eterm string = BIF_ARG_1; - Sint len = erts_unicode_list_to_buf_len(string); + Eterm string = BIF_ARG_2; + Sint len; Sint written; byte *str; int res; + byte *temp_alloc = NULL; - if (len < 0) { - BIF_ERROR(p, BADARG); +#ifdef __WIN32__ + HANDLE fd; + if (ERTS_IS_ATOM_STR("stdout", BIF_ARG_1)) { + fd = GetStdHandle(STD_OUTPUT_HANDLE); + } else if (ERTS_IS_ATOM_STR("stderr", BIF_ARG_1)) { + fd = GetStdHandle(STD_ERROR_HANDLE); + } +#else + int fd; + if (ERTS_IS_ATOM_STR("stdout", BIF_ARG_1)) { + fd = fileno(stdout); + } else if (ERTS_IS_ATOM_STR("stderr", BIF_ARG_1)) { + fd = fileno(stderr); + } +#if defined(HAVE_SYS_IOCTL_H) && defined(TIOCSTI) + else if (ERTS_IS_ATOM_STR("stdin", BIF_ARG_1)) { + fd = open("/proc/self/fd/0",0); + } +#endif +#endif + else { + BIF_ERROR(p, BADARG); + } + if (is_list(string) || is_nil(string)) { + len = erts_unicode_list_to_buf_len(string); + if (len < 0) BIF_ERROR(p, BADARG); + str = temp_alloc = (byte *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*len); + res = erts_unicode_list_to_buf(string, str, len, &written); + if (res != 0 || written != len) + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error (%d)\n", __FILE__, __LINE__, res); + } else if (is_binary(string)) { + Uint bitoffs, bitsize; + ERTS_GET_BINARY_BYTES(string, str, bitoffs, bitsize); + if (bitsize % 8 != 0) BIF_ERROR(p, BADARG); + len = binary_size(string); + if (bitoffs != 0) { + str = erts_get_aligned_binary_bytes(string, &temp_alloc); + } + } else { + BIF_ERROR(p, BADARG); } - str = (byte *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); - res = erts_unicode_list_to_buf(string, str, len, &written); - if (res != 0 || written != len) - erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error (%d)\n", __FILE__, __LINE__, res); - str[len] = '\0'; - erts_fprintf(stderr, "%s", str); - erts_free(ERTS_ALC_T_TMP, (void *) str); - BIF_RET(am_true); -} -BIF_RETTYPE display_nl_0(BIF_ALIST_0) -{ - erts_fprintf(stderr, "\n"); +#if defined(HAVE_SYS_IOCTL_H) && defined(TIOCSTI) + if (ERTS_IS_ATOM_STR("stdin", BIF_ARG_1)) { + for (int i = 0; i < len; i++) { + if (ioctl(fd, TIOCSTI, str+i) < 0) { + fprintf(stderr,"failed to write to %s (%s)\r\n", "/proc/self/fd/0", + strerror(errno)); + close(fd); + goto error; + } + } + close(fd); + } else +#endif + { +#ifdef __WIN32__ + if (!WriteFile(fd, str, len, &written, NULL)) { + goto error; + } +#else + written = 0; + do { + res = write(fd, str+written, len-written); + if (res < 0 && errno != ERRNO_BLOCK && errno != EINTR) + goto error; + written += res; + } while (written < len); +#endif + } + if (temp_alloc) + erts_free(ERTS_ALC_T_TMP, (void *) temp_alloc); BIF_RET(am_true); + +error: { +#ifdef __WIN32__ + char *errnostr = last_error(); +#else + char *errnostr = erl_errno_id(errno); +#endif + BIF_P->fvalue = am_atom_put(errnostr, strlen(errnostr)); + erts_free(ERTS_ALC_T_TMP, (void *) str); + BIF_ERROR(p, BADARG | EXF_HAS_EXT_INFO); + } } /**********************************************************************/ @@ -4674,7 +4744,7 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) ErtsMagicBinary *mb; Uint32 sid; if (refn[0] >= MAX_REFERENCE) goto bad; - if (n != ERTS_REF_NUMBERS) goto bad; + if (n != ERTS_REF_NUMBERS && n != ERTS_PID_REF_NUMBERS) goto bad; sid = erts_get_ref_numbers_thr_id(refn); if (sid > erts_no_schedulers) goto bad; if (erts_is_ordinary_ref_numbers(refn)) { @@ -4685,12 +4755,15 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) } else { /* Check if it is a pid reference... */ - Eterm pid = erts_pid_ref_lookup(refn); + Eterm pid = erts_pid_ref_lookup(refn, n); if (is_internal_pid(pid)) { hp = HAlloc(BIF_P, ERTS_PID_REF_THING_SIZE); write_pid_ref_thing(hp, refn[0], refn[1], refn[2], pid); res = make_internal_ref(hp); } + else if (is_non_value(pid)) { + goto bad; + } else { /* Check if it is a magic reference... */ mb = erts_magic_ref_lookup_bin(refn); @@ -5812,8 +5885,6 @@ BIF_RETTYPE unalias_1(BIF_ALIST_1) ASSERT(erts_monitor_is_origin(mon)); - erts_pid_ref_delete(BIF_ARG_1); - mon->flags &= ~ERTS_ML_STATE_ALIAS_MASK; if (mon->type == ERTS_MON_TYPE_ALIAS) { erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), mon); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index ebf815570c..455992e0e9 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -56,8 +56,7 @@ bif erlang:crc32_combine/3 bif erlang:date/0 bif erlang:delete_module/1 bif erlang:display/1 -bif erlang:display_string/1 -bif erlang:display_nl/0 +bif erlang:display_string/2 ubif erlang:element/2 bif erlang:erase/0 hbif erlang:erase/1 @@ -362,6 +361,7 @@ bif ets:first/1 bif ets:is_compiled_ms/1 bif ets:lookup/2 bif ets:lookup_element/3 +bif ets:lookup_element/4 bif ets:info/1 bif ets:info/2 bif ets:last/1 diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 3b6de45587..b19d9e4994 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -576,19 +576,29 @@ do_break(void) " (l)oaded (v)ersion (k)ill (D)b-tables (d)istribution\n"; int i; #ifdef __WIN32__ + char *clearscreen = "\033[J"; char *mode; /* enough for storing "window" */ /* check if we're in console mode and, if so, halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); - if (mode && sys_strcmp(mode, "window") != 0) + if (mode && sys_strcmp(mode, "detached") == 0) erts_exit(0, ""); erts_free_read_env(mode); -#endif /* __WIN32__ */ +#else + char *clearscreen = "\E[J"; +#endif ASSERT(erts_thr_progress_is_blocking()); - erts_printf("\n%s", helpstring); + /* If we are writing to something known to be a tty we clear the screen + after doing newline as the shell tab completion may have written + things there. */ + if (!isatty(fileno(stdin)) || !isatty(fileno(stdout))) { + clearscreen = ""; + } + + erts_printf("\n%s%s", clearscreen, helpstring); while (1) { if ((i = sys_get_key(0)) <= 0) @@ -781,7 +791,7 @@ crash_dump_limited_writer(void* vfdp, char* buf, size_t len) } /* We assume that crash dump was called from erts_exit_vv() */ - erts_exit_epilogue(); + erts_exit_epilogue(0); } /* XXX THIS SHOULD BE IN SYSTEM !!!! */ diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 0789813a9c..4195cd3cfd 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1341,28 +1341,14 @@ erts_dsig_send_unlink(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, Uint6 Eterm big_heap[ERTS_MAX_UINT64_HEAP_SIZE]; Eterm unlink_id; Eterm ctl; - if (ctx->dflags & DFLAG_UNLINK_ID) { - if (IS_USMALL(0, id)) - unlink_id = make_small(id); - else { - Eterm *hp = &big_heap[0]; - unlink_id = erts_uint64_to_big(id, &hp); - } - ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_UNLINK_ID), - unlink_id, local, remote); - } + if (IS_USMALL(0, id)) + unlink_id = make_small(id); else { - /* - * A node that isn't capable of talking the new link protocol. - * - * Send an old unlink op, and send ourselves an unlink-ack. We may - * end up in an inconsistent state as we could before the new link - * protocol was introduced... - */ - erts_proc_sig_send_dist_unlink_ack(ctx->dep, ctx->connection_id, - remote, local, id); - ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_UNLINK), local, remote); + Eterm *hp = &big_heap[0]; + unlink_id = erts_uint64_to_big(id, &hp); } + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_UNLINK_ID), + unlink_id, local, remote); return dsig_send_ctl(ctx, ctl); } @@ -1373,11 +1359,6 @@ erts_dsig_send_unlink_ack(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, U Eterm unlink_id; Eterm ctl; - if (!(ctx->dflags & DFLAG_UNLINK_ID)) { - /* Receiving node does not understand it, so drop it... */ - return ERTS_DSIG_SEND_OK; - } - if (IS_USMALL(0, id)) unlink_id = make_small(id); else { @@ -2146,6 +2127,13 @@ int erts_net_message(Port *prt, break; } + case DOP_UNLINK: + /* + * DOP_UNLINK should never be passed. The new link protocol is + * mandatory as of OTP 26. + */ + goto invalid_message; + case DOP_UNLINK_ID: { Eterm *element; Uint64 id; @@ -2159,14 +2147,6 @@ int erts_net_message(Port *prt, if (id == 0) goto invalid_message; - if (0) { - case DOP_UNLINK: - if (tuple_arity != 3) - goto invalid_message; - element = &tuple[2]; - id = 0; - } - from = *(element++); to = *element; if (is_not_external_pid(from)) diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 9d10bbcd36..372cdfc1a7 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -84,8 +84,13 @@ | DFLAG_BIT_BINARIES \ | DFLAG_HANDSHAKE_23) +/* New mandatory flags for distribution in OTP 26 */ +#define DFLAG_DIST_MANDATORY_26 (DFLAG_V4_NC \ + | DFLAG_UNLINK_ID) + /* Mandatory flags for distribution. */ -#define DFLAG_DIST_MANDATORY DFLAG_DIST_MANDATORY_25 +#define DFLAG_DIST_MANDATORY (DFLAG_DIST_MANDATORY_25 \ + | DFLAG_DIST_MANDATORY_26) /* * Additional optimistic flags when encoding toward pending connection. @@ -95,8 +100,7 @@ #define DFLAG_DIST_HOPEFULLY (DFLAG_DIST_MONITOR \ | DFLAG_DIST_MONITOR_NAME \ | DFLAG_SPAWN \ - | DFLAG_ALIAS \ - | DFLAG_UNLINK_ID) + | DFLAG_ALIAS) /* Our preferred set of flags. Used for connection setup handshake */ #define DFLAG_DIST_DEFAULT (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY \ @@ -108,9 +112,7 @@ | DFLAG_EXIT_PAYLOAD \ | DFLAG_FRAGMENTS \ | DFLAG_SPAWN \ - | DFLAG_V4_NC \ | DFLAG_ALIAS \ - | DFLAG_UNLINK_ID \ | DFLAG_MANDATORY_25_DIGEST) /* Flags addable by local distr implementations */ diff --git a/erts/emulator/beam/emu/beam_emu.c b/erts/emulator/beam/emu/beam_emu.c index dff6a274dc..5130eed5a2 100644 --- a/erts/emulator/beam/emu/beam_emu.c +++ b/erts/emulator/beam/emu/beam_emu.c @@ -313,6 +313,8 @@ void process_main(ErtsSchedulerData *esdp) #endif #endif + Uint bitdata = 0; + Uint64 start_time = 0; /* Monitor long schedule */ ErtsCodePtr start_time_i = NULL; diff --git a/erts/emulator/beam/emu/bs_instrs.tab b/erts/emulator/beam/emu/bs_instrs.tab index c8bba1e70a..accd6aa91e 100644 --- a/erts/emulator/beam/emu/bs_instrs.tab +++ b/erts/emulator/beam/emu/bs_instrs.tab @@ -427,7 +427,6 @@ bs_init.verify(Fail) { bs_init.execute(Live, Dst) { erts_bin_offset = 0; - erts_writable_bin = 0; if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb; @@ -539,7 +538,6 @@ bs_init_bits.execute(Live, Dst) { } erts_bin_offset = 0; - erts_writable_bin = 0; /* num_bits = Number of bits to build * num_bytes = Number of bytes to allocate in the binary @@ -1113,7 +1111,6 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { * alloc = Total number of words to allocate on heap */ erts_bin_offset = 0; - erts_writable_bin = 0; if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb; @@ -1856,3 +1853,286 @@ i_bs_start_match3.execute(Live, Fail, Dst) { } %endif + +// +// New instructions introduced in OTP 26 for matching of integers and +// binaries of fixed sizes follow. +// + +// +// i_bs_ensure_bits Ctx Size Fail +// + +i_bs_ensure_bits := i_bs_ensure_bits.fetch.execute; + +i_bs_ensure_bits.head() { + Eterm context; +} + +i_bs_ensure_bits.fetch(Src) { + context = $Src; +} + +i_bs_ensure_bits.execute(NumBits, Fail) { + ErlBinMatchBuffer* mb = ms_matchbuffer(context); + Uint size = $NumBits; + if (mb->size - mb->offset < size) { + $FAIL($Fail); + } +} + +// +// i_bs_ensure_bits_unit Ctx Size Unit Fail +// + +i_bs_ensure_bits_unit := i_bs_ensure_bits_unit.fetch.execute; + +i_bs_ensure_bits_unit.head() { + Eterm context; +} + +i_bs_ensure_bits_unit.fetch(Src) { + context = $Src; +} + +i_bs_ensure_bits_unit.execute(NumBits, Unit, Fail) { + ErlBinMatchBuffer* mb = ms_matchbuffer(context); + Uint size = $NumBits; + Uint diff; + + if ((diff = mb->size - mb->offset) < size) { + $FAIL($Fail); + } + if ((diff - size) % $Unit != 0) { + $FAIL($Fail); + } +} + +// +// i_bs_read_bits Ctx Size +// i_bs_ensure_bits_read Ctx Size Fail +// + +i_bs_read_bits := i_bs_read_bits.fetch.execute; +i_bs_ensure_bits_read := i_bs_read_bits.fetch.ensure_bits.execute; + +i_bs_read_bits.head() { + ErlBinMatchBuffer* mb; + Uint size; +} + +i_bs_read_bits.fetch(Src, NumBits) { + mb = ms_matchbuffer($Src); + size = $NumBits; +} + +i_bs_read_bits.ensure_bits(Fail) { + if (mb->size - mb->offset < size) { + $FAIL($Fail); + } +} + +i_bs_read_bits.execute() { + byte *byte_ptr; + Uint bit_offset = mb->offset % 8; + Uint num_bytes_to_read = (size + 7) / 8; + Uint num_partial = size % 8; + + if ((num_partial == 0 && bit_offset != 0) || + (num_partial != 0 && bit_offset > 8 - num_partial)) { + num_bytes_to_read++; + } + + bitdata = 0; + byte_ptr = mb->base + (mb->offset >> 3); + mb->offset += size; + switch (num_bytes_to_read) { +#ifdef ARCH_64 + case 9: + case 8: + bitdata = bitdata << 8 | *byte_ptr++; + case 7: + bitdata = bitdata << 8 | *byte_ptr++; + case 6: + bitdata = bitdata << 8 | *byte_ptr++; + case 5: + bitdata = bitdata << 8 | *byte_ptr++; +#else + case 5: +#endif + case 4: + bitdata = bitdata << 8 | *byte_ptr++; + case 3: + bitdata = bitdata << 8 | *byte_ptr++; + case 2: + bitdata = bitdata << 8 | *byte_ptr++; + case 1: + bitdata = bitdata << 8 | *byte_ptr++; + } + + if (num_bytes_to_read <= sizeof(Uint)) { + bitdata <<= 8 * (sizeof(Uint) - num_bytes_to_read) + bit_offset; + } else { + bitdata <<= bit_offset; + bitdata = bitdata | ((*byte_ptr << bit_offset) >> 8); + } +} + +// i_bs_eq Fail Size Value +i_bs_eq(Fail, NumBits, Value) { + Uint size = $NumBits; + Eterm result; + + result = bitdata >> (8 * sizeof(Uint) - size); + bitdata <<= size; + if (result != $Value) { + $FAIL($Fail); + } +} + +// i_bs_extract_integer Size Dst +i_bs_extract_integer(NumBits, Dst) { + Uint size = $NumBits; + Eterm result; + + result = bitdata >> (8 * sizeof(Uint) - size); + result = make_small(result); + bitdata <<= size; + $Dst = result; +} + +// i_bs_read_integer_8 Ctx Dst +i_bs_read_integer_8(Ctx, Dst) { + ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); + byte *byte_ptr; + Uint bit_offset = mb->offset % 8; + Eterm result; + + byte_ptr = mb->base + (mb->offset >> 3); + mb->offset += 8; + result = byte_ptr[0]; + if (bit_offset != 0) { + result = result << 8 | byte_ptr[1]; + result = ((result << bit_offset) >> 8) & 0xff; + } + result = make_small(result); + $Dst = result; +} + +// +// i_bs_get_fixed_integer Ctx Size Flags Dst +// + +i_bs_get_fixed_integer := i_bs_get_fixed_integer.fetch.execute; + +i_bs_get_fixed_integer.head() { + Eterm context; +} + +i_bs_get_fixed_integer.fetch(Src) { + context = $Src; +} + +i_bs_get_fixed_integer.execute(Size, Flags, Dst) { + ErlBinMatchBuffer* mb; + Uint size = $Size; + Eterm result; + + mb = ms_matchbuffer(context); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, size, $Flags, mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + $Dst = result; +} + +// +// i_get_fixed_binary Ctx Size Dst +// + +i_bs_get_fixed_binary := i_bs_get_fixed_binary.fetch.execute; + +i_bs_get_fixed_binary.head() { + Eterm context; +} + +i_bs_get_fixed_binary.fetch(Src) { + context = $Src; +} + +i_bs_get_fixed_binary.execute(Size, Dst) { + ErlBinMatchBuffer* mb; + Uint size = $Size; + Eterm* htop; + Eterm result; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + htop = HTOP; + mb = ms_matchbuffer(context); + result = erts_extract_sub_binary(&htop, mb->orig, mb->base, + mb->offset, size); + HTOP = htop; + + mb->offset += size; + + $Dst = result; +} + +// +// i_get_tail Ctx Dst +// + +i_bs_get_tail := i_bs_get_tail.fetch.execute; + +i_bs_get_tail.head() { + Eterm context; +} + +i_bs_get_tail.fetch(Src) { + context = $Src; +} + +i_bs_get_tail.execute(Dst) { + ErlBinMatchBuffer* mb; + Eterm* htop; + Eterm result; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + htop = HTOP; + mb = ms_matchbuffer(context); + result = erts_extract_sub_binary(&htop, mb->orig, mb->base, + mb->offset, mb->size - mb->offset); + HTOP = htop; + + $Dst = result; +} + +// +// i_bs_skip Ctx Size +// + +i_bs_skip := i_bs_skip.fetch.execute; + +i_bs_skip.head() { + Eterm context; +} + +i_bs_skip.fetch(Src) { + context = $Src; +} + +i_bs_skip.execute(Size) { + ErlBinMatchBuffer* mb; + Uint size = $Size; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + mb = ms_matchbuffer(context); + mb->offset += size; +} + +// i_bs_drop Size +i_bs_drop(Size) { + bitdata <<= $Size; +} diff --git a/erts/emulator/beam/emu/generators.tab b/erts/emulator/beam/emu/generators.tab index 65d5c60175..97689e230e 100644 --- a/erts/emulator/beam/emu/generators.tab +++ b/erts/emulator/beam/emu/generators.tab @@ -1025,3 +1025,367 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { } return op; } + +gen.update_record(Size, Src, Dst, N, Updates) { + BeamOp *begin, *prev; + Sint count, i; + + ASSERT(Size.type == TAG_u && Size.val < SCRATCH_X_REG); + ASSERT(N.type == TAG_u && !(N.val % 2) && (N.val / 2) <= Size.val); + + $NewBeamOp(S, begin); + $BeamOpNameArity(begin, i_update_record, 5); + + begin->a[0] = Size; + begin->a[1] = Src; + begin->a[2] = Dst; + begin->a[3] = Updates[0]; + begin->a[4] = Updates[1]; + + count = N.val; + prev = begin; + + for (i = 2; i < count; i += 2) { + BeamOp *next; + + $NewBeamOp(S, next); + $BeamOpNameArity(next, i_update_record_continue, 2); + + /* Encode the offset from the _end_ of the tuple so that we can act + * relative to HTOP. */ + next->a[0].type = TAG_u; + next->a[0].val = (Size.val + 1) - Updates[i].val; + + /* The first instruction overwrites the destination register after + * stashing its contents to SCRATCH_X_REG, so all updates must be + * rewritten accordingly. */ + if (Updates[i + 1].type == Dst.type && Updates[i + 1].val == Dst.val) { + next->a[1].type = TAG_x; + next->a[1].val = SCRATCH_X_REG; + } else { + next->a[1] = Updates[i + 1]; + } + + next->next = NULL; + prev->next = next; + + prev = next; + } + + return begin; +} + +gen.bs_match(Fail, Ctx, N, List) { + BeamOp* first_op = 0; + BeamOp** next_ptr = &first_op; + BeamOp* test_heap_op = 0; + BeamOp* read_op = 0; +#ifdef ARCH_64 + BeamOp* eq_op = 0; +#endif + int src; + + src = 0; + while (src < N.val) { + Uint unit; + Uint size; + Uint words_needed; + BeamOp* op; + + /* Calculate the number of heap words needed for this + * instruction. */ + words_needed = 0; + switch (List[src].val) { + case am_binary: + ASSERT(List[src+3].type == TAG_u); + ASSERT(List[src+4].type == TAG_u); + size = List[src+3].val * List[src+4].val; + words_needed = heap_bin_size((size + 7) / 8); + break; + case am_integer: + ASSERT(List[src+3].type == TAG_u); + ASSERT(List[src+4].type == TAG_u); + size = List[src+3].val * List[src+4].val; + if (size >= SMALL_BITS) { + words_needed = BIG_NEED_FOR_BITS(size); + } + break; + case am_get_tail: + words_needed = EXTRACT_SUB_BIN_HEAP_NEED; + break; + } + + /* Emit a test_heap instrution if needed and there is + * no previous one. */ + if ((List[src].val == am_Eq || words_needed) && test_heap_op == 0 && + List[src+1].type == TAG_u) { + $NewBeamOp(S, test_heap_op); + $BeamOpNameArity(test_heap_op, test_heap, 2); + + test_heap_op->a[0].type = TAG_u; + test_heap_op->a[0].val = 0; /* Number of heap words */ + test_heap_op->a[1] = List[src+1]; /* Live */ + + *next_ptr = test_heap_op; + next_ptr = &test_heap_op->next; + } + + if (words_needed) { + test_heap_op->a[0].val += words_needed; + } + + /* Translate this sub-instruction to a BEAM instruction. */ + op = 0; + switch (List[src].val) { + case am_ensure_at_least: { + Uint size = List[src+1].val; + unit = List[src+2].val; + if (size != 0 && unit == 1) { + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_ensure_bits, 3); + op->a[0] = Ctx; + op->a[1].type = TAG_u; + op->a[1].val = size; + op->a[2] = Fail; + } else if (size != 0 && unit != 1) { + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_ensure_bits_unit, 4); + + op->a[0] = Ctx; + op->a[1].type = TAG_u; + op->a[1].val = size; /* Size */ + op->a[2].type = TAG_u; + op->a[2].val = unit; /* Unit */ + op->a[3] = Fail; + } else if (size == 0 && unit != 1) { + $NewBeamOp(S, op); + $BeamOpNameArity(op, bs_test_unit, 3); + + op->a[0] = Fail; + op->a[1] = Ctx; + op->a[2].type = TAG_u; + op->a[2].val = unit; + } else if (size == 0 && unit == 1) { + /* This test is redundant because it always + * succeeds. This should only happen for unoptimized + * code. Generate a dummy instruction to ensure that + * we don't trigger the sanity check at the end of + * this generator. */ + $NewBeamOp(S, op); + $BeamOpNameArity(op, delete_me, 0); + } + src += 3; + break; + } + case am_ensure_exactly: { + $NewBeamOp(S, op); + $BeamOpNameArity(op, bs_test_tail2, 3); + + op->a[0] = Fail; + op->a[1] = Ctx; + op->a[2]= List[src+1]; /* Size */ + + src += 2; + break; + } + case am_binary: { + ASSERT(List[src+3].type == TAG_u); + ASSERT(List[src+4].type == TAG_u); + size = List[src+3].val; + unit = List[src+4].val; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_get_fixed_binary, 3); + + op->a[0] = Ctx; + op->a[1].type = TAG_u; + op->a[1].val = size * unit; /* Size */ + op->a[2] = List[src+5]; /* Dst */ + + read_op = 0; + src += 6; + break; + } + case am_integer: { + Uint flags = 0; + BeamOpArg Flags; + + /* Translate flags. */ + Flags = List[src+2]; + if (Flags.type == TAG_n) { + Flags.type = TAG_u; + Flags.val = 0; + } else if (Flags.type == TAG_q) { + Eterm term = beamfile_get_literal(&S->beam, Flags.val); + while (is_list(term)) { + Eterm* consp = list_val(term); + Eterm elem = CAR(consp); + switch (elem) { + case am_little: + flags |= BSF_LITTLE; + break; + case am_native: + flags |= BSF_NATIVE; + break; + case am_signed: + flags |= BSF_SIGNED; + break; + } + term = CDR(consp); + } + ASSERT(is_nil(term)); + Flags.type = TAG_u; + Flags.val = flags; + $NativeEndian(Flags); + } + + ASSERT(List[src+3].type == TAG_u); + ASSERT(List[src+4].type == TAG_u); + size = List[src+3].val * List[src+4].val; + +#define READ_OP_SIZE 1 + if (size < SMALL_BITS && flags == 0) { + /* This is a suitable segment -- an unsigned big + * endian integer that fits in a small. */ + if (read_op == 0 || read_op->a[READ_OP_SIZE].val + size > 8*sizeof(Uint)) { + /* There is either no previous i_bs_read_bits instruction or + * the size of this segment don't fit into it. */ + $NewBeamOp(S, read_op); + $BeamOpNameArity(read_op, i_bs_read_bits, 2); + + read_op->a[0] = Ctx; + read_op->a[1].type = TAG_u; + read_op->a[1].val = 0; + + *next_ptr = read_op; + next_ptr = &read_op->next; + } + + read_op->a[READ_OP_SIZE].val += size; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_extract_integer, 2); + op->a[0].type = TAG_u; + op->a[0].val = size; + op->a[1] = List[src+5]; /* Dst */ + } else { + /* Little endian, signed, or might not fit in a small. */ + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_get_fixed_integer, 4); + + op->a[0] = Ctx; + op->a[1].type = TAG_u; + op->a[1].val = size; /* Size */ + op->a[2] = Flags; /* Flags */ + op->a[3] = List[src+5]; /* Dst */ + + read_op = 0; + } + + src += 6; + break; + } + case am_Eq: { + ASSERT(List[src+2].type == TAG_u); + ASSERT(List[src+3].type == TAG_u); + size = List[src+2].val; + + if (read_op == 0 || read_op->a[READ_OP_SIZE].val + size > 8*sizeof(Uint)) { + /* There is either no previous i_bs_read_bits instruction or + * the size of this segment don't fit into it. */ + $NewBeamOp(S, read_op); + $BeamOpNameArity(read_op, i_bs_read_bits, 2); + + read_op->a[0] = Ctx; + read_op->a[1].type = TAG_u; + read_op->a[1].val = 0; + + *next_ptr = read_op; + next_ptr = &read_op->next; + } + + read_op->a[READ_OP_SIZE].val += size; + +#ifdef ARCH_64 + if (eq_op && + eq_op->next == 0 && /* Previous instruction? */ + eq_op->a[1].val + size <= 8*sizeof(Uint)) { + /* Coalesce with the previous `=:=` instruction. */ + eq_op->a[1].val += size; + eq_op->a[2].val = eq_op->a[2].val << size | List[src+3].val; + } +#else + if (0) { + ; + } +#endif + else { + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_eq, 3); + +#ifdef ARCH_64 + eq_op = op; +#endif + + op->a[0] = Fail; + op->a[1] = List[src+2]; /* Size */ + op->a[2] = List[src+3]; /* Value */ + } + + src += 4; + break; + } + case am_get_tail: + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_get_tail, 2); + + op->a[0] = Ctx; + op->a[1] = List[src+3]; /* Dst */ + + read_op = 0; + src += 4; + break; + case am_skip: + ASSERT(List[src+1].type == TAG_u); + size = List[src+1].val; + + $NewBeamOp(S, op); + + if (read_op && read_op->a[READ_OP_SIZE].val + size <= 8*sizeof(Uint)) { + read_op->a[READ_OP_SIZE].val += size; + $BeamOpNameArity(op, i_bs_drop, 1); + op->a[0] = List[src+1]; /* Size */ + } else { + $BeamOpNameArity(op, i_bs_skip, 2); + op->a[0] = Ctx; + op->a[1] = List[src+1]; /* Size */ + read_op = 0; + } + + src += 2; + break; + default: + abort(); + } + + if (op) { + *next_ptr = op; + next_ptr = &op->next; + } + } + + if (test_heap_op && test_heap_op->a[0].val == 0) { + /* This test_heap instruction was forced by the `=:=` sub + * instruction, but it turned out that no test_heap instruction was + * needed. */ + $BeamOpNameArity(test_heap_op, delete_me, 0); + } + + if (first_op == 0) { + erts_exit(ERTS_ERROR_EXIT, "loading bs_match in %T:%T/%d: no instructions loaded", + S->module, S->function, S->arity); + } + + ASSERT(first_op); + return first_op; +} diff --git a/erts/emulator/beam/emu/instrs.tab b/erts/emulator/beam/emu/instrs.tab index 2c39d885ef..668cc9db77 100644 --- a/erts/emulator/beam/emu/instrs.tab +++ b/erts/emulator/beam/emu/instrs.tab @@ -762,6 +762,30 @@ self(Dst) { $Dst = c_p->common.id; } +i_update_record(Size, Src, Dst, Offset, Element) { + Eterm *untagged_source = tuple_val($Src); + Uint size_on_heap = $Size + 1; + + /* Copy the entire tuple up-front, mimicking the behavior of the old + * setelement/3 + set_tuple_element method. This overwrites every field + * we're setting, but that cost is generally pretty small. */ + sys_memcpy(HTOP, untagged_source, size_on_heap * sizeof(Eterm)); + HTOP[$Offset] = $Element; + + /* We stash the contents of the destination register in SCRATCH_X_REG in + * case it's used in subsequent `i_update_record_continue` instructions. + * The updates have been rewritten accordingly. */ + reg[SCRATCH_X_REG] = $Dst; + $Dst = make_tuple(HTOP); + + HTOP += size_on_heap; +} + +i_update_record_continue(OffsetFromEnd, Element) { + Sint offset = -(Sint)$OffsetFromEnd; + HTOP[offset] = $Element; +} + set_tuple_element(Element, Tuple, Offset) { Eterm* p; diff --git a/erts/emulator/beam/emu/ops.tab b/erts/emulator/beam/emu/ops.tab index 814fd23b1b..dbcb05566a 100644 --- a/erts/emulator/beam/emu/ops.tab +++ b/erts/emulator/beam/emu/ops.tab @@ -1106,11 +1106,67 @@ is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A # ================================================================ -# New bit syntax matching (R11B). +# New bit syntax matching for fixed sizes (from OTP 26). # ================================================================ %warm +bs_match Fail Ctx Size Rest=* => bs_match(Fail, Ctx, Size, Rest) + +# The bs_match generator breaks the bs_match instruction into +# the instructions that follow. + +i_bs_ensure_bits xy I f +i_bs_ensure_bits_unit xy I t f + +i_bs_read_bits xy t + +i_bs_eq f t W + +i_bs_extract_integer t d + +i_bs_read_bits Ctx=x u==8 | i_bs_extract_integer u==8 Dst=x => + i_bs_read_integer_8 Ctx Dst + +i_bs_read_integer_8 x x + +i_bs_get_fixed_integer xy I t d + +i_bs_get_fixed_binary xy I d + +i_bs_get_tail xy d + +i_bs_skip xy I + +i_bs_drop I + +i_bs_ensure_bits Ctx1 Size1 Fail | i_bs_read_bits Ctx2 Size2 | + equal(Ctx1, Ctx2) | equal(Size1, Size2) => + i_bs_ensure_bits_read Ctx1 Size1 Fail + +i_bs_ensure_bits_read xy t f + +# Optimize extraction of a single segment for some popular sizes. + +i_bs_ensure_bits Ctx1 u==8 Fail | i_bs_read_bits Ctx2 u==8 | + i_bs_extract_integer u==8 Dst=x | equal(Ctx1, Ctx2) => + i_bs_get_integer_8 Ctx1 Fail Dst + +i_bs_ensure_bits Ctx1 u==16 Fail | i_bs_read_bits Ctx2 u==16 | + i_bs_extract_integer u==16 Dst=x | equal(Ctx1, Ctx2) => + i_bs_get_integer_16 Ctx1 Fail Dst + +%if ARCH_64 +i_bs_ensure_bits Ctx1 u==32 Fail | i_bs_read_bits Ctx2 u==32 | + i_bs_extract_integer u==32 Dst=x | equal(Ctx1, Ctx2) => + i_bs_get_integer_32 Ctx1 Fail Dst +%endif + + +# ================================================================ +# Bit syntax matching (from R11B). +# ================================================================ + # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val @@ -1714,3 +1770,15 @@ recv_marker_reserve S recv_marker_bind S S recv_marker_clear S recv_marker_use S + +# +# OTP 26 +# + +update_record Hint=a Size=u Src=s Dst=d N=u Updates=* => + update_record(Size, Src, Dst, N, Updates) + +i_update_record Size=u Src=c Dst=xy Offset=u Element=s => + move Src x | i_update_record Size x Dst Offset Element +i_update_record t xy xy t s +i_update_record_continue t s diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 0deced34de..ac11d82758 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -62,7 +62,6 @@ static Uint32 max_thr_id; #endif static void init_magic_ref_tables(void); -static void init_pid_ref_tables(void); static Uint64 ref_init_value; @@ -84,7 +83,6 @@ init_reference(void) erts_atomic64_init_nob(&global_reference.count, (erts_aint64_t) ref_init_value); init_magic_ref_tables(); - init_pid_ref_tables(); } static ERTS_INLINE void @@ -146,8 +144,6 @@ Eterm erts_make_ref(Process *c_p) return make_internal_ref(hp); } -static void pid_ref_save(Uint32 refn[ERTS_REF_NUMBERS], Eterm pid); - Eterm erts_make_pid_ref(Process *c_p) { Eterm* hp; @@ -167,8 +163,6 @@ Eterm erts_make_pid_ref(Process *c_p) write_pid_ref_thing(hp, ref[0], ref[1], ref[2], pid); - pid_ref_save(ref, pid); - return make_internal_ref(hp); } @@ -439,268 +433,6 @@ void erts_ref_bin_free(ErtsMagicBinary *mb) erts_bin_free((Binary *) mb); } - -/* - * Pid reference tables. - * - * These tables are intended to be temporary until huge - * references (containing the pid) can be mandatory in - * the external format. - */ - -typedef struct { - HashBucket hash; - Eterm pid; - Uint64 value; - Uint32 thr_id; -} ErtsPidRefTableEntry; - -typedef struct { - erts_rwmtx_t rwmtx; - Hash hash; - char name[32]; -} ErtsPidRefTable; - -typedef struct { - union { - ErtsPidRefTable table; - char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPidRefTable))]; - } u; -} ErtsAlignedPidRefTable; - -ErtsAlignedPidRefTable *pid_ref_table; - -Eterm -erts_pid_ref_lookup__(Uint32 refn[ERTS_REF_NUMBERS]) -{ - ErtsPidRefTableEntry tmpl; - ErtsPidRefTableEntry *tep; - ErtsPidRefTable *tblp; - Eterm pid; - - ASSERT(erts_is_pid_ref_numbers(refn)); - - tmpl.value = erts_get_ref_numbers_value(refn); - tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); - if (tmpl.thr_id > erts_no_schedulers) - tblp = &pid_ref_table[0].u.table; - else - tblp = &pid_ref_table[tmpl.thr_id].u.table; - - erts_rwmtx_rlock(&tblp->rwmtx); - - tep = (ErtsPidRefTableEntry *) hash_get(&tblp->hash, &tmpl); - pid = tep ? tep->pid : THE_NON_VALUE; - - erts_rwmtx_runlock(&tblp->rwmtx); - - return pid; -} - -static void -pid_ref_save(Uint32 refn[ERTS_REF_NUMBERS], Eterm pid) -{ - ErtsPidRefTableEntry tmpl; - ErtsPidRefTableEntry *tep; - ErtsPidRefTable *tblp; - - ASSERT(erts_is_pid_ref_numbers(refn)); - - tmpl.value = erts_get_ref_numbers_value(refn); - tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); - tmpl.pid = pid; - - if (tmpl.thr_id > erts_no_schedulers) - tblp = &pid_ref_table[0].u.table; - else - tblp = &pid_ref_table[tmpl.thr_id].u.table; - - erts_rwmtx_rlock(&tblp->rwmtx); - - tep = (ErtsPidRefTableEntry *) hash_get(&tblp->hash, &tmpl); - - erts_rwmtx_runlock(&tblp->rwmtx); - - if (!tep) { - ErtsPidRefTableEntry *used_tep; - - ASSERT(tmpl.value == erts_get_ref_numbers_value(refn)); - ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn)); - - if (tblp != &pid_ref_table[0].u.table) { - tep = erts_alloc(ERTS_ALC_T_PREF_NSCHED_ENT, - sizeof(ErtsNSchedPidRefTableEntry)); - } - else { - tep = erts_alloc(ERTS_ALC_T_PREF_ENT, - sizeof(ErtsPidRefTableEntry)); - tep->thr_id = tmpl.thr_id; - } - - tep->value = tmpl.value; - tep->pid = pid; - - erts_rwmtx_rwlock(&tblp->rwmtx); - - used_tep = hash_put(&tblp->hash, tep); - - erts_rwmtx_rwunlock(&tblp->rwmtx); - - if (used_tep != tep) { - if (tblp != &pid_ref_table[0].u.table) - erts_free(ERTS_ALC_T_PREF_NSCHED_ENT, (void *) tep); - else - erts_free(ERTS_ALC_T_PREF_ENT, (void *) tep); - } - } -} - -void -erts_pid_ref_delete(Eterm ref) -{ - ErtsPidRefTableEntry tmpl; - ErtsPidRefTableEntry *tep; - ErtsPidRefTable *tblp; - Uint32 *refn; - - ASSERT(is_internal_pid_ref(ref)); - - refn = internal_pid_ref_numbers(ref); - - ASSERT(erts_is_pid_ref_numbers(refn)); - - tmpl.value = erts_get_ref_numbers_value(refn); - tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); - - if (tmpl.thr_id > erts_no_schedulers) - tblp = &pid_ref_table[0].u.table; - else - tblp = &pid_ref_table[tmpl.thr_id].u.table; - - erts_rwmtx_rlock(&tblp->rwmtx); - - tep = (ErtsPidRefTableEntry *) hash_get(&tblp->hash, &tmpl); - - erts_rwmtx_runlock(&tblp->rwmtx); - - if (tep) { - - ASSERT(tmpl.value == erts_get_ref_numbers_value(refn)); - ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn)); - - erts_rwmtx_rwlock(&tblp->rwmtx); - - tep = hash_remove(&tblp->hash, &tmpl); - ASSERT(tep); - - erts_rwmtx_rwunlock(&tblp->rwmtx); - - if (tblp != &pid_ref_table[0].u.table) - erts_free(ERTS_ALC_T_PREF_NSCHED_ENT, (void *) tep); - else - erts_free(ERTS_ALC_T_PREF_ENT, (void *) tep); - } -} - -static int nsched_preft_cmp(void *ve1, void *ve2) -{ - ErtsNSchedPidRefTableEntry *e1 = ve1; - ErtsNSchedPidRefTableEntry *e2 = ve2; - return e1->value != e2->value; -} - -static int non_nsched_preft_cmp(void *ve1, void *ve2) -{ - ErtsPidRefTableEntry *e1 = ve1; - ErtsPidRefTableEntry *e2 = ve2; - return e1->value != e2->value || e1->thr_id != e2->thr_id; -} - -static HashValue nsched_preft_hash(void *ve) -{ - ErtsNSchedPidRefTableEntry *e = ve; - return (HashValue) e->value; -} - -static HashValue non_nsched_preft_hash(void *ve) -{ - ErtsPidRefTableEntry *e = ve; - HashValue h; - h = (HashValue) e->thr_id; - h *= 268440163; - h += (HashValue) e->value; - return h; -} - -static void *preft_alloc(void *ve) -{ - /* - * We allocate the element before - * hash_put() and pass it as - * template which we get as - * input... - */ - return ve; -} - -static void preft_free(void *ve) -{ - /* - * We free the element ourselves - * after hash_remove()... - */ -} - -static void *preft_meta_alloc(int i, size_t size) -{ - return erts_alloc(ERTS_ALC_T_PREF_TAB_BKTS, size); -} - -static void preft_meta_free(int i, void *ptr) -{ - erts_free(ERTS_ALC_T_PREF_TAB_BKTS, ptr); -} - -static void -init_pid_ref_tables(void) -{ - HashFunctions hash_funcs; - int i; - ErtsPidRefTable *tblp; - - pid_ref_table = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PREF_TAB, - (sizeof(ErtsAlignedPidRefTable) - * (erts_no_schedulers + 1))); - - hash_funcs.hash = non_nsched_preft_hash; - hash_funcs.cmp = non_nsched_preft_cmp; - - hash_funcs.alloc = preft_alloc; - hash_funcs.free = preft_free; - hash_funcs.meta_alloc = preft_meta_alloc; - hash_funcs.meta_free = preft_meta_free; - hash_funcs.meta_print = erts_print; - - tblp = &pid_ref_table[0].u.table; - erts_snprintf(&tblp->name[0], sizeof(tblp->name), - "pid_ref_table_0"); - hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); - erts_rwmtx_init(&tblp->rwmtx, "pid_ref_table", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); - - hash_funcs.hash = nsched_preft_hash; - hash_funcs.cmp = nsched_preft_cmp; - - for (i = 1; i <= erts_no_schedulers; i++) { - ErtsPidRefTable *tblp = &pid_ref_table[i].u.table; - erts_snprintf(&tblp->name[0], sizeof(tblp->name), - "pid_ref_table_%d", i); - hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); - erts_rwmtx_init(&tblp->rwmtx, "pid_ref_table", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); - } -} - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Unique Integer * \* */ diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 58426b0b55..f88094046d 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -40,8 +40,6 @@ void erts_make_magic_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]); void erts_magic_ref_remove_bin(Uint32 refn[ERTS_REF_NUMBERS]); void erts_magic_ref_save_bin__(Eterm ref); ErtsMagicBinary *erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]); -void erts_pid_ref_delete(Eterm ref); -Eterm erts_pid_ref_lookup__(Uint32 refn[ERTS_REF_NUMBERS]); /* strict monotonic counter */ @@ -96,7 +94,7 @@ ERTS_GLB_INLINE Eterm erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *mb ERTS_GLB_INLINE Binary *erts_magic_ref2bin(Eterm mref); ERTS_GLB_INLINE void erts_magic_ref_save_bin(Eterm ref); ERTS_GLB_INLINE ErtsMagicBinary *erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS]); -ERTS_GLB_INLINE Eterm erts_pid_ref_lookup(Uint32 *refn); +ERTS_GLB_INLINE Eterm erts_pid_ref_lookup(Uint32 *refn, int len); ERTS_GLB_INLINE Eterm erts_get_pid_of_ref(Eterm ref); #define ERTS_REF1_MAGIC_MARKER_BIT_NO__ \ @@ -256,11 +254,24 @@ erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS]) * from the outside world... */ ERTS_GLB_INLINE Eterm -erts_pid_ref_lookup(Uint32 *refn) +erts_pid_ref_lookup(Uint32 *refn, int len) { + Eterm pid; + if (len != ERTS_PID_REF_NUMBERS) + return am_undefined; if (!erts_is_pid_ref_numbers(refn)) - return THE_NON_VALUE; - return erts_pid_ref_lookup__(refn); + return am_undefined; + + pid = (((Eterm) refn[3]) +#ifdef ARCH_64 + | (((Eterm) refn[4]) << 32) +#endif + ); + + if (!is_internal_pid(pid)) + return THE_NON_VALUE; /* We got garbage; we have not created this... */ + + return pid; } ERTS_GLB_INLINE Eterm erts_get_pid_of_ref(Eterm ref) diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 4596c65959..694aabd852 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -51,23 +51,20 @@ typedef struct erl_bin_match_buffer { struct erl_bits_state { /* - * Used for building binaries. + * Temporary buffer sometimes used by erts_new_bs_put_integer(). */ byte *byte_buf_; int byte_buf_len_; + /* - * Used for building binaries using the new instruction set. + * Pointer to the beginning of the current binary. */ - byte* erts_current_bin_; /* Pointer to beginning of current binary. */ + byte* erts_current_bin_; + /* - * Offset in bits into the current binary (new instruction set) or - * buffer (old instruction set). + * Offset in bits into the current binary. */ Uint erts_bin_offset_; - /* - * Whether the current binary is writable. - */ - unsigned erts_writable_bin_; }; typedef struct erl_bin_match_struct{ @@ -117,7 +114,6 @@ typedef struct erl_bin_match_struct{ #define erts_bin_offset (ErlBitsState.erts_bin_offset_) #define erts_current_bin (ErlBitsState.erts_current_bin_) -#define erts_writable_bin (ErlBitsState.erts_writable_bin_) #define copy_binary_to_buffer(DstBuffer, DstBufOffset, SrcBuffer, SrcBufferOffset, NumBits) \ do { \ @@ -151,6 +147,11 @@ void erts_bits_destroy_state(ERL_BITS_PROTO_0); #define WSIZE(n) ((n + sizeof(Eterm) - 1) / sizeof(Eterm)) /* + * Define the maximum number of bits in a unit for the binary syntax. + */ +#define ERL_UNIT_BITS 8 + +/* * Binary matching. */ @@ -192,7 +193,7 @@ Eterm erts_bs_init_writable(Process* p, Eterm sz); * Common utilities. */ void erts_copy_bits(byte* src, size_t soffs, int sdir, - byte* dst, size_t doffs,int ddir, size_t n); + byte* dst, size_t doffs,int ddir, size_t n); int erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t size); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 9ce88c35d6..a50ab2d00c 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -507,13 +507,13 @@ save_sched_table(Process *c_p, DbTable *tb) DbTable *first; ASSERT(esdp); - erts_atomic_inc_nob(&esdp->ets_tables.count); + erts_atomic_inc_nob(&esdp->u.ets_tables.count); erts_refc_inc(&tb->common.refc, 1); - first = esdp->ets_tables.clist; + first = esdp->u.ets_tables.clist; if (!first) { tb->common.all.next = tb->common.all.prev = tb; - esdp->ets_tables.clist = tb; + esdp->u.ets_tables.clist = tb; } else { tb->common.all.prev = first->common.all.prev; @@ -531,14 +531,14 @@ remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb) ASSERT(erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid)) == (Uint32) esdp->no); - ASSERT(erts_atomic_read_nob(&esdp->ets_tables.count) > 0); - erts_atomic_dec_nob(&esdp->ets_tables.count); + ASSERT(erts_atomic_read_nob(&esdp->u.ets_tables.count) > 0); + erts_atomic_dec_nob(&esdp->u.ets_tables.count); eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all); if (eaydp->ongoing) { /* ets:all() op process list from last to first... */ if (eaydp->tab == tb) { - if (eaydp->tab == esdp->ets_tables.clist) + if (eaydp->tab == esdp->u.ets_tables.clist) eaydp->tab = NULL; else eaydp->tab = tb->common.all.prev; @@ -547,23 +547,23 @@ remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb) if (tb->common.all.next == tb) { ASSERT(tb->common.all.prev == tb); - ASSERT(esdp->ets_tables.clist == tb); - esdp->ets_tables.clist = NULL; + ASSERT(esdp->u.ets_tables.clist == tb); + esdp->u.ets_tables.clist = NULL; } else { #ifdef DEBUG - DbTable *tmp = esdp->ets_tables.clist; + DbTable *tmp = esdp->u.ets_tables.clist; do { if (tmp == tb) break; tmp = tmp->common.all.next; - } while (tmp != esdp->ets_tables.clist); + } while (tmp != esdp->u.ets_tables.clist); ASSERT(tmp == tb); #endif tb->common.all.prev->common.all.next = tb->common.all.next; tb->common.all.next->common.all.prev = tb->common.all.prev; - if (esdp->ets_tables.clist == tb) - esdp->ets_tables.clist = tb->common.all.next; + if (esdp->u.ets_tables.clist == tb) + esdp->u.ets_tables.clist = tb->common.all.next; } @@ -2764,8 +2764,44 @@ BIF_RETTYPE ets_lookup_element_3(BIF_ALIST_3) } } -/* - * BIF to erase a whole table and release all memory it holds +/* +** Get an element from a term +** get_element_4(Tab, Key, Index, Default) +** return the element or a list of elements if bag or Default if the element is not present +*/ +BIF_RETTYPE ets_lookup_element_4(BIF_ALIST_4) +{ + DbTable* tb; + Sint index; + int cret; + Eterm ret; + + CHECK_TABLES(); + + DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_lookup_element_4); + + if (is_not_small(BIF_ARG_3) || ((index = signed_val(BIF_ARG_3)) < 1)) { + db_unlock(tb, LCK_READ); + BIF_ERROR(BIF_P, BADARG); + } + + cret = tb->common.meth->db_get_element(BIF_P, tb, + BIF_ARG_2, index, &ret); + db_unlock(tb, LCK_READ); + switch (cret) { + case DB_ERROR_NONE: + BIF_RET(ret); + case DB_ERROR_BADKEY: + BIF_RET(BIF_ARG_4); + case DB_ERROR_SYSRES: + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + default: + BIF_ERROR(BIF_P, BADARG); + } +} + +/* + * BIF to erase a whole table and release all memory it holds */ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { @@ -3310,7 +3346,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, hp = &hfragp->mem[hfragp->used_size]; list = *hp; hfragp->used_size = hfragp->alloc_size; - first = esdp->ets_tables.clist; + first = esdp->u.ets_tables.clist; tb = *tablepp; } else { @@ -3318,7 +3354,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, ASSERT(!*tablepp); /* Max heap size needed... */ - sz = erts_atomic_read_nob(&esdp->ets_tables.count); + sz = erts_atomic_read_nob(&esdp->u.ets_tables.count); sz *= ERTS_MAGIC_REF_THING_SIZE + 2; sz += 3 + ERTS_REF_THING_SIZE; hfragp = new_message_buffer(sz); @@ -3326,7 +3362,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, hp = &hfragp->mem[0]; ohp = &hfragp->off_heap; list = NIL; - first = esdp->ets_tables.clist; + first = esdp->u.ets_tables.clist; tb = first ? first->common.all.prev : NULL; } @@ -3412,7 +3448,7 @@ erts_handle_yielded_ets_all_request(ErtsAuxWorkData *awdp) return 0; /* All work completed! */ if (yc < ERTS_ETS_ALL_TB_YCNT_START && - yc > erts_atomic_read_nob(&esdp->ets_tables.count)) + yc > erts_atomic_read_nob(&esdp->u.ets_tables.count)) return 1; /* Yield! */ eaydp->ongoing = ongoing = eaydp->queue; @@ -4614,8 +4650,8 @@ erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp) eaydp->hfrag = NULL; eaydp->tab = NULL; eaydp->queue = NULL; - esdp->ets_tables.clist = NULL; - erts_atomic_init_nob(&esdp->ets_tables.count, 0); + esdp->u.ets_tables.clist = NULL; + erts_atomic_init_nob(&esdp->u.ets_tables.count, 0); } @@ -5454,7 +5490,7 @@ erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg, int alive_only for (ix = 0; ix < erts_no_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); - DbTable *first = esdp->ets_tables.clist; + DbTable *first = esdp->u.ets_tables.clist; if (first) { DbTable *tb = first; do { @@ -5498,7 +5534,7 @@ Uint erts_ets_table_count(void) for (six = 0; six < erts_no_schedulers; six++) { ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd; - tb_count += erts_atomic_read_nob(&esdp->ets_tables.count); + tb_count += erts_atomic_read_nob(&esdp->u.ets_tables.count); } return tb_count; } @@ -5565,7 +5601,7 @@ static void lcnt_update_db_locks_per_sched(void *enable) { DbTable *head; esdp = erts_get_scheduler_data(); - head = esdp->ets_tables.clist; + head = esdp->u.ets_tables.clist; if(head) { DbTable *iterator = head; diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab index 3f16f3e0f3..9245a19be6 100644 --- a/erts/emulator/beam/erl_dirty_bif.tab +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -50,6 +50,7 @@ dirty-io erts_debug:dirty_io/2 dirty-cpu erts_debug:lcnt_control/2 dirty-cpu erts_debug:lcnt_collect/0 dirty-cpu erts_debug:lcnt_clear/0 +dirty-cpu erlang:display_string/2 # --- TEST of Dirty BIF functionality --- # Functions below will execute on dirty schedulers when emulator has diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 95e41226fc..a4f6a1b723 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -275,7 +275,7 @@ static ERTS_INLINE void set_default_time_adj(int *time_correction_p, ErtsTimeWarpMode *time_warp_mode_p) { *time_correction_p = 1; - *time_warp_mode_p = ERTS_NO_TIME_WARP_MODE; + *time_warp_mode_p = ERTS_MULTI_TIME_WARP_MODE; if (!erts_check_time_adj_support(*time_correction_p, *time_warp_mode_p)) { *time_correction_p = 0; @@ -652,6 +652,7 @@ void erts_usage(void) #ifdef BEAMASM erts_fprintf(stderr, "-JDdump bool enable or disable dumping of generated assembly code for each module loaded\n"); erts_fprintf(stderr, "-JPperf bool enable or disable support for perf on Linux\n"); + erts_fprintf(stderr, "-JMsingle bool enable the use of single-mapped RWX memory for JIT:ed code\n"); erts_fprintf(stderr, "\n"); #endif @@ -1730,6 +1731,23 @@ erl_start(int argc, char **argv) #endif } break; + case 'M': + sub_param++; + if (has_prefix("single", sub_param)) { + arg = get_arg(sub_param+6, argv[i + 1], &i); + if (sys_strcmp(arg, "true") == 0) { + erts_jit_single_map = 1; + } else if (sys_strcmp(arg, "false") == 0) { + erts_jit_single_map = 0; + } else { + erts_fprintf(stderr, "bad +JMsingle support flag %s\n", arg); + erts_usage(); + } + } else { + erts_fprintf(stderr, "bad +JM sub-option %s\n", arg); + erts_usage(); + } + break; default: erts_fprintf(stderr, "invalid JIT option %s\n", argv[i]); break; @@ -2538,7 +2556,7 @@ __decl_noreturn void erts_thr_fatal_error(int err, const char *what) static void -system_cleanup(int flush_async) +system_cleanup(int flush) { /* * Make sure only one thread exits the runtime system. @@ -2568,24 +2586,43 @@ system_cleanup(int flush_async) * (in threaded non smp case). */ - if (!flush_async - || !erts_initialized - ) + if (!flush || !erts_initialized) return; + /* + * We only flush as a result of calling erts_halt() (which in turn + * is called from the erlang:halt() BIF when flushing is enabled); + * otherwise, flushing wont work properly. If erts_halt() has + * been called, 'erts_halt_code' won't equal INT_MIN... + */ + ASSERT(erts_halt_code != INT_MIN); + + /* + * Nif on-halt handlers may have been added after we initiated + * a halt. If so, make sure that these late added handlers are + * executed as well.. + */ + erts_nif_execute_on_halt(); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif erts_exit_flush_async(); + + /* + * Wait for all NIF calls with delayed halt functionality + * enabled to complete before we continue... + */ + erts_nif_wait_calls(); } static int erts_exit_code; static __decl_noreturn void __noreturn -erts_exit_vv(int n, int flush_async, const char *fmt, va_list args1, va_list args2) +erts_exit_vv(int n, int flush, const char *fmt, va_list args1, va_list args2) { - system_cleanup(flush_async); + system_cleanup(flush); if (fmt != NULL && *fmt != '\0') erl_error(fmt, args2); /* Print error message. */ @@ -2598,25 +2635,25 @@ erts_exit_vv(int n, int flush_async, const char *fmt, va_list args1, va_list arg erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - erts_exit_epilogue(); + erts_exit_epilogue(flush); } -__decl_noreturn void __noreturn erts_exit_epilogue(void) +__decl_noreturn void __noreturn erts_exit_epilogue(int flush) { int n = erts_exit_code; sys_tty_reset(n); if (n == ERTS_INTR_EXIT) - exit(0); + (void) (flush ? exit(0) : _exit(0)); else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT) abort(); - exit(n); + (void) (flush ? exit(n) : _exit(n)); } -/* Exit without flushing async threads */ +/* Exit without flushing */ __decl_noreturn void __noreturn erts_exit(int n, const char *fmt, ...) { va_list args1, args2; @@ -2627,8 +2664,12 @@ __decl_noreturn void __noreturn erts_exit(int n, const char *fmt, ...) va_end(args1); } -/* Exit after flushing async threads */ -__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...) +/* + * Exit after flushing. This is a continuation of erts_halt() and wont + * work properly if called by its own without proper initialization + * as made in erts_halt(). + */ +__decl_noreturn void __noreturn erts_flush_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index c6f127e5a4..f299889dcc 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -108,6 +108,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "fun_tab", NULL }, { "environ", NULL }, { "release_literal_areas", NULL }, + { "on_halt", NULL }, { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 02ab64ec95..12cf76c66a 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -261,12 +261,14 @@ int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref); /** @brief Convenience macro to re/enable counting on an already initialized * reference. Don't forget to specify the lock type in \c flags! */ -#define erts_lcnt_install_new_lock_info(ref, name, id, flags) \ - if(!erts_lcnt_check_ref_installed(ref)) { \ - erts_lcnt_lock_info_carrier_t *__carrier; \ - __carrier = erts_lcnt_create_lock_info_carrier(1);\ - erts_lcnt_init_lock_info_idx(__carrier, 0, name, id, flags); \ - erts_lcnt_install(ref, __carrier);\ +#define erts_lcnt_install_new_lock_info(ref, name, id, flags) \ + do { \ + if(!erts_lcnt_check_ref_installed(ref)) { \ + erts_lcnt_lock_info_carrier_t *__carrier; \ + __carrier = erts_lcnt_create_lock_info_carrier(1); \ + erts_lcnt_init_lock_info_idx(__carrier, 0, name, id, flags); \ + erts_lcnt_install(ref, __carrier); \ + } \ } while(0) erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count); diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index e0b881db0a..cb78bfe0f2 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1378,7 +1378,7 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { vs2 = flatmap_get_values(mp2); while(i1 < n1 && i2 < n2) { - c = CMP_TERM(ks1[i1],ks2[i2]); + c = (ks1[i1] == ks2[i2]) ? 0 : CMP_TERM(ks1[i1],ks2[i2]); if (c == 0) { /* use righthand side arguments map value, * but advance both maps */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 50ac6b92d9..b63c220625 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -71,6 +71,17 @@ #include <limits.h> #include <stddef.h> /* offsetof */ +#define ERTS_NIF_HALT_INFO_FLAG_BLOCK (1 << 0) +#define ERTS_NIF_HALT_INFO_FLAG_HALTING (1 << 1) +#define ERTS_NIF_HALT_INFO_FLAG_WAITING (1 << 2) + +typedef struct ErtsNifOnHaltData_ ErtsNifOnHaltData; +struct ErtsNifOnHaltData_ { + ErtsNifOnHaltData *next; + ErtsNifOnHaltData *prev; + ErlNifOnHaltCallback *callback; +}; + /* Information about a loaded nif library. * Each successful call to erlang:load_nif will allocate an instance of * erl_module_nif. Two calls opening the same library will thus have the same @@ -95,11 +106,21 @@ struct erl_module_nif { +1 for each owned resource type with callbacks +1 for each ongoing dirty NIF call */ + int flags; + ErtsNifOnHaltData on_halt; Module* mod; /* Can be NULL if purged and dynlib_refc > 0 */ ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; +#define ERTS_MOD_NIF_FLG_LOADING (1 << 0) +#define ERTS_MOD_NIF_FLG_DELAY_HALT (1 << 1) +#define ERTS_MOD_NIF_FLG_ON_HALT (1 << 2) + +static erts_atomic_t halt_tse; +static erts_mtx_t on_halt_mtx; +static ErtsNifOnHaltData *on_halt_requests; + typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); #ifdef DEBUG @@ -131,6 +152,8 @@ void dtrace_nifenv_str(ErlNifEnv *, char *); #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp); +static void install_on_halt_callback(ErtsNifOnHaltData *ohdp); +static void uninstall_on_halt_callback(ErtsNifOnHaltData *ohdp); static ERTS_INLINE int is_scheduler(void) @@ -370,6 +393,25 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, return (ERL_NIF_TERM) THE_NON_VALUE; } +static ERTS_NOINLINE void +eternal_sleep(void) +{ + while (!0) + erts_milli_sleep(1000*1000); +} + +static ERTS_NOINLINE void +handle_halting_unblocked_halt(erts_aint32_t info) +{ + if (info & ERTS_NIF_HALT_INFO_FLAG_WAITING) { + erts_tse_t *tse; + ERTS_THR_MEMORY_BARRIER; + tse = (erts_tse_t *) erts_atomic_read_nob(&halt_tse); + ASSERT(tse); + erts_tse_set(tse); + } + eternal_sleep(); +} static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -419,7 +461,34 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + if (!(env.mod_nif->flags & ERTS_MOD_NIF_FLG_DELAY_HALT)) { + result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + } + else { + erts_atomic32_t *dirty_nif_halt_info = &esdp->u.dirty_nif_halt_info; + erts_aint32_t info; + info = erts_atomic32_cmpxchg_nob(dirty_nif_halt_info, + ERTS_NIF_HALT_INFO_FLAG_BLOCK, + 0); + if (info != 0) { + ASSERT(info == ERTS_NIF_HALT_INFO_FLAG_HALTING + || info == (ERTS_NIF_HALT_INFO_FLAG_HALTING + | ERTS_NIF_HALT_INFO_FLAG_WAITING)); + eternal_sleep(); + } + result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + info = erts_atomic32_read_band_relb(dirty_nif_halt_info, + ~ERTS_NIF_HALT_INFO_FLAG_BLOCK); + if (info & ERTS_NIF_HALT_INFO_FLAG_HALTING) { + ASSERT(info == (ERTS_NIF_HALT_INFO_FLAG_BLOCK + | ERTS_NIF_HALT_INFO_FLAG_HALTING) + || info == (ERTS_NIF_HALT_INFO_FLAG_BLOCK + | ERTS_NIF_HALT_INFO_FLAG_HALTING + | ERTS_NIF_HALT_INFO_FLAG_WAITING)); + handle_halting_unblocked_halt(info); + } + ASSERT(info == ERTS_NIF_HALT_INFO_FLAG_BLOCK); + } erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); @@ -2274,9 +2343,11 @@ static void close_dynlib(struct erl_module_nif* lib) { ASSERT(lib != NULL); ASSERT(lib->mod == NULL); - ASSERT(lib->handle != NULL); ASSERT(erts_refc_read(&lib->dynlib_refc,0) == 0); + if (lib->flags & ERTS_MOD_NIF_FLG_ON_HALT) + uninstall_on_halt_callback(&lib->on_halt); + if (lib->entry.unload != NULL) { struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, lib, NULL); @@ -4524,6 +4595,8 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args) lib->handle = handle; erts_refc_init(&lib->refc, 2); /* Erlang code + NIF code */ erts_refc_init(&lib->dynlib_refc, 1); + lib->flags = 0; + lib->on_halt.callback = NULL; ASSERT(opened_rt_list == NULL); lib->mod = module_p; @@ -4642,7 +4715,7 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args) /* Call load or upgrade: */ env.mod_nif = lib; - + lib->flags |= ERTS_MOD_NIF_FLG_LOADING; lib->priv_data = NULL; if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ void* prev_old_data = prev_mi->nif->priv_data; @@ -4676,11 +4749,14 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args) ret = load_nif_error(c_p, "load", "Library load-call unsuccessful (%d).", veto); } } + lib->flags &= ~ERTS_MOD_NIF_FLG_LOADING; if (ret == am_ok) { /* * Everything ok, make NIF code callable. */ + if (lib->flags & ERTS_MOD_NIF_FLG_ON_HALT) + install_on_halt_callback(&lib->on_halt); this_mi->nif = lib; prepare_opened_rt(lib); /* @@ -5046,6 +5122,20 @@ void erl_nif_init() nif_call_table_init(); static_nifs_init(); + + erts_atomic_init_nob(&halt_tse, (erts_aint_t) NULL); + erts_mtx_init(&on_halt_mtx, "on_halt", NIL, + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + on_halt_requests = NULL; +} + +void +erts_nif_sched_init(ErtsSchedulerData *esdp) +{ + if (esdp->type == ERTS_SCHED_DIRTY_CPU + || esdp->type == ERTS_SCHED_DIRTY_IO) { + erts_atomic32_init_nob(&esdp->u.dirty_nif_halt_info, 0); + } } int erts_nif_get_funcs(struct erl_module_nif* mod, @@ -5133,6 +5223,191 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, return nif_result; } +/* + * Set options... + */ + +int +enif_set_option(ErlNifEnv *env, ErlNifOption opt, ...) +{ + if (!env) + return EINVAL; + + switch (opt) { + + case ERL_NIF_OPT_DELAY_HALT: { + struct erl_module_nif *m = env->mod_nif; + + if (!m || !(m->flags & ERTS_MOD_NIF_FLG_LOADING) + || (m->flags & ERTS_MOD_NIF_FLG_DELAY_HALT)) { + return EINVAL; + } + + m->flags |= ERTS_MOD_NIF_FLG_DELAY_HALT; + + return 0; + } + + case ERL_NIF_OPT_ON_HALT: { + struct erl_module_nif *m = env->mod_nif; + ErlNifOnHaltCallback *on_halt; + va_list argp; + + if (!m || ((m->flags & (ERTS_MOD_NIF_FLG_LOADING + | ERTS_MOD_NIF_FLG_ON_HALT)) + != ERTS_MOD_NIF_FLG_LOADING)) { + return EINVAL; + } + + ASSERT(!m->on_halt.callback); + + va_start(argp, opt); + on_halt = va_arg(argp, ErlNifOnHaltCallback *); + va_end(argp); + if (!on_halt) + return EINVAL; + + m->on_halt.callback = on_halt; + m->flags |= ERTS_MOD_NIF_FLG_ON_HALT; + + return 0; + } + + default: + return EINVAL; + + } +} + +/* + * Halt functionality... + */ + +void +erts_nif_execute_on_halt(void) +{ + ErtsNifOnHaltData *ohdp; + + erts_mtx_lock(&on_halt_mtx); + for (ohdp = on_halt_requests; ohdp; ohdp = ohdp->next) { + struct erl_module_nif *m; + m = ErtsContainerStruct(ohdp, struct erl_module_nif, on_halt); + ohdp->callback(m->priv_data); + } + on_halt_requests = NULL; + erts_mtx_unlock(&on_halt_mtx); +} + +void +erts_nif_notify_halt(void) +{ + int ix; + + erts_nif_execute_on_halt(); + + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + ASSERT(esdp->type == ERTS_SCHED_DIRTY_CPU); + (void) erts_atomic32_read_bor_nob(&esdp->u.dirty_nif_halt_info, + ERTS_NIF_HALT_INFO_FLAG_HALTING); + } + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); + ASSERT(esdp->type == ERTS_SCHED_DIRTY_IO); + (void) erts_atomic32_read_bor_nob(&esdp->u.dirty_nif_halt_info, + ERTS_NIF_HALT_INFO_FLAG_HALTING); + } +} + +static ERTS_INLINE void +wait_dirty_call_blocking_halt(ErtsSchedulerData *esdp, erts_tse_t *tse) +{ + erts_atomic32_t *dirty_nif_halt_info = &esdp->u.dirty_nif_halt_info; + erts_aint32_t info = erts_atomic32_read_acqb(dirty_nif_halt_info); + ASSERT(info & ERTS_NIF_HALT_INFO_FLAG_HALTING); + if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK)) + return; + info = erts_atomic32_read_bor_mb(dirty_nif_halt_info, + ERTS_NIF_HALT_INFO_FLAG_WAITING); + if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK)) + return; + while (!0) { + erts_tse_reset(tse); + info = erts_atomic32_read_acqb(dirty_nif_halt_info); + if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK)) + return; + while (!0) { + if (erts_tse_wait(tse) != EINTR) + break; + } + } +} + +void +erts_nif_wait_calls(void) +{ + erts_tse_t *tse; + int ix; + + tse = erts_tse_fetch(); + erts_atomic_set_nob(&halt_tse, (erts_aint_t) tse); + + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + ASSERT(esdp->type == ERTS_SCHED_DIRTY_CPU); + wait_dirty_call_blocking_halt(esdp, tse); + } + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); + ASSERT(esdp->type == ERTS_SCHED_DIRTY_IO); + wait_dirty_call_blocking_halt(esdp, tse); + } +} + +static void +install_on_halt_callback(ErtsNifOnHaltData *ohdp) +{ + ASSERT(ohdp->callback); + + erts_mtx_lock(&on_halt_mtx); + ohdp->next = on_halt_requests; + ohdp->prev = NULL; + if (on_halt_requests) + on_halt_requests->prev = ohdp; + on_halt_requests = ohdp; + erts_mtx_unlock(&on_halt_mtx); +} + +static void +uninstall_on_halt_callback(ErtsNifOnHaltData *ohdp) +{ + erts_mtx_lock(&on_halt_mtx); + ohdp->callback = NULL; + if (ohdp->prev) { + ASSERT(on_halt_requests != ohdp); + ohdp->prev->next = ohdp->next; + } + else if (on_halt_requests == ohdp) { + ASSERT(erts_halt_code == INT_MIN); + on_halt_requests = ohdp->next; + } + else { + /* + * Uninstall during halt; and our callback + * has already been called... + */ + ASSERT(erts_halt_code != INT_MIN); + } + if (ohdp->next) { + ohdp->next->prev = ohdp->prev; + } + erts_mtx_unlock(&on_halt_mtx); +} + +/* + * End of halt functionality... + */ + #ifdef USE_VM_PROBES void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index a9a81483f6..1ed1201cf2 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -57,9 +57,10 @@ ** 2.15: 22.0 ERL_NIF_SELECT_CANCEL, enif_select_(read|write) ** enif_term_type ** 2.16: 24.0 enif_init_resource_type, enif_dynamic_resource_call +** 2.17 26.0 enif_set_option */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 16 +#define ERL_NIF_MINOR_VERSION 17 /* * WHEN CHANGING INTERFACE VERSION, also replace erts version below with @@ -70,7 +71,7 @@ * If you're not on the OTP team, you should use a placeholder like * erts-@MyName@ instead. */ -#define ERL_NIF_MIN_ERTS_VERSION "erts-12.0" +#define ERL_NIF_MIN_ERTS_VERSION "erts-@OTP-17771@" /* * The emulator will refuse to load a nif-lib with a major version @@ -201,6 +202,8 @@ typedef struct typedef ErlDrvMonitor ErlNifMonitor; +typedef void ErlNifOnHaltCallback(void *priv_data); + typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call); @@ -325,6 +328,11 @@ typedef enum { #define ERL_NIF_THR_DIRTY_CPU_SCHEDULER 2 #define ERL_NIF_THR_DIRTY_IO_SCHEDULER 3 +typedef enum { + ERL_NIF_OPT_DELAY_HALT = 1, + ERL_NIF_OPT_ON_HALT = 2 +} ErlNifOption; + #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index f9004eb64b..a2a1052b59 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -219,6 +219,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifTermType,enif_term_type,(ErlNifEnv* env, ERL_NIF_TER ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_init_resource_type,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); ERL_NIF_API_FUNC_DECL(int,enif_dynamic_resource_call,(ErlNifEnv*, ERL_NIF_TERM mod, ERL_NIF_TERM name, ERL_NIF_TERM rsrc, void* call_data)); +ERL_NIF_API_FUNC_DECL(int, enif_set_option, (ErlNifEnv *env, ErlNifOption opt, ...)); + /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ @@ -408,6 +410,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_dynamic_resource_call,(ErlNifEnv*, ERL_NIF_TERM m # define enif_term_type ERL_NIF_API_FUNC_MACRO(enif_term_type) # define enif_init_resource_type ERL_NIF_API_FUNC_MACRO(enif_init_resource_type) # define enif_dynamic_resource_call ERL_NIF_API_FUNC_MACRO(enif_dynamic_resource_call) +# define enif_set_option ERL_NIF_API_FUNC_MACRO(enif_set_option) /* ** ADD NEW ENTRIES HERE (before this comment) */ diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index a92f031971..6d787a0e18 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -256,7 +256,9 @@ extern ErtsPTab erts_port; * Refs * \* */ -#define internal_ref_no_numbers(x) ERTS_REF_NUMBERS +#define internal_ref_no_numbers(x) (is_internal_pid_ref((x)) \ + ? ERTS_PID_REF_NUMBERS \ + : ERTS_REF_NUMBERS) #define internal_ref_numbers(x) (is_internal_magic_ref((x)) \ ? internal_magic_ref_numbers((x)) \ : internal_non_magic_ref_numbers((x))) diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 1d80ac6793..0997519e7d 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -5312,8 +5312,6 @@ handle_alias_message(Process *c_p, ErtsMessage *sig, ErtsMessage ***next_nm_sig) erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), mon); - erts_pid_ref_delete(alias); - switch (mon->type) { case ERTS_MON_TYPE_DIST_PORT: case ERTS_MON_TYPE_ALIAS: @@ -5639,7 +5637,6 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, case ERTS_ML_STATE_ALIAS_ONCE: case ERTS_ML_STATE_ALIAS_DEMONITOR: ASSERT(is_internal_pid_ref(mdp->ref)); - erts_pid_ref_delete(mdp->ref); /* fall through... */ default: if (type != ERTS_MON_TYPE_NODE) @@ -5676,7 +5673,6 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, if ((mon->flags & ERTS_ML_STATE_ALIAS_MASK) == ERTS_ML_STATE_ALIAS_ONCE) { mon->flags &= ~ERTS_ML_STATE_ALIAS_MASK; - erts_pid_ref_delete(key); } } else { diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5c9bf248f2..b40e52ff21 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2508,7 +2508,7 @@ notify_reap_ports_relb(void) } erts_atomic32_t erts_halt_progress; -int erts_halt_code; +int erts_halt_code = INT_MIN; static ERTS_INLINE erts_aint32_t handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) @@ -2553,7 +2553,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erts_flush_async_exit(erts_halt_code, ""); + erts_flush_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -6307,7 +6307,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th erts_atomic32_init_relb(&erts_halt_progress, -1); - erts_halt_code = 0; + erts_halt_code = INT_MIN; } @@ -8665,6 +8665,7 @@ sched_thread_func(void *vesdp) erts_ets_sched_spec_data_init(esdp); erts_utils_sched_spec_data_init(); + erts_nif_sched_init(esdp); process_main(esdp); @@ -8715,6 +8716,8 @@ sched_dirty_cpu_thread_func(void *vesdp) erts_proc_lock_prepare_proc_lock_waiter(); + erts_nif_sched_init(esdp); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, @@ -8763,6 +8766,8 @@ sched_dirty_io_thread_func(void *vesdp) erts_proc_lock_prepare_proc_lock_waiter(); + erts_nif_sched_init(esdp); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, @@ -14829,6 +14834,7 @@ void erts_halt(int code) ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); erts_halt_code = code; + erts_nif_notify_halt(); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 949727d950..3f1f937052 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -728,7 +728,10 @@ struct ErtsSchedulerData_ { ErtsSchedWallTime sched_wall_time; ErtsGCInfo gc_info; ErtsPortTaskHandle nosuspend_port_task_handle; - ErtsEtsTables ets_tables; + union { + ErtsEtsTables ets_tables; + erts_atomic32_t dirty_nif_halt_info; + } u; #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; Allctr_t *verify_unused_temp_alloc_data; diff --git a/erts/emulator/beam/erl_term_hashing.c b/erts/emulator/beam/erl_term_hashing.c new file mode 100644 index 0000000000..0a894d4d2b --- /dev/null +++ b/erts/emulator/beam/erl_term_hashing.c @@ -0,0 +1,1793 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022-2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" +#include "erl_term_hashing.h" + +#include "big.h" +#include "bif.h" +#include "erl_map.h" +#include "erl_binary.h" +#include "erl_bits.h" + +#ifdef ERL_INTERNAL_HASH_CRC32C +# if defined(__x86_64__) +# include <immintrin.h> +# elif defined(__aarch64__) +# include <arm_acle.h> +# endif +#endif + +/* *\ + * * +\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* make a hash index from an erlang term */ + +/* +** There are two hash functions. +** +** make_hash: A hash function that will give the same values for the same +** terms regardless of the internal representation. Small integers are +** hashed using the same algorithm as bignums and bignums are hashed +** independent of the CPU endianess. +** Make_hash also hashes pids, ports and references like 32 bit numbers +** (but with different constants). +** make_hash() is called from the bif erlang:phash/2 +** +** The idea behind the hash algorithm is to produce values suitable for +** linear dynamic hashing. We cannot choose the range at all while hashing +** (it's not even supplied to the hashing functions). The good old algorithm +** [H = H*C+X mod M, where H is the hash value, C is a "random" constant(or M), +** M is the range, preferably a prime, and X is each byte value] is therefore +** modified to: +** H = H*C+X mod 2^32, where C is a large prime. This gives acceptable +** "spreading" of the hashes, so that later modulo calculations also will give +** acceptable "spreading" in the range. +** We really need to hash on bytes, otherwise the +** upper bytes of a word will be less significant than the lower ones. That's +** not acceptable at all. For internal use one could maybe optimize by using +** another hash function, that is less strict but faster. That is, however, not +** implemented. +** +** Short semi-formal description of make_hash: +** +** In make_hash, the number N is treated like this: +** Abs(N) is hashed bytewise with the least significant byte, B(0), first. +** The number of bytes (J) to calculate hash on in N is +** (the number of _32_ bit words needed to store the unsigned +** value of abs(N)) * 4. +** X = FUNNY_NUMBER2 +** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3. +** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like +** h(0) = <initial hash> +** h(i) = h(i-1)*X + B(i-1) +** The above should hold regardless of internal representation. +** Pids are hashed like small numbers but with different constants, as are +** ports. +** References are hashed like ports but only on the least significant byte. +** Binaries are hashed on all bytes (not on the 15 first as in +** make_broken_hash()). +** Bytes in lists (possibly text strings) use a simpler multiplication inlined +** in the handling of lists, that is an optimization. +** Everything else is like in the old hash (make_broken_hash()). +** +** make_hash2() is faster than make_hash, in particular for bignums +** and binaries, and produces better hash values. +*/ + +/* some prime numbers just above 2 ^ 28 */ + +#define FUNNY_NUMBER1 268440163 +#define FUNNY_NUMBER2 268439161 +#define FUNNY_NUMBER3 268435459 +#define FUNNY_NUMBER4 268436141 +#define FUNNY_NUMBER5 268438633 +#define FUNNY_NUMBER6 268437017 +#define FUNNY_NUMBER7 268438039 +#define FUNNY_NUMBER8 268437511 +#define FUNNY_NUMBER9 268439627 +#define FUNNY_NUMBER10 268440479 +#define FUNNY_NUMBER11 268440577 +#define FUNNY_NUMBER12 268440581 +#define FUNNY_NUMBER13 268440593 +#define FUNNY_NUMBER14 268440611 + +static Uint32 +hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) +{ + byte* ptr; + Uint bitoffs; + Uint bitsize; + + ERTS_GET_BINARY_BYTES(bin, ptr, bitoffs, bitsize); + if (bitoffs == 0) { + while (sz--) { + hash = hash*FUNNY_NUMBER1 + *ptr++; + } + if (bitsize > 0) { + byte b = *ptr; + + b >>= 8 - bitsize; + hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; + } + } else { + Uint previous = *ptr++; + Uint b; + Uint lshift = bitoffs; + Uint rshift = 8 - lshift; + + while (sz--) { + b = (previous << lshift) & 0xFF; + previous = *ptr++; + b |= previous >> rshift; + hash = hash*FUNNY_NUMBER1 + b; + } + if (bitsize > 0) { + b = (previous << lshift) & 0xFF; + previous = *ptr++; + b |= previous >> rshift; + + b >>= 8 - bitsize; + hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; + } + } + return hash; +} + +Uint32 make_hash(Eterm term_arg) +{ + DECLARE_WSTACK(stack); + Eterm term = term_arg; + Eterm hash = 0; + unsigned op; + +#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF) +#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1) +#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2) +#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3) + + /* + ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit + ** integer. + ** If the endianess is known, we could be smarter here, + ** but that gives no significant speedup (on a sparc at least) + */ +#define UINT32_HASH_STEP(Expr, Prime1) \ + do { \ + Uint32 x = (Uint32) (Expr); \ + hash = \ + (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ + ((x >> 8) & 0xFF)) * (Prime1) + \ + ((x >> 16) & 0xFF)) * (Prime1) + \ + (x >> 24)); \ + } while(0) + +#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ + UINT32_HASH_STEP(Expr, Prime1); \ + hash = hash * (Prime2); \ + break + + + /* + * Significant additions needed for real 64 bit port with larger fixnums. + */ + + /* + * Note, for the simple 64bit port, not utilizing the + * larger word size this function will work without modification. + */ +tail_recur: + op = tag_val_def(term); + + for (;;) { + switch (op) { + case NIL_DEF: + hash = hash*FUNNY_NUMBER3 + 1; + break; + case ATOM_DEF: + hash = hash*FUNNY_NUMBER1 + + (atom_tab(atom_val(term))->slot.bucket.hvalue); + break; + case SMALL_DEF: + { + Sint y1 = signed_val(term); + Uint y2 = y1 < 0 ? -(Uint)y1 : y1; + + UINT32_HASH_STEP(y2, FUNNY_NUMBER2); +#if defined(ARCH_64) + if (y2 >> 32) + UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); +#endif + hash *= (y1 < 0 ? FUNNY_NUMBER4 : FUNNY_NUMBER3); + break; + } + case BINARY_DEF: + { + Uint sz = binary_size(term); + + hash = hash_binary_bytes(term, sz, hash); + hash = hash*FUNNY_NUMBER4 + sz; + break; + } + case FUN_DEF: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + + if (is_local_fun(funp)) { + + ErlFunEntry* fe = funp->entry.fun; + Uint num_free = funp->num_free; + + hash = hash * FUNNY_NUMBER10 + num_free; + hash = hash*FUNNY_NUMBER1 + + (atom_tab(atom_val(fe->module))->slot.bucket.hvalue); + hash = hash*FUNNY_NUMBER2 + fe->index; + hash = hash*FUNNY_NUMBER2 + fe->old_uniq; + + if (num_free > 0) { + if (num_free > 1) { + WSTACK_PUSH3(stack, (UWord) &funp->env[1], + (num_free-1), MAKE_HASH_TERM_ARRAY_OP); + } + + term = funp->env[0]; + goto tail_recur; + } + } else { + const ErtsCodeMFA *mfa = &funp->entry.exp->info.mfa; + + ASSERT(is_external_fun(funp) && funp->next == NULL); + + hash = hash * FUNNY_NUMBER11 + mfa->arity; + hash = hash*FUNNY_NUMBER1 + + (atom_tab(atom_val(mfa->module))->slot.bucket.hvalue); + hash = hash*FUNNY_NUMBER1 + + (atom_tab(atom_val(mfa->function))->slot.bucket.hvalue); + } + break; + } + case PID_DEF: + /* only 15 bits... */ + UINT32_HASH_RET(internal_pid_number(term),FUNNY_NUMBER5,FUNNY_NUMBER6); + case EXTERNAL_PID_DEF: + /* only 15 bits... */ + UINT32_HASH_RET(external_pid_number(term),FUNNY_NUMBER5,FUNNY_NUMBER6); + case PORT_DEF: + case EXTERNAL_PORT_DEF: { + Uint64 number = port_number(term); + Uint32 low = (Uint32) (number & 0xffffffff); + Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); + if (high) + UINT32_HASH_STEP(high, FUNNY_NUMBER11); + UINT32_HASH_RET(low,FUNNY_NUMBER9,FUNNY_NUMBER10); + } + case REF_DEF: + UINT32_HASH_RET(internal_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); + case EXTERNAL_REF_DEF: + UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); + case FLOAT_DEF: + { + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + break; + } + case MAKE_HASH_CDR_PRE_OP: + term = (Eterm) WSTACK_POP(stack); + if (is_not_list(term)) { + WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); + goto tail_recur; + } + /* fall through */ + case LIST_DEF: + { + Eterm* list = list_val(term); + while(is_byte(*list)) { + /* Optimization for strings. + ** Note that this hash is different from a 'small' hash, + ** as multiplications on a Sparc is so slow. + */ + hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); + + if (is_not_list(CDR(list))) { + WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + term = CDR(list); + goto tail_recur; + } + list = list_val(CDR(list)); + } + WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); + term = CAR(list); + goto tail_recur; + } + case MAKE_HASH_CDR_POST_OP: + hash *= FUNNY_NUMBER8; + break; + + case BIG_DEF: + /* Note that this is the exact same thing as the hashing of smalls.*/ + { + Eterm* ptr = big_val(term); + Uint n = BIG_SIZE(ptr); + Uint k = n-1; + ErtsDigit d; + int is_neg = BIG_SIGN(ptr); + Uint i; + int j; + + for (i = 0; i < k; i++) { + d = BIG_DIGIT(ptr, i); + for(j = 0; j < sizeof(ErtsDigit); ++j) { + hash = (hash*FUNNY_NUMBER2) + (d & 0xff); + d >>= 8; + } + } + d = BIG_DIGIT(ptr, k); + k = sizeof(ErtsDigit); +#if defined(ARCH_64) + if (!(d >> 32)) + k /= 2; +#endif + for(j = 0; j < (int)k; ++j) { + hash = (hash*FUNNY_NUMBER2) + (d & 0xff); + d >>= 8; + } + hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3; + break; + } + case MAP_DEF: + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; + case TUPLE_DEF: + { + Eterm* ptr = tuple_val(term); + Uint arity = arityval(*ptr); + + WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); + op = MAKE_HASH_TUPLE_OP; + }/*fall through*/ + case MAKE_HASH_TUPLE_OP: + case MAKE_HASH_TERM_ARRAY_OP: + { + Uint i = (Uint) WSTACK_POP(stack); + Eterm* ptr = (Eterm*) WSTACK_POP(stack); + if (i != 0) { + term = *ptr; + WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); + goto tail_recur; + } + if (op == MAKE_HASH_TUPLE_OP) { + Uint32 arity = (Uint32) WSTACK_POP(stack); + hash = hash*FUNNY_NUMBER9 + arity; + } + break; + } + + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); + return 0; + } + if (WSTACK_ISEMPTY(stack)) break; + op = WSTACK_POP(stack); + } + DESTROY_WSTACK(stack); + return hash; + +#undef MAKE_HASH_TUPLE_OP +#undef MAKE_HASH_TERM_ARRAY_OP +#undef MAKE_HASH_CDR_PRE_OP +#undef MAKE_HASH_CDR_POST_OP +#undef UINT32_HASH_STEP +#undef UINT32_HASH_RET +} + +/* Hash function suggested by Bob Jenkins. */ +#define MIX(a,b,c) \ + do { \ + a -= b; a -= c; a ^= (c>>13); \ + b -= c; b -= a; b ^= (a<<8); \ + c -= a; c -= b; c ^= (b>>13); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<16); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>3); \ + b -= c; b -= a; b ^= (a<<10); \ + c -= a; c -= b; c ^= (b>>15); \ + } while(0) + +#define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */ + +typedef struct { + Uint32 a,b,c; +} ErtsBlockHashHelperCtx; + +#define BLOCK_HASH_BYTES_PER_ITER 12 + +/* The three functions below are separated into different functions even + though they are always used together to make trapping and handling + of unaligned binaries easier. Examples of how they are used can be + found in block_hash and make_hash2_helper.*/ +static ERTS_INLINE +void block_hash_setup(Uint32 initval, + ErtsBlockHashHelperCtx* ctx /* out parameter */) +{ + ctx->a = ctx->b = HCONST; + ctx->c = initval; /* the previous hash value */ +} + +static ERTS_INLINE +void block_hash_buffer(byte *buf, + Uint buf_length, + ErtsBlockHashHelperCtx* ctx /* out parameter */) +{ + Uint len = buf_length; + byte *k = buf; + ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0); + while (len >= BLOCK_HASH_BYTES_PER_ITER) { + ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); + ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); + ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); + MIX(ctx->a,ctx->b,ctx->c); + k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER; + } +} + +static ERTS_INLINE +Uint32 block_hash_final_bytes(byte *buf, + Uint buf_length, + Uint full_length, + ErtsBlockHashHelperCtx* ctx) +{ + Uint len = buf_length; + byte *k = buf; + ctx->c += full_length; + switch(len) + { /* all the case statements fall through */ + case 11: ctx->c+=((Uint32)k[10]<<24); + case 10: ctx->c+=((Uint32)k[9]<<16); + case 9 : ctx->c+=((Uint32)k[8]<<8); + /* the first byte of c is reserved for the length */ + case 8 : ctx->b+=((Uint32)k[7]<<24); + case 7 : ctx->b+=((Uint32)k[6]<<16); + case 6 : ctx->b+=((Uint32)k[5]<<8); + case 5 : ctx->b+=k[4]; + case 4 : ctx->a+=((Uint32)k[3]<<24); + case 3 : ctx->a+=((Uint32)k[2]<<16); + case 2 : ctx->a+=((Uint32)k[1]<<8); + case 1 : ctx->a+=k[0]; + /* case 0: nothing left to add */ + } + MIX(ctx->a,ctx->b,ctx->c); + return ctx->c; +} + +static +Uint32 +block_hash(byte *block, Uint block_length, Uint32 initval) +{ + ErtsBlockHashHelperCtx ctx; + Uint no_bytes_not_in_loop = + (block_length % BLOCK_HASH_BYTES_PER_ITER); + Uint no_bytes_to_process_in_loop = + block_length - no_bytes_not_in_loop; + byte *final_bytes = block + no_bytes_to_process_in_loop; + block_hash_setup(initval, &ctx); + block_hash_buffer(block, + no_bytes_to_process_in_loop, + &ctx); + return block_hash_final_bytes(final_bytes, + no_bytes_not_in_loop, + block_length, + &ctx); +} + +typedef enum { + tag_primary_list, + arityval_subtag, + hamt_subtag_head_flatmap, + map_subtag, + fun_subtag, + neg_big_subtag, + sub_binary_subtag_1, + sub_binary_subtag_2, + hash2_common_1, + hash2_common_2, + hash2_common_3, +} ErtsMakeHash2TrapLocation; + +typedef struct { + int c; + Uint32 sh; + Eterm* ptr; +} ErtsMakeHash2Context_TAG_PRIMARY_LIST; + +typedef struct { + int i; + int arity; + Eterm* elem; +} ErtsMakeHash2Context_ARITYVAL_SUBTAG; + +typedef struct { + Eterm *ks; + Eterm *vs; + int i; + Uint size; +} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP; + +typedef struct { + Eterm* ptr; + int i; +} ErtsMakeHash2Context_MAP_SUBTAG; + +typedef struct { + Uint num_free; + Eterm* bptr; +} ErtsMakeHash2Context_FUN_SUBTAG; + +typedef struct { + Eterm* ptr; + Uint i; + Uint n; + Uint32 con; +} ErtsMakeHash2Context_NEG_BIG_SUBTAG; + +typedef struct { + byte* bptr; + Uint sz; + Uint bitsize; + Uint bitoffs; + Uint no_bytes_processed; + ErtsBlockHashHelperCtx block_hash_ctx; + /* The following fields are only used when bitoffs != 0 */ + byte* buf; + int done; + +} ErtsMakeHash2Context_SUB_BINARY_SUBTAG; + +typedef struct { + int dummy__; /* Empty structs are not supported on all platforms */ +} ErtsMakeHash2Context_EMPTY; + +typedef struct { + ErtsMakeHash2TrapLocation trap_location; + /* specific to the trap location: */ + union { + ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list; + ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag; + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap; + ErtsMakeHash2Context_MAP_SUBTAG map_subtag; + ErtsMakeHash2Context_FUN_SUBTAG fun_subtag; + ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2; + ErtsMakeHash2Context_EMPTY hash2_common_1; + ErtsMakeHash2Context_EMPTY hash2_common_2; + ErtsMakeHash2Context_EMPTY hash2_common_3; + } trap_location_state; + /* same for all trap locations: */ + Eterm term; + Uint32 hash; + Uint32 hash_xor_pairs; + ErtsEStack stack; +} ErtsMakeHash2Context; + +static int make_hash2_ctx_bin_dtor(Binary *context_bin) { + ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin); + DESTROY_SAVED_ESTACK(&context->stack); + if (context->trap_location == sub_binary_subtag_2 && + context->trap_location_state.sub_binary_subtag_2.buf != NULL) { + erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf); + } + return 1; +} + +/* hash2_save_trap_state is called seldom so we want to avoid inlining */ +static ERTS_NOINLINE +Eterm hash2_save_trap_state(Eterm state_mref, + Uint32 hash_xor_pairs, + Uint32 hash, + Process* p, + Eterm term, + Eterm* ESTK_DEF_STACK(s), + ErtsEStack s, + ErtsMakeHash2TrapLocation trap_location, + void* trap_location_state_ptr, + size_t trap_location_state_size) { + Binary* state_bin; + ErtsMakeHash2Context* context; + if (state_mref == THE_NON_VALUE) { + Eterm* hp; + state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context), + make_hash2_ctx_bin_dtor); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin); + } else { + state_bin = erts_magic_ref2bin(state_mref); + } + context = ERTS_MAGIC_BIN_DATA(state_bin); + context->term = term; + context->hash = hash; + context->hash_xor_pairs = hash_xor_pairs; + ESTACK_SAVE(s, &context->stack); + context->trap_location = trap_location; + sys_memcpy(&context->trap_location_state, + trap_location_state_ptr, + trap_location_state_size); + erts_set_gc_state(p, 0); + BUMP_ALL_REDS(p); + return state_mref; +} +#undef NOINLINE_HASH2_SAVE_TRAP_STATE + +/* Writes back a magic reference to *state_mref_write_back when the + function traps */ +static ERTS_INLINE Uint32 +make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p) +{ + static const Uint ITERATIONS_PER_RED = 64; + Uint32 hash; + Uint32 hash_xor_pairs; + Eterm term = term_param; + ERTS_UNDEF(hash_xor_pairs, 0); + +/* (HCONST * {2, ..., 22}) mod 2^32 */ +#define HCONST_2 0x3c6ef372UL +#define HCONST_3 0xdaa66d2bUL +#define HCONST_4 0x78dde6e4UL +#define HCONST_5 0x1715609dUL +#define HCONST_6 0xb54cda56UL +#define HCONST_7 0x5384540fUL +#define HCONST_8 0xf1bbcdc8UL +#define HCONST_9 0x8ff34781UL +#define HCONST_10 0x2e2ac13aUL +#define HCONST_11 0xcc623af3UL +#define HCONST_12 0x6a99b4acUL +#define HCONST_13 0x08d12e65UL +#define HCONST_14 0xa708a81eUL +#define HCONST_15 0x454021d7UL +#define HCONST_16 0xe3779b90UL +#define HCONST_17 0x81af1549UL +#define HCONST_18 0x1fe68f02UL +#define HCONST_19 0xbe1e08bbUL +#define HCONST_20 0x5c558274UL +#define HCONST_21 0xfa8cfc2dUL +#define HCONST_22 0x98c475e6UL + +#define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) +#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) +#define HASH_CDR (_make_header(3,_TAG_HEADER_REF)) + +#define UINT32_HASH_2(Expr1, Expr2, AConst) \ + do { \ + Uint32 a,b; \ + a = AConst + (Uint32) (Expr1); \ + b = AConst + (Uint32) (Expr2); \ + MIX(a,b,hash); \ + } while(0) + +#define UINT32_HASH(Expr, AConst) UINT32_HASH_2(Expr, 0, AConst) + +#define SINT32_HASH(Expr, AConst) \ + do { \ + Sint32 y = (Sint32) (Expr); \ + if (y < 0) { \ + UINT32_HASH(-y, AConst); \ + /* Negative numbers are unnecessarily mixed twice. */ \ + } \ + UINT32_HASH(y, AConst); \ + } while(0) + +#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) + +#define NOT_SSMALL28_HASH(SMALL) \ + do { \ + Uint64 t; \ + Uint32 x, y; \ + Uint32 con; \ + if (SMALL < 0) { \ + con = HCONST_10; \ + t = (Uint64)(SMALL * (-1)); \ + } else { \ + con = HCONST_11; \ + t = SMALL; \ + } \ + x = t & 0xffffffff; \ + y = t >> 32; \ + UINT32_HASH_2(x, y, con); \ + } while(0) + +#ifdef ARCH_64 +# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) +#else +# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) +#endif + +#define TRAP_LOCATION_NO_RED(location_name) \ + do { \ + if(can_trap && iterations_until_trap <= 0) { \ + *state_mref_write_back = \ + hash2_save_trap_state(state_mref, \ + hash_xor_pairs, \ + hash, \ + p, \ + term, \ + ESTK_DEF_STACK(s), \ + s, \ + location_name, \ + &ctx, \ + sizeof(ctx)); \ + return 0; \ + L_##location_name: \ + ctx = context->trap_location_state. location_name; \ + } \ + } while(0) + +#define TRAP_LOCATION(location_name) \ + do { \ + if (can_trap) { \ + iterations_until_trap--; \ + TRAP_LOCATION_NO_RED(location_name); \ + } \ + } while(0) + +#define TRAP_LOCATION_NO_CTX(location_name) \ + do { \ + ErtsMakeHash2Context_EMPTY ctx; \ + TRAP_LOCATION(location_name); \ + } while(0) + + /* Optimization. Simple cases before declaration of estack. */ + if (primary_tag(term) == TAG_PRIMARY_IMMED1) { + switch (term & _TAG_IMMED1_MASK) { + case _TAG_IMMED1_IMMED2: + switch (term & _TAG_IMMED2_MASK) { + case _TAG_IMMED2_ATOM: + /* Fast, but the poor hash value should be mixed. */ + return atom_tab(atom_val(term))->slot.bucket.hvalue; + } + break; + case _TAG_IMMED1_SMALL: + { + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + hash = 0; + NOT_SSMALL28_HASH(small); + return hash; + } + hash = 0; + SINT32_HASH(small, HCONST); + return hash; + } + } + }; + { + Eterm tmp; + long max_iterations = 0; + long iterations_until_trap = 0; + Eterm state_mref = THE_NON_VALUE; + ErtsMakeHash2Context* context = NULL; + DECLARE_ESTACK(s); + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + if(can_trap){ +#ifdef DEBUG + (void)ITERATIONS_PER_RED; + iterations_until_trap = max_iterations = + (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227; +#else + iterations_until_trap = max_iterations = + ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); +#endif + } + if (can_trap && is_internal_magic_ref(term)) { + Binary* state_bin; + state_mref = term; + state_bin = erts_magic_ref2bin(state_mref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) { + /* Restore state after a trap */ + context = ERTS_MAGIC_BIN_DATA(state_bin); + term = context->term; + hash = context->hash; + hash_xor_pairs = context->hash_xor_pairs; + ESTACK_RESTORE(s, &context->stack); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + switch (context->trap_location) { + case hash2_common_3: goto L_hash2_common_3; + case tag_primary_list: goto L_tag_primary_list; + case arityval_subtag: goto L_arityval_subtag; + case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap; + case map_subtag: goto L_map_subtag; + case fun_subtag: goto L_fun_subtag; + case neg_big_subtag: goto L_neg_big_subtag; + case sub_binary_subtag_1: goto L_sub_binary_subtag_1; + case sub_binary_subtag_2: goto L_sub_binary_subtag_2; + case hash2_common_1: goto L_hash2_common_1; + case hash2_common_2: goto L_hash2_common_2; + } + } + } + hash = 0; + for (;;) { + switch (primary_tag(term)) { + case TAG_PRIMARY_LIST: + { + ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = { + .c = 0, + .sh = 0, + .ptr = list_val(term)}; + while (is_byte(*ctx.ptr)) { + /* Optimization for strings. */ + ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr); + if (ctx.c == 3) { + UINT32_HASH(ctx.sh, HCONST_4); + ctx.c = ctx.sh = 0; + } else { + ctx.c++; + } + term = CDR(ctx.ptr); + if (is_not_list(term)) + break; + ctx.ptr = list_val(term); + TRAP_LOCATION(tag_primary_list); + } + if (ctx.c > 0) + UINT32_HASH(ctx.sh, HCONST_4); + if (is_list(term)) { + tmp = CDR(ctx.ptr); + ESTACK_PUSH(s, tmp); + term = CAR(ctx.ptr); + } + } + break; + case TAG_PRIMARY_BOXED: + { + Eterm hdr = *boxed_val(term); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + { + ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = { + .i = 0, + .arity = header_arity(hdr), + .elem = tuple_val(term)}; + UINT32_HASH(ctx.arity, HCONST_9); + if (ctx.arity == 0) /* Empty tuple */ + goto hash2_common; + for (ctx.i = ctx.arity; ; ctx.i--) { + term = ctx.elem[ctx.i]; + if (ctx.i == 1) + break; + ESTACK_PUSH(s, term); + TRAP_LOCATION(arityval_subtag); + } + } + break; + case MAP_SUBTAG: + { + Uint size; + ErtsMakeHash2Context_MAP_SUBTAG ctx = { + .ptr = boxed_val(term) + 1, + .i = 0}; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = { + .ks = flatmap_get_keys(mp), + .vs = flatmap_get_values(mp), + .i = 0, + .size = flatmap_get_size(mp)}; + UINT32_HASH(ctx.size, HCONST_16); + if (ctx.size == 0) + goto hash2_common; + + /* We want a portable hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, ctx.vs[ctx.i]); + ESTACK_PUSH(s, ctx.ks[ctx.i]); + TRAP_LOCATION(hamt_subtag_head_flatmap); + } + goto hash2_common; + } + + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ctx.ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ctx.i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (ctx.i) { + if (is_list(*ctx.ptr)) { + Eterm* cons = list_val(*ctx.ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ctx.ptr)); + ESTACK_PUSH(s, *ctx.ptr); + } + ctx.i--; ctx.ptr++; + TRAP_LOCATION(map_subtag); + } + goto hash2_common; + } + break; + + case FUN_SUBTAG: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + + if (is_local_fun(funp)) { + ErlFunEntry* fe = funp->entry.fun; + ErtsMakeHash2Context_FUN_SUBTAG ctx = { + .num_free = funp->num_free, + .bptr = NULL}; + + UINT32_HASH_2 + (ctx.num_free, + atom_tab(atom_val(fe->module))->slot.bucket.hvalue, + HCONST); + UINT32_HASH_2 + (fe->index, fe->old_uniq, HCONST); + if (ctx.num_free == 0) { + goto hash2_common; + } else { + ctx.bptr = funp->env + ctx.num_free - 1; + while (ctx.num_free-- > 1) { + term = *ctx.bptr--; + ESTACK_PUSH(s, term); + TRAP_LOCATION(fun_subtag); + } + term = *ctx.bptr; + } + } else { + Export *ep = funp->entry.exp; + + ASSERT(is_external_fun(funp) && funp->next == NULL); + + UINT32_HASH_2 + (ep->info.mfa.arity, + atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, + HCONST); + UINT32_HASH + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, + HCONST_14); + + goto hash2_common; + } + } + break; + case REFC_BINARY_SUBTAG: + case HEAP_BINARY_SUBTAG: + case SUB_BINARY_SUBTAG: + { +#define BYTE_BITS 8 + ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = { + .bptr = 0, + /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + * + * The size is truncated to 32 bits on the line + * below so that the code is compatible with old + * versions of the code. This means that hash + * values for binaries with a size greater than + * 4GB do not take all bytes in consideration. + * + * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + */ + .sz = (0xFFFFFFFF & binary_size(term)), + .bitsize = 0, + .bitoffs = 0, + .no_bytes_processed = 0 + }; + Uint32 con = HCONST_13 + hash; + Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER); + ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize); + if (ctx.sz == 0 && ctx.bitsize == 0) { + hash = con; + } else if (ctx.bitoffs == 0 && + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + if (can_trap) iterations_until_trap -= iters_for_bin; + hash = block_hash(ctx.bptr, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (ctx.bitoffs == 0) { + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + block_hash_setup(con, block_hash_ctx); + do { + Uint max_bytes_to_process = + iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER : + iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER; + Uint bytes_left = ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(max_bytes_to_process, even_bytes_left); + block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed], + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_1); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while ((ctx.sz - ctx.no_bytes_processed) >= + BLOCK_HASH_BYTES_PER_ITER); + hash = block_hash_final_bytes(ctx.bptr + + ctx.no_bytes_processed, + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (/* ctx.bitoffs != 0 && */ + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes); + Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize; + if (can_trap) iterations_until_trap -= iters_for_bin; + erts_copy_bits(ctx.bptr, + ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy); + hash = block_hash(buf, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, buf); + } else /* ctx.bitoffs != 0 && */ { +#ifdef DEBUG +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3) +#else +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256) +#endif +#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS) + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0); + ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP, + MIN(nr_of_bytes, BINARY_BUF_SIZE)); + block_hash_setup(con, block_hash_ctx); + do { + Uint bytes_left = + ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(BINARY_BUF_SIZE, even_bytes_left); + Uint nr_of_bits_left = + (ctx.sz*BYTE_BITS+ctx.bitsize) - + ctx.no_bytes_processed*BYTE_BITS; + Uint nr_of_bits_to_copy = + MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS); + ctx.done = nr_of_bits_left == nr_of_bits_to_copy; + erts_copy_bits(ctx.bptr + ctx.no_bytes_processed, + ctx.bitoffs, 1, ctx.buf, 0, 1, + nr_of_bits_to_copy); + block_hash_buffer(ctx.buf, + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_2); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while (!ctx.done); + nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + hash = block_hash_final_bytes(ctx.buf + + (ctx.no_bytes_processed - + ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE), + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + Uint last_byte_index = + nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1; + UINT32_HASH_2(ctx.bitsize, + (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf); + context->trap_location_state.sub_binary_subtag_2.buf = NULL; + } + goto hash2_common; +#undef BYTE_BITS +#undef BINARY_BUF_SIZE +#undef BINARY_BUF_SIZE_BITS + } + break; + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + { + Eterm* big_val_ptr = big_val(term); + ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = { + .ptr = big_val_ptr, + .i = 0, + .n = BIG_SIZE(big_val_ptr), + .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11}; +#if D_EXP == 16 + do { + Uint32 x, y; + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); +#elif D_EXP == 32 + do { + Uint32 x, y; + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); +#elif D_EXP == 64 + do { + Uint t; + Uint32 x, y; + ASSERT(ctx.i < ctx.n); + t = BIG_DIGIT(ctx.ptr, ctx.i++); + x = t & 0xffffffff; + y = t >> 32; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); +#else +#error "unsupported D_EXP size" +#endif + goto hash2_common; + } + break; + case REF_SUBTAG: + /* All parts of the ref should be hashed. */ + UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); + goto hash2_common; + break; + case EXTERNAL_REF_SUBTAG: + /* All parts of the ref should be hashed. */ + UINT32_HASH(external_ref_numbers(term)[0], HCONST_7); + goto hash2_common; + break; + case EXTERNAL_PID_SUBTAG: + /* Only 15 bits are hashed. */ + UINT32_HASH(external_pid_number(term), HCONST_5); + goto hash2_common; + case EXTERNAL_PORT_SUBTAG: { + Uint64 number = external_port_number(term); + Uint32 low = (Uint32) (number & 0xffffffff); + Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); + UINT32_HASH_2(low, high, HCONST_6); + goto hash2_common; + } + case FLOAT_SUBTAG: + { + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) + UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); +#else + UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); +#endif + goto hash2_common; + } + break; + + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + } + } + break; + case TAG_PRIMARY_IMMED1: + switch (term & _TAG_IMMED1_MASK) { + case _TAG_IMMED1_PID: + /* Only 15 bits are hashed. */ + UINT32_HASH(internal_pid_number(term), HCONST_5); + goto hash2_common; + case _TAG_IMMED1_PORT: { + Uint64 number = internal_port_number(term); + Uint32 low = (Uint32) (number & 0xffffffff); + Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); + UINT32_HASH_2(low, high, HCONST_6); + goto hash2_common; + } + case _TAG_IMMED1_IMMED2: + switch (term & _TAG_IMMED2_MASK) { + case _TAG_IMMED2_ATOM: + if (hash == 0) + /* Fast, but the poor hash value should be mixed. */ + hash = atom_tab(atom_val(term))->slot.bucket.hvalue; + else + UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue, + HCONST_3); + goto hash2_common; + case _TAG_IMMED2_NIL: + if (hash == 0) + hash = 3468870702UL; + else + UINT32_HASH(NIL_DEF, HCONST_2); + goto hash2_common; + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + } + case _TAG_IMMED1_SMALL: + { + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + NOT_SSMALL28_HASH(small); + } else { + SINT32_HASH(small, HCONST); + } + + goto hash2_common; + } + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + hash2_common: + + /* Uint32 hash always has the hash value of the previous term, + * compounded or otherwise. + */ + + if (ESTACK_ISEMPTY(s)) { + DESTROY_ESTACK(s); + if (can_trap) { + BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); + ASSERT(!(p->flags & F_DISABLE_GC)); + } + return hash; + } + + term = ESTACK_POP(s); + + switch (term) { + case HASH_MAP_TAIL: { + hash = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); + TRAP_LOCATION_NO_CTX(hash2_common_1); + goto hash2_common; + } + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; + hash = 0; + TRAP_LOCATION_NO_CTX(hash2_common_2); + goto hash2_common; + default: + break; + } + + } + TRAP_LOCATION_NO_CTX(hash2_common_3); + } + } +#undef TRAP_LOCATION_NO_RED +#undef TRAP_LOCATION +#undef TRAP_LOCATION_NO_CTX +} + +Uint32 +make_hash2(Eterm term) +{ + return make_hash2_helper(term, 0, NULL, NULL); +} + +Uint32 +trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p) +{ + return make_hash2_helper(term, 1, state_mref_write_back, p); +} + +/* Term hash function for internal use. + * + * Limitation #1: Is not "portable" in any way between different VM instances. + * + * Limitation #2: The hash value is only valid as long as the term exists + * somewhere in the VM. Why? Because external pids, ports and refs are hashed + * by mixing the node *pointer* value. If a node disappears and later reappears + * with a new ErlNode struct, externals from that node will hash different than + * before. + * + * One IMPORTANT property must hold (for hamt). + * EVERY BIT of the term that is significant for equality (see EQ) + * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a + * chance of hashing different when salted. + * + * This is why we cannot use cached hash values for atoms for example. + * + */ + +/* Use a better mixing function if available. */ +#if defined(ERL_INTERNAL_HASH_CRC32C) +# undef MIX +# if defined(__x86_64__) +# define MIX(a,b,c) \ + do { \ + Uint32 initial_hash = c; \ + c = __builtin_ia32_crc32si(c, a); \ + c = __builtin_ia32_crc32si(c + initial_hash, b); \ + } while(0) +# elif defined(__aarch64__) +# define MIX(a,b,c) \ + do { \ + Uint32 initial_hash = c; \ + c = __crc32cw(c, a); \ + c = __crc32cw(c + initial_hash, b); \ + } while(0) +# else +# error "No suitable CRC32 intrinsic available." +# endif +#endif + +#define CONST_HASH(AConst) \ + do { /* Lightweight mixing of constant (type info) */ \ + hash ^= AConst; \ + hash = (hash << 17) ^ (hash >> (32-17)); \ + } while (0) + +/* + * Start with salt, 32-bit prime number, to avoid getting same hash as phash2 + * which can cause bad hashing in distributed ETS tables for example. + */ +#define INTERNAL_HASH_SALT 3432918353U + +Uint32 +make_internal_hash(Eterm term, Uint32 salt) +{ + Uint32 hash = salt ^ INTERNAL_HASH_SALT; + + /* Optimization. Simple cases before declaration of estack. */ + if (primary_tag(term) == TAG_PRIMARY_IMMED1) { + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #elif ERTS_SIZEOF_ETERM == 4 + UINT32_HASH(term, HCONST); + #else + # error "No you don't" + #endif + return hash; + } + { + Eterm tmp; + DECLARE_ESTACK(s); + + for (;;) { + switch (primary_tag(term)) { + case TAG_PRIMARY_LIST: + { + int c = 0; + Uint32 sh = 0; + Eterm* ptr = list_val(term); + while (is_byte(*ptr)) { + /* Optimization for strings. */ + sh = (sh << 8) + unsigned_val(*ptr); + if (c == 3) { + UINT32_HASH(sh, HCONST_4); + c = sh = 0; + } else { + c++; + } + term = CDR(ptr); + if (is_not_list(term)) + break; + ptr = list_val(term); + } + if (c > 0) + UINT32_HASH_2(sh, (Uint32)c, HCONST_22); + + if (is_list(term)) { + tmp = CDR(ptr); + CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ + ESTACK_PUSH(s, tmp); + if (is_not_list(tmp)) { + ESTACK_PUSH(s, HASH_CDR); + } + term = CAR(ptr); + } + } + break; + case TAG_PRIMARY_BOXED: + { + Eterm hdr = *boxed_val(term); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + { + int i; + int arity = header_arity(hdr); + Eterm* elem = tuple_val(term); + UINT32_HASH(arity, HCONST_9); + if (arity == 0) /* Empty tuple */ + goto pop_next; + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); + } + } + break; + + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + + /* + * We rely on key-value iteration order being constant + * for identical maps (in this VM instance). + */ + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto pop_next; + } + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto pop_next; + } + break; + case FUN_SUBTAG: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + + if (is_local_fun(funp)) { + ErlFunEntry* fe = funp->entry.fun; + Uint num_free = funp->num_free; + UINT32_HASH_2(num_free, fe->module, HCONST_20); + UINT32_HASH_2(fe->index, fe->old_uniq, HCONST_21); + if (num_free == 0) { + goto pop_next; + } else { + Eterm* bptr = funp->env + num_free - 1; + while (num_free-- > 1) { + term = *bptr--; + ESTACK_PUSH(s, term); + } + term = *bptr; + } + } else { + ASSERT(is_external_fun(funp) && funp->next == NULL); + + /* Assumes Export entries never move */ + POINTER_HASH(funp->entry.exp, HCONST_14); + goto pop_next; + } + } + break; + case REFC_BINARY_SUBTAG: + case HEAP_BINARY_SUBTAG: + case SUB_BINARY_SUBTAG: + { + byte* bptr; + Uint sz = binary_size(term); + Uint32 con = HCONST_13 + hash; + Uint bitoffs; + Uint bitsize; + + ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); + if (sz == 0 && bitsize == 0) { + hash = con; + } else { + if (bitoffs == 0) { + hash = block_hash(bptr, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), + HCONST_15); + } + } else { + byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, + sz + (bitsize != 0)); + erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); + hash = block_hash(buf, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, (void *) buf); + } + } + goto pop_next; + } + break; + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + { + Eterm* ptr = big_val(term); + Uint i = 0; + Uint n = BIG_SIZE(ptr); + Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; +#if D_EXP == 16 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 32 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 64 + do { + Uint t; + Uint32 x, y; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); + x = t & 0xffffffff; + y = t >> 32; + UINT32_HASH_2(x, y, con); + } while (i < n); +#else +#error "unsupported D_EXP size" +#endif + goto pop_next; + } + break; + case REF_SUBTAG: { + Uint32 *numbers = internal_ref_numbers(term); + ASSERT(internal_ref_no_numbers(term) >= 3); + UINT32_HASH(numbers[0], HCONST_7); + UINT32_HASH_2(numbers[1], numbers[2], HCONST_8); + if (is_internal_pid_ref(term)) { +#ifdef ARCH_64 + ASSERT(internal_ref_no_numbers(term) == 5); + UINT32_HASH_2(numbers[3], numbers[4], HCONST_9); +#else + ASSERT(internal_ref_no_numbers(term) == 4); + UINT32_HASH(numbers[3], HCONST_9); +#endif + } + goto pop_next; + } + case EXTERNAL_REF_SUBTAG: + { + ExternalThing* thing = external_thing_ptr(term); + Uint n = external_thing_ref_no_numbers(thing); + Uint32 *numbers = external_thing_ref_numbers(thing); + + /* Can contain 0 to 5 32-bit numbers... */ + + /* See limitation #2 */ + switch (n) { + case 5: { + Uint32 num4 = numbers[4]; + if (0) { + case 4: + num4 = 0; + /* Fall through... */ + } + UINT32_HASH_2(numbers[3], num4, HCONST_9); + /* Fall through... */ + } + case 3: { + Uint32 num2 = numbers[2]; + if (0) { + case 2: + num2 = 0; + /* Fall through... */ + } + UINT32_HASH_2(numbers[1], num2, HCONST_8); + /* Fall through... */ + } + case 1: +#ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_7); + UINT32_HASH(numbers[0], HCONST_7); +#else + UINT32_HASH_2(thing->node, numbers[0], HCONST_7); +#endif + break; + case 0: + POINTER_HASH(thing->node, HCONST_7); + break; + default: + ASSERT(!"Invalid amount of external reference numbers"); + break; + } + goto pop_next; + } + case EXTERNAL_PID_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + POINTER_HASH(thing->node, HCONST_5); + UINT32_HASH_2(thing->data.pid.num, thing->data.pid.ser, HCONST_5); + goto pop_next; + } + case EXTERNAL_PORT_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + POINTER_HASH(thing->node, HCONST_6); + UINT32_HASH_2(thing->data.ui32[0], thing->data.ui32[1], HCONST_6); + goto pop_next; + } + case FLOAT_SUBTAG: + { + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); + goto pop_next; + } + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); + } + } + break; + case TAG_PRIMARY_IMMED1: + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #else + UINT32_HASH(term, HCONST); + #endif + goto pop_next; + + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); + + pop_next: + if (ESTACK_ISEMPTY(s)) { + DESTROY_ESTACK(s); + + return hash; + } + + term = ESTACK_POP(s); + + switch (term) { + case HASH_CDR: + CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ + goto pop_next; + default: + break; + } + } + } + } + +} + +/* Term hash function for maps, with a separate depth parameter */ +Uint32 make_map_hash(Eterm key, int depth) { + Uint32 hash = 0; + + if (depth > 0) { + UINT32_HASH_2(depth, 1, HCONST_22); + } + + return make_internal_hash(key, hash); +} + +#undef CONST_HASH +#undef HASH_MAP_TAIL +#undef HASH_MAP_PAIR +#undef HASH_CDR + +#undef UINT32_HASH_2 +#undef UINT32_HASH +#undef SINT32_HASH + +#undef HCONST +#undef MIX diff --git a/erts/emulator/beam/erl_term_hashing.h b/erts/emulator/beam/erl_term_hashing.h new file mode 100644 index 0000000000..79c28d52bd --- /dev/null +++ b/erts/emulator/beam/erl_term_hashing.h @@ -0,0 +1,37 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022-2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_TERM_HASHING_H__ +#define ERL_TERM_HASHING_H__ + +#include "sys.h" + +#if (defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)) || \ + (defined(__x86_64__) && defined(__SSE4_2__)) +# define ERL_INTERNAL_HASH_CRC32C +#endif + +Uint32 make_hash2(Eterm); +Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*); +Uint32 make_hash(Eterm); +Uint32 make_internal_hash(Eterm, Uint32 salt); +Uint32 make_map_hash(Eterm key, int level); + +#endif diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index e83881f52f..2142e8d757 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -24,6 +24,7 @@ #include "sys.h" #include "atom.h" #include "erl_printf.h" +#include "erl_term_hashing.h" struct process; @@ -63,17 +64,11 @@ erts_current_interval_acqb(erts_interval_t *icp) */ void erts_silence_warn_unused_result(long unused); - int erts_fit_in_bits_int64(Sint64); int erts_fit_in_bits_int32(Sint32); int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); -Uint32 make_hash2(Eterm); -Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*); -Uint32 make_hash(Eterm); -Uint32 make_internal_hash(Eterm, Uint32 salt); -Uint32 make_map_hash(Eterm key, int level); void erts_save_emu_args(int argc, char **argv); Eterm erts_get_emu_args(struct process *c_p); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 95b3aebf48..7da160169b 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4409,7 +4409,7 @@ dec_term_atom_common: } if (ref_words != ERTS_REF_NUMBERS) { int i; - if (ref_words > ERTS_REF_NUMBERS) + if (ref_words > ERTS_MAX_REF_NUMBERS) goto error; /* Not a ref that we created... */ for (i = ref_words; i < ERTS_REF_NUMBERS; i++) ref_num[i] = 0; @@ -4421,12 +4421,15 @@ dec_term_atom_common: } else { /* Check if it is a pid reference... */ - Eterm pid = erts_pid_ref_lookup(ref_num); + Eterm pid = erts_pid_ref_lookup(ref_num, ref_words); if (is_internal_pid(pid)) { write_pid_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2], pid); hp += ERTS_PID_REF_THING_SIZE; } + else if (is_non_value(pid)) { + goto error; /* invalid reference... */ + } else { /* Check if it is a magic reference... */ ErtsMagicBinary *mb = erts_magic_ref_lookup_bin(ref_num); @@ -5916,118 +5919,6 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, return 0; return reds; } - - if ((~dflags & DFLAG_UNLINK_ID) - && ep[0] == SMALL_TUPLE_EXT - && ep[1] == 4 - && ep[2] == SMALL_INTEGER_EXT - && (ep[3] == DOP_UNLINK_ID_ACK || ep[3] == DOP_UNLINK_ID)) { - - if (ep[3] == DOP_UNLINK_ID_ACK) { - /* Drop DOP_UNLINK_ID_ACK signal... */ - int i; - for (i = 1; i < ob->eiov->vsize; i++) { - if (ob->eiov->binv[i]) - driver_free_binary(ob->eiov->binv[i]); - } - ob->eiov->vsize = 1; - ob->eiov->size = 0; - } - else { - Eterm ctl_msg, remote, local, *tp; - ErtsTranscodeDecodeState tds; - Uint64 id; - byte *ptr; - ASSERT(ep[3] == DOP_UNLINK_ID); - /* - * Rewrite the DOP_UNLINK_ID signal into a - * DOP_UNLINK signal and send an unlink ack - * to the local sender. - */ - - /* - * decode control message so we get info - * needed for unlink ack signal to send... - */ - ASSERT(get_int32(hdr + 4) == 0); /* No payload */ - ctl_msg = transcode_decode_ctl_msg(&tds, iov, eiov->vsize); - - ASSERT(is_tuple_arity(ctl_msg, 4)); - - tp = tuple_val(ctl_msg); - ASSERT(tp[1] == make_small(DOP_UNLINK_ID)); - - if (!term_to_Uint64(tp[2], &id)) - ERTS_INTERNAL_ERROR("Invalid encoding of DOP_UNLINK_ID signal"); - - local = tp[3]; - remote = tp[4]; - - ASSERT(is_internal_pid(local)); - ASSERT(is_external_pid(remote)); - - /* - * Rewrite buffer to an unlink signal by removing - * second element and change first element to - * DOP_UNLINK. That is, to: {DOP_UNLINK, local, remote} - */ - - ptr = &ep[4]; - switch (*ptr) { - case SMALL_INTEGER_EXT: - ptr += 1; - break; - case INTEGER_EXT: - ptr += 4; - break; - case SMALL_BIG_EXT: - ptr += 1; - ASSERT(*ptr <= 8); - ptr += *ptr + 1; - break; - default: - ERTS_INTERNAL_ERROR("Invalid encoding of DOP_UNLINK_ID signal"); - break; - } - - ASSERT((ptr - ep) <= 16); - ASSERT((ptr - ep) <= iov[2].iov_len); - - *(ptr--) = DOP_UNLINK; - *(ptr--) = SMALL_INTEGER_EXT; - *(ptr--) = 3; - *ptr = SMALL_TUPLE_EXT; - - iov[2].iov_base = ptr; - iov[2].iov_len -= (ptr - ep); - -#ifdef DEBUG - { - ErtsTranscodeDecodeState dbg_tds; - Eterm new_ctl_msg = transcode_decode_ctl_msg(&dbg_tds, - iov, - eiov->vsize); - ASSERT(is_tuple_arity(new_ctl_msg, 3)); - tp = tuple_val(new_ctl_msg); - ASSERT(tp[1] == make_small(DOP_UNLINK)); - ASSERT(tp[2] == local); - ASSERT(eq(tp[3], remote)); - transcode_decode_state_destroy(&dbg_tds); - } -#endif - - /* Send unlink ack to local sender... */ - erts_proc_sig_send_dist_unlink_ack(dep, dep->connection_id, - remote, local, id); - - transcode_decode_state_destroy(&tds); - - reds -= 5; - } - if (reds < 0) - return 0; - return reds; - } start_r = r = reds*ERTS_TRANSCODE_REDS_FACT; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 8c4dd90966..a6b26bc50b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -127,6 +127,10 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); +extern void erts_nif_sched_init(ErtsSchedulerData *esdp); +extern void erts_nif_execute_on_halt(void); +extern void erts_nif_notify_halt(void); +extern void erts_nif_wait_calls(void); extern int erts_nif_get_funcs(struct erl_module_nif*, struct enif_func_t **funcs); extern Module *erts_nif_get_module(struct erl_module_nif*); @@ -1044,9 +1048,9 @@ double erts_get_positive_zero_float(void); /* config.c */ -__decl_noreturn void __noreturn erts_exit_epilogue(void); +__decl_noreturn void __noreturn erts_exit_epilogue(int flush); __decl_noreturn void __noreturn erts_exit(int n, const char*, ...); -__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_flush_exit(int n, char*, ...); void erl_error(const char*, va_list); /* This controls whether sharing-preserving copy is used by Erlang */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index dfdbe475b3..81c3ca29ae 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3772,7 +3772,7 @@ terminate_port(Port *prt) if ((state & ERTS_PORT_SFLG_HALT) && (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ - erts_flush_async_exit(erts_halt_code, ""); + erts_flush_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(NULL, send_closed_port_id, connected_id, am_closed); diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp index 3d2996b5f2..a84851b07b 100644 --- a/erts/emulator/beam/jit/arm/beam_asm.hpp +++ b/erts/emulator/beam/jit/arm/beam_asm.hpp @@ -204,6 +204,7 @@ protected: /* Number of highest element displacement for L/SDP and L/STR. */ static const size_t MAX_LDP_STP_DISPLACEMENT = 0x3F; static const size_t MAX_LDR_STR_DISPLACEMENT = 0xFFF; + static const size_t MAX_LDUR_STUR_DISPLACEMENT = 0xFF; /* Constants for "alternate flag state" operands, which are distinct from * `arm::CondCode::xyz`. Mainly used in `CCMP` instructions. */ @@ -743,12 +744,27 @@ protected: } } + void subs(arm::Gp to, arm::Gp src, int64_t val) { + if (Support::isUInt12(val)) { + a.subs(to, src, imm(val)); + } else if (Support::isUInt12(-val)) { + a.adds(to, src, imm(-val)); + } else { + mov_imm(SUPER_TMP, val); + a.subs(to, src, SUPER_TMP); + } + } + void cmp(arm::Gp src, int64_t val) { if (Support::isUInt12(val)) { a.cmp(src, imm(val)); } else if (Support::isUInt12(-val)) { a.cmn(src, imm(-val)); + } else if (src.isGpW()) { + mov_imm(SUPER_TMP.w(), val); + a.cmp(src, SUPER_TMP.w()); } else { + ERTS_ASSERT(src.isGpX()); mov_imm(SUPER_TMP, val); a.cmp(src, SUPER_TMP); } @@ -992,6 +1008,115 @@ class BeamModuleAssembler : public BeamAssembler { * fragments as if they were local. */ std::unordered_map<void (*)(), Label> _dispatchTable; + /* Skip unnecessary moves in load_source(), load_sources(), and + * mov_arg(). Don't use these variables directly. */ + size_t last_destination_offset = 0; + arm::Gp last_destination_from1, last_destination_from2; + arm::Mem last_destination_to1, last_destination_to2; + + /* Private helper. */ + void preserve__cache(arm::Gp dst) { + last_destination_offset = a.offset(); + invalidate_cache(dst); + } + + /* Works as the STR instruction, but also updates the cache. */ + void str_cache(arm::Gp src, arm::Mem dst) { + if (a.offset() == last_destination_offset && + dst != last_destination_to1) { + /* Something is already cached in the first slot. Use the + * second slot. */ + a.str(src, dst); + last_destination_offset = a.offset(); + last_destination_to2 = dst; + last_destination_from2 = src; + } else { + /* Nothing cached yet, or the first slot has the same + * memory address as we will store into. Use the first + * slot and invalidate the second slot. */ + a.str(src, dst); + last_destination_offset = a.offset(); + last_destination_to1 = dst; + last_destination_from1 = src; + last_destination_to2 = arm::Mem(); + } + } + + /* Works as the STP instruction, but also updates the cache. */ + void stp_cache(arm::Gp src1, arm::Gp src2, arm::Mem dst) { + safe_stp(src1, src2, dst); + last_destination_offset = a.offset(); + last_destination_to1 = dst; + last_destination_from1 = src1; + last_destination_to2 = + arm::Mem(arm::GpX(dst.baseId()), dst.offset() + 8); + last_destination_from2 = src2; + } + + void invalidate_cache(arm::Gp dst) { + if (dst == last_destination_from1) { + last_destination_to1 = arm::Mem(); + last_destination_from1 = arm::Gp(); + } + if (dst == last_destination_from2) { + last_destination_to2 = arm::Mem(); + last_destination_from2 = arm::Gp(); + } + } + + /* Works like LDR, but looks in the cache first. */ + void ldr_cached(arm::Gp dst, arm::Mem mem) { + if (a.offset() == last_destination_offset) { + arm::Gp cached_reg; + if (mem == last_destination_to1) { + cached_reg = last_destination_from1; + } else if (mem == last_destination_to2) { + cached_reg = last_destination_from2; + } + + if (cached_reg.isValid()) { + /* This memory location is cached. */ + if (cached_reg != dst) { + comment("simplified fetching of BEAM register"); + a.mov(dst, cached_reg); + preserve__cache(dst); + } else { + comment("skipped fetching of BEAM register"); + invalidate_cache(dst); + } + } else { + /* Not cached. Load and preserve the cache. */ + a.ldr(dst, mem); + preserve__cache(dst); + } + } else { + /* The cache is invalid. */ + a.ldr(dst, mem); + } + } + + void mov_preserve_cache(arm::VecD dst, arm::VecD src) { + a.mov(dst, src); + } + + void mov_preserve_cache(arm::Gp dst, arm::Gp src) { + if (a.offset() == last_destination_offset) { + a.mov(dst, src); + preserve__cache(dst); + } else { + a.mov(dst, src); + } + } + + void mov_imm_preserve_cache(arm::Gp dst, UWord value) { + if (a.offset() == last_destination_offset) { + mov_imm(dst, value); + preserve__cache(dst); + } else { + mov_imm(dst, value); + } + } + public: BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, @@ -1057,7 +1182,27 @@ protected: return beam->types.entries[typeIndex].type_union; } - auto getIntRange(const ArgSource &arg) const { + int getExtendedTypeUnion(const ArgSource &arg) const { + if (arg.isLiteral()) { + Eterm literal = + beamfile_get_literal(beam, arg.as<ArgLiteral>().get()); + if (is_binary(literal)) { + return BEAM_TYPE_BITSTRING; + } else if (is_list(literal)) { + return BEAM_TYPE_CONS; + } else if (is_tuple(literal)) { + return BEAM_TYPE_TUPLE; + } else if (is_map(literal)) { + return BEAM_TYPE_MAP; + } else { + return BEAM_TYPE_ANY; + } + } else { + return getTypeUnion(arg); + } + } + + auto getClampedRange(const ArgSource &arg) const { if (arg.isSmall()) { Sint value = arg.as<ArgSmall>().getSigned(); return std::make_pair(value, value); @@ -1067,23 +1212,52 @@ protected: ASSERT(typeIndex < beam->types.count); const auto &entry = beam->types.entries[typeIndex]; - ASSERT(entry.type_union & BEAM_TYPE_INTEGER); - return std::make_pair(entry.min, entry.max); + if (entry.min <= entry.max) { + return std::make_pair(entry.min, entry.max); + } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) { + return std::make_pair(entry.min, MAX_SMALL); + } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) { + return std::make_pair(MIN_SMALL, entry.max); + } else { + return std::make_pair(MIN_SMALL, MAX_SMALL); + } } } + int getSizeUnit(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + + ASSERT(typeIndex < beam->types.count); + return beam->types.entries[typeIndex].size_unit; + } + + bool hasLowerBound(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max); + } + + bool hasUpperBound(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max); + } + bool always_small(const ArgSource &arg) const { if (arg.isSmall()) { return true; } - int type_union = getTypeUnion(arg); - if (type_union == BEAM_TYPE_INTEGER) { - auto [min, max] = getIntRange(arg); - return min <= max; - } else { - return false; - } + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return entry.type_union == BEAM_TYPE_INTEGER && entry.min <= entry.max; } bool always_immediate(const ArgSource &arg) const { @@ -1096,8 +1270,8 @@ protected: } bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const { - int lhs_types = getTypeUnion(lhs); - int rhs_types = getTypeUnion(rhs); + int lhs_types = getExtendedTypeUnion(lhs); + int rhs_types = getExtendedTypeUnion(rhs); /* We can only be certain that the types are the same when there's * one possible type. For example, if one is a number and the other @@ -1146,67 +1320,53 @@ protected: return always_one_of(arg, type_id); } - bool is_sum_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - Sint min, max; - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - min = min1 + min2; - max = max1 + max2; - return IS_SSMALL(min) && IS_SSMALL(max); - } + bool is_sum_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 + min2; + max = max1 + max2; + return IS_SSMALL(min) && IS_SSMALL(max); } - bool is_difference_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - Sint min, max; - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - min = min1 - max2; - max = max1 - min2; - return IS_SSMALL(min) && IS_SSMALL(max); - } + bool is_diff_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 - max2; + max = max1 - min2; + return IS_SSMALL(min) && IS_SSMALL(max); } - bool is_product_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - auto mag1 = std::max(std::abs(min1), std::abs(max1)); - auto mag2 = std::max(std::abs(min2), std::abs(max2)); - - /* - * mag1 * mag2 <= MAX_SMALL - * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) - */ - ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); - return mag2 == 0 || mag1 <= MAX_SMALL / mag2; - } - } + bool is_product_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + auto mag1 = std::max(std::abs(min1), std::abs(max1)); + auto mag2 = std::max(std::abs(min2), std::abs(max2)); - bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) { /* - * In the code compiled by scripts/diffable, there never - * seems to be any range information for the RHS. Therefore, - * don't bother unless RHS is an immediate small. + * mag1 * mag2 <= MAX_SMALL + * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) */ - if (!(always_small(LHS) && RHS.isSmall())) { + ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); + return mag2 == 0 || mag1 <= MAX_SMALL / mag2; + } + + bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) { + if (!(always_small(LHS) && always_small(RHS))) { return false; } else { - auto [min1, max1] = getIntRange(LHS); - auto rhs_val = RHS.as<ArgSmall>().getSigned(); + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); - if (min1 < 0 || max1 == 0 || rhs_val < 0) { + if (min1 < 0 || max1 == 0 || min2 < 0) { return false; } - return rhs_val < Support::clz(max1) - _TAG_IMMED1_SIZE; + return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE; } } @@ -1216,7 +1376,8 @@ protected: const ArgWord &Live); void emit_gc_test_preserve(const ArgWord &Need, const ArgWord &Live, - arm::Gp term); + const ArgSource &Preserve, + arm::Gp preserve_reg); arm::Mem emit_variable_apply(bool includeI); arm::Mem emit_fixed_apply(const ArgWord &arity, bool includeI); @@ -1225,11 +1386,6 @@ protected: bool skip_fun_test = false, bool skip_arity_test = false); - arm::Gp emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src, - Label next, - Label subbin); - void emit_is_boxed(Label Fail, arm::Gp Src) { BeamAssembler::emit_is_boxed(Fail, Src); } @@ -1243,10 +1399,28 @@ protected: BeamAssembler::emit_is_boxed(Fail, Src); } + /* Copies `count` words from the address at `from`, to the address at `to`. + * + * Clobbers v30 and v31. */ + void emit_copy_words_increment(arm::Gp from, arm::Gp to, size_t count); + void emit_get_list(const arm::Gp boxed_ptr, const ArgRegister &Hd, const ArgRegister &Tl); + void emit_add_sub_types(bool is_small_result, + const ArgSource &LHS, + const a64::Gp lhs_reg, + const ArgSource &RHS, + const a64::Gp rhs_reg, + const Label next); + + void emit_are_both_small(const ArgSource &LHS, + const a64::Gp lhs_reg, + const ArgSource &RHS, + const a64::Gp rhs_reg, + const Label next); + void emit_div_rem(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, @@ -1271,6 +1445,30 @@ protected: void emit_bs_get_utf16(const ArgRegister &Ctx, const ArgLabel &Fail, const ArgWord &Flags); + void update_bin_state(arm::Gp bin_offset, + Sint bit_offset, + Sint size, + arm::Gp size_reg); + void set_zero(Sint effectiveSize); + void emit_construct_utf8(const ArgVal &Src, + Sint bit_offset, + bool is_byte_aligned); + + void emit_read_bits(Uint bits, + const arm::Gp bin_offset, + const arm::Gp bin_base, + const arm::Gp bitdata); + + void emit_extract_integer(const arm::Gp bitdata, + Uint flags, + Uint bits, + const ArgRegister &Dst); + + void emit_extract_binary(const arm::Gp bitdata, + Uint bits, + const ArgRegister &Dst); + + UWord bs_get_flags(const ArgVal &val); void emit_raise_exception(); void emit_raise_exception(const ErtsCodeMFA *exp); @@ -1288,11 +1486,14 @@ protected: void emit_validate_unicode(Label next, Label fail, arm::Gp value); - void emit_bif_is_eq_ne_exact_immed(const ArgSource &LHS, - const ArgSource &RHS, - const ArgRegister &Dst, - Eterm fail_value, - Eterm succ_value); + void ubif_comment(const ArgWord &Bif); + + void emit_cmp_immed_to_bool(arm::CondCode cc, + const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst); + + void emit_cond_to_bool(arm::CondCode cc, const ArgRegister &Dst); void emit_proc_lc_unrequire(void); void emit_proc_lc_require(void); @@ -1302,7 +1503,8 @@ protected: /* Returns a vector of the untagged and rebased `args`. The adjusted * `comparand` is stored in ARG1. */ - const std::vector<ArgVal> emit_select_untag(const Span<ArgVal> &args, + const std::vector<ArgVal> emit_select_untag(const ArgSource &Src, + const Span<ArgVal> &args, a64::Gp comparand, Label fail, UWord base, @@ -1457,11 +1659,13 @@ protected: } else if (arg.isRegister()) { if (isRegisterBacked(arg)) { auto index = arg.as<ArgXRegister>().get(); - return Variable(register_backed_xregs[index]); + arm::Gp reg = register_backed_xregs[index]; + invalidate_cache(reg); + return Variable(reg); } auto ref = getArgRef(arg); - a.ldr(tmp, ref); + ldr_cached(tmp, ref); return Variable(tmp, ref); } else { if (arg.isImmed() || arg.isWord()) { @@ -1469,7 +1673,7 @@ protected: : arg.as<ArgWord>().get(); if (Support::isIntOrUInt32(val)) { - mov_imm(tmp, val); + mov_imm_preserve_cache(tmp, val); return Variable(tmp); } } @@ -1483,8 +1687,7 @@ protected: arm::Gp tmp1, const ArgVal &Src2, arm::Gp tmp2) { - if (Src1.isRegister() && Src2.isRegister() && !isRegisterBacked(Src1) && - !isRegisterBacked(Src2)) { + if (!isRegisterBacked(Src1) && !isRegisterBacked(Src2)) { switch (ArgVal::memory_relation(Src1, Src2)) { case ArgVal::Relation::consecutive: safe_ldp(tmp1, tmp2, Src1, Src2); @@ -1520,19 +1723,24 @@ protected: template<typename Reg> void mov_var(const Variable<Reg> &to, Reg from) { if (to.reg != from) { - a.mov(to.reg, from); + mov_preserve_cache(to.reg, from); } } template<typename Reg> void mov_var(Reg to, const Variable<Reg> &from) { if (to != from.reg) { - a.mov(to, from.reg); + mov_preserve_cache(to, from.reg); } } - template<typename Reg> - void flush_var(const Variable<Reg> &to) { + void flush_var(const Variable<arm::Gp> &to) { + if (to.mem.hasBase()) { + str_cache(to.reg, to.mem); + } + } + + void flush_var(const Variable<arm::VecD> &to) { if (to.mem.hasBase()) { a.str(to.reg, to.mem); } @@ -1546,10 +1754,10 @@ protected: if (mem1.hasBaseReg() && mem2.hasBaseReg() && mem1.baseId() == mem2.baseId()) { if (mem1.offset() + 8 == mem2.offset()) { - safe_stp(to1.reg, to2.reg, mem1); + stp_cache(to1.reg, to2.reg, mem1); return; } else if (mem1.offset() == mem2.offset() + 8) { - safe_stp(to2.reg, to1.reg, mem2); + stp_cache(to2.reg, to1.reg, mem2); return; } } @@ -1604,18 +1812,12 @@ protected: if (arg.isImmed() || arg.isWord()) { Sint val = arg.isImmed() ? arg.as<ArgImmed>().get() : arg.as<ArgWord>().get(); - - if (Support::isUInt12(val)) { - a.cmp(gp, imm(val)); - return; - } else if (Support::isUInt12(-val)) { - a.cmn(gp, imm(-val)); - return; - } + cmp(gp, val); + return; } - mov_arg(SUPER_TMP, arg); - a.cmp(gp, SUPER_TMP); + auto tmp = load_source(arg, SUPER_TMP); + a.cmp(gp, tmp.reg); } void safe_stp(arm::Gp gp1, @@ -1660,6 +1862,20 @@ protected: } } + void safe_ldur(arm::Gp gp, arm::Mem mem) { + auto offset = mem.offset(); + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGpX()); + + if (std::abs(offset) <= MAX_LDUR_STUR_DISPLACEMENT) { + a.ldur(gp, mem); + } else { + add(SUPER_TMP, arm::GpX(mem.baseId()), offset); + a.ldr(gp, arm::Mem(SUPER_TMP)); + } + } + void safe_ldp(arm::Gp gp1, arm::Gp gp2, const ArgVal &Src1, diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl index f42d16e853..893830d180 100644 --- a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl +++ b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl @@ -49,6 +49,7 @@ my @beam_global_funcs = qw( call_nif_early call_nif_shared check_float_error + construct_utf8_shared debug_bp dispatch_bif dispatch_nif @@ -56,10 +57,13 @@ my @beam_global_funcs = qw( dispatch_save_calls export_trampoline fconv_shared + get_sint64_shared handle_and_error handle_call_fun_error handle_element_error_shared handle_hd_error + handle_map_get_badkey + handle_map_get_badmap handle_map_size_error handle_not_error handle_or_error @@ -82,12 +86,13 @@ my @beam_global_funcs = qw( i_length_guard_shared i_length_body_shared i_loop_rec_shared - i_new_small_map_lit_shared i_test_yield_shared i_bxor_body_shared int_div_rem_body_shared int_div_rem_guard_shared internal_hash_helper + is_in_range_shared + is_ge_lt_shared minus_body_shared new_map_shared update_map_assoc_shared @@ -97,6 +102,7 @@ my @beam_global_funcs = qw( process_main raise_exception raise_exception_shared + store_unaligned times_body_shared times_guard_shared unary_minus_body_shared diff --git a/erts/emulator/beam/jit/arm/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/beam_asm_module.cpp index 219df1f125..9adc000451 100644 --- a/erts/emulator/beam/jit/arm/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/arm/beam_asm_module.cpp @@ -350,6 +350,8 @@ void BeamModuleAssembler::emit_label(const ArgLabel &Label) { currLabel = rawLabels[Label.get()]; bind_veneer_target(currLabel); + + last_destination_offset = ~0; } void BeamModuleAssembler::emit_aligned_label(const ArgLabel &Label, diff --git a/erts/emulator/beam/jit/arm/generators.tab b/erts/emulator/beam/jit/arm/generators.tab index b6ec48c678..dfda099dca 100644 --- a/erts/emulator/beam/jit/arm/generators.tab +++ b/erts/emulator/beam/jit/arm/generators.tab @@ -19,47 +19,6 @@ // %CopyrightEnd% // -// Generate the fastest instruction to fetch an integer from a binary. -gen.get_integer2(Fail, Ms, Live, Size, Unit, Flags, Dst) { - BeamOp* op; - UWord bits; - - $NewBeamOp(S, op); - $NativeEndian(Flags); - - if (Size.type == TAG_i) { - if (!beam_load_safe_mul(Size.val, Unit.val, &bits)) { - $BeamOpNameArity(op, jump, 1); - op->a[0] = Fail; - - return op; - } else if (bits <= 1023) { - $BeamOpNameArity(op, i_bs_get_fixed_integer, 6); - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Live; - op->a[3].type = TAG_u; - op->a[3].val = Flags.val; - op->a[4].type = TAG_u; - op->a[4].val = bits; - op->a[5] = Dst; - - return op; - } - } - - $BeamOpNameArity(op, i_bs_get_integer, 6); - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Live; - op->a[3].type = TAG_u; - op->a[3].val = (Unit.val << 3) | Flags.val; - op->a[4] = Size; - op->a[5] = Dst; - - return op; -} - gen.select_tuple_arity(Src, Fail, Size, Rest) { BeamOp* op; BeamOpArg *tmp; @@ -567,6 +526,28 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { Flags.val = flags; $NativeEndian(Flags); op->a[i+fixed_args+3] = Flags; + + /* + * Replace short string segments with integer segments. + * Integer segments can be combined with adjacent integer + * segments for better performance. + */ + if (op->a[i+fixed_args+0].val == am_string) { + Sint num_chars = op->a[i+fixed_args+5].val; + if (num_chars <= 4) { + Sint index = op->a[i+fixed_args+4].val; + const byte* s = S->beam.strings.data + index; + Uint num = 0; + op->a[i+fixed_args+0].val = am_integer; + op->a[i+fixed_args+2].val = 8; + op->a[i+fixed_args+5].val = num_chars; + while (num_chars-- > 0) { + num = num << 8 | *s++; + } + op->a[i+fixed_args+4].type = TAG_i; + op->a[i+fixed_args+4].val = num; + } + } } if (op->a[4].val == am_private_append && Alloc.val != 0) { @@ -584,3 +565,33 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { return op; } + +gen.bs_match(Fail, Ctx, N, List) { + BeamOp* op; + int fixed_args; + int i; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_match, 2); + fixed_args = op->arity; + $BeamOpArity(op, (N.val + fixed_args)); + + op->a[0] = Fail; + op->a[1] = Ctx; + + for (i = 0; i < N.val; i++) { + BeamOpArg current; + + current = List[i]; + if (current.type == TAG_o) { + /* An overflow tag (in ensure_at_least or ensure_exactly) + * means that the match will always fail. */ + $BeamOpNameArity(op, jump, 1); + op->a[0] = Fail; + return op; + } + op->a[i+fixed_args] = current; + } + + return op; +} diff --git a/erts/emulator/beam/jit/arm/instr_arith.cpp b/erts/emulator/beam/jit/arm/instr_arith.cpp index 95d1ab0019..12f4eff3b1 100644 --- a/erts/emulator/beam/jit/arm/instr_arith.cpp +++ b/erts/emulator/beam/jit/arm/instr_arith.cpp @@ -26,6 +26,68 @@ extern "C" #include "big.h" } +void BeamModuleAssembler::emit_add_sub_types(bool is_small_result, + const ArgSource &LHS, + const a64::Gp lhs_reg, + const ArgSource &RHS, + const a64::Gp rhs_reg, + const Label next) { + if (is_small_result && + always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + comment("skipped overflow test because the result is always small"); + emit_are_both_small(LHS, lhs_reg, RHS, rhs_reg, next); + } else { + if (always_small(RHS)) { + a.and_(TMP1, lhs_reg, imm(_TAG_IMMED1_MASK)); + } else if (always_small(LHS)) { + a.and_(TMP1, rhs_reg, imm(_TAG_IMMED1_MASK)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, lhs_reg, rhs_reg); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + } + + comment("test for not overflow and small operands"); + a.ccmp(TMP1, + imm(_TAG_IMMED1_SMALL), + imm(NZCV::kNone), + imm(arm::CondCode::kVC)); + a.b_eq(next); + } +} + +void BeamModuleAssembler::emit_are_both_small(const ArgSource &LHS, + const a64::Gp lhs_reg, + const ArgSource &RHS, + const a64::Gp rhs_reg, + const Label next) { + if (always_small(RHS) && + always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operand since other types are " + "boxed"); + emit_is_boxed(next, lhs_reg); + } else if (always_small(LHS) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operand since other types are " + "boxed"); + emit_is_boxed(next, rhs_reg); + } else if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operands since other types are " + "boxed"); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, lhs_reg, rhs_reg); + emit_is_boxed(next, TMP1); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, lhs_reg, rhs_reg); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_eq(next); + } +} + /* * ARG2 = LHS * ARG3 = RHS @@ -73,8 +135,9 @@ void BeamModuleAssembler::emit_i_plus(const ArgLabel &Fail, const ArgRegister &Dst) { bool rhs_is_arm_literal = RHS.isSmall() && Support::isUInt12(RHS.as<ArgSmall>().get()); + bool is_small_result = is_sum_small_if_args_are_small(LHS, RHS); - if (is_sum_small(LHS, RHS)) { + if (always_small(LHS) && always_small(RHS) && is_small_result) { auto dst = init_destination(Dst, ARG1); if (rhs_is_arm_literal) { auto lhs = load_source(LHS, ARG2); @@ -96,31 +159,14 @@ void BeamModuleAssembler::emit_i_plus(const ArgLabel &Fail, auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); if (rhs_is_arm_literal) { - comment("add small constant with overflow check"); Uint cleared_tag = RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_MASK; a.adds(ARG1, lhs.reg, imm(cleared_tag)); } else { - comment("addition with overflow check"); a.and_(TMP1, rhs.reg, imm(~_TAG_IMMED1_MASK)); a.adds(ARG1, lhs.reg, TMP1); } - if (always_small(RHS)) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (always_small(LHS)) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); - } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } - - /* Test for not overflow AND small operands. */ - a.ccmp(TMP1, - imm(_TAG_IMMED1_SMALL), - imm(NZCV::kNone), - imm(arm::CondCode::kVC)); - a.b_eq(next); + emit_add_sub_types(is_small_result, LHS, lhs.reg, RHS, rhs.reg, next); mov_var(ARG2, lhs); mov_var(ARG3, rhs); @@ -186,11 +232,13 @@ void BeamModuleAssembler::emit_i_unary_minus(const ArgLabel &Fail, const ArgSource &Src, const ArgRegister &Dst) { auto src = load_source(Src, ARG2); + auto zero = ArgImmed(make_small(0)); + bool is_small_result = is_diff_small_if_args_are_small(zero, Src); a.mov(TMP1, imm(_TAG_IMMED1_SMALL)); a.and_(TMP2, src.reg, imm(~_TAG_IMMED1_MASK)); - if (is_difference_small(ArgImmed(make_small(0)), Src)) { + if (always_small(Src) && is_small_result) { auto dst = init_destination(Dst, ARG1); comment("no overflow test because result is always small"); a.sub(dst.reg, TMP1, TMP2); @@ -274,8 +322,9 @@ void BeamModuleAssembler::emit_i_minus(const ArgLabel &Fail, const ArgRegister &Dst) { bool rhs_is_arm_literal = RHS.isSmall() && Support::isUInt12(RHS.as<ArgSmall>().get()); + bool is_small_result = is_diff_small_if_args_are_small(LHS, RHS); - if (is_difference_small(LHS, RHS)) { + if (always_small(LHS) && always_small(RHS) && is_small_result) { auto dst = init_destination(Dst, ARG1); if (rhs_is_arm_literal) { auto lhs = load_source(LHS, ARG2); @@ -297,31 +346,14 @@ void BeamModuleAssembler::emit_i_minus(const ArgLabel &Fail, auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); if (rhs_is_arm_literal) { - comment("subtract small constant with overflow check"); Uint cleared_tag = RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_MASK; a.subs(ARG1, lhs.reg, imm(cleared_tag)); } else { - comment("subtraction with overflow check"); a.and_(TMP1, rhs.reg, imm(~_TAG_IMMED1_MASK)); a.subs(ARG1, lhs.reg, TMP1); } - if (always_small(RHS)) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (always_small(LHS)) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); - } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } - - /* Test for not overflow AND small operands. */ - a.ccmp(TMP1, - imm(_TAG_IMMED1_SMALL), - imm(NZCV::kNone), - imm(arm::CondCode::kVC)); - a.b_eq(next); + emit_add_sub_types(is_small_result, LHS, lhs.reg, RHS, rhs.reg, next); mov_var(ARG2, lhs); mov_var(ARG3, rhs); @@ -453,20 +485,32 @@ void BeamModuleAssembler::emit_i_times(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - if (is_product_small(LHS, RHS)) { + bool is_small_result = is_product_small_if_args_are_small(LHS, RHS); + + if (always_small(LHS) && always_small(RHS) && is_small_result) { auto dst = init_destination(Dst, ARG1); comment("multiplication without overflow check"); if (RHS.isSmall()) { auto lhs = load_source(LHS, ARG2); + Sint factor = RHS.as<ArgSmall>().getSigned(); + a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK)); - mov_imm(TMP2, RHS.as<ArgSmall>().getSigned()); + if (Support::isPowerOf2(factor)) { + int trailing_bits = Support::ctz<Eterm>(factor); + comment("optimized multiplication by replacing with left " + "shift"); + a.lsl(TMP1, TMP1, imm(trailing_bits)); + } else { + mov_imm(TMP2, factor); + a.mul(TMP1, TMP1, TMP2); + } } else { auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK)); a.asr(TMP2, rhs.reg, imm(_TAG_IMMED1_SIZE)); + a.mul(TMP1, TMP1, TMP2); } - a.mul(dst.reg, TMP1, TMP2); - a.orr(dst.reg, dst.reg, imm(_TAG_IMMED1_SMALL)); + a.orr(dst.reg, TMP1, imm(_TAG_IMMED1_SMALL)); flush_var(dst); } else { auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); @@ -647,21 +691,39 @@ void BeamModuleAssembler::emit_div_rem(const ArgLabel &Fail, auto remainder = init_destination(Remainder, ARG2); comment("skipped test for smalls operands and overflow"); - a.asr(TMP1, lhs.reg, imm(_TAG_IMMED1_SIZE)); - mov_imm(TMP2, divisor); - a.sdiv(quotient.reg, TMP1, TMP2); - if (need_rem) { - a.msub(remainder.reg, quotient.reg, TMP2, TMP1); - } - mov_imm(TMP3, _TAG_IMMED1_SMALL); - arm::Shift tagShift = arm::lsl(_TAG_IMMED1_SIZE); + if (Support::isPowerOf2(divisor) && + std::get<0>(getClampedRange(LHS)) >= 0) { + int trailing_bits = Support::ctz<Eterm>(divisor); + if (need_div) { + comment("optimized div by replacing with right shift"); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.lsr(quotient.reg, lhs.reg, imm(trailing_bits)); + a.orr(quotient.reg, quotient.reg, imm(_TAG_IMMED1_SMALL)); + } + if (need_rem) { + comment("optimized rem by replacing with masking"); + auto mask = Support::lsbMask<Uint>(trailing_bits + + _TAG_IMMED1_SIZE); + a.and_(remainder.reg, lhs.reg, imm(mask)); + } + } else { + a.asr(TMP1, lhs.reg, imm(_TAG_IMMED1_SIZE)); + mov_imm(TMP2, divisor); + a.sdiv(quotient.reg, TMP1, TMP2); + if (need_rem) { + a.msub(remainder.reg, quotient.reg, TMP2, TMP1); + } - if (need_div) { - a.orr(quotient.reg, TMP3, quotient.reg, tagShift); - } - if (need_rem) { - a.orr(remainder.reg, TMP3, remainder.reg, tagShift); + mov_imm(TMP3, _TAG_IMMED1_SMALL); + const arm::Shift tagShift = arm::lsl(_TAG_IMMED1_SIZE); + if (need_div) { + a.orr(quotient.reg, TMP3, quotient.reg, tagShift); + } + if (need_rem) { + a.orr(remainder.reg, TMP3, remainder.reg, tagShift); + } } + if (need_div) { flush_var(quotient); } @@ -825,24 +887,52 @@ void BeamModuleAssembler::emit_i_band(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { + if (always_small(LHS) && RHS.isSmall()) { + a64::Utils::LogicalImm ignore; + if (a64::Utils::encodeLogicalImm(RHS.as<ArgSmall>().get(), + 64, + &ignore)) { + comment("skipped test for small operands since they are always " + "small"); + auto lhs = load_source(LHS, ARG2); + auto dst = init_destination(Dst, ARG1); + + /* TAG & TAG = TAG, so we don't need to tag it again. */ + a.and_(dst.reg, lhs.reg, RHS.as<ArgSmall>().get()); + flush_var(dst); + return; + } + } + auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); auto dst = init_destination(Dst, ARG1); - /* TAG & TAG = TAG, so we don't need to tag it again. */ - a.and_(ARG1, lhs.reg, rhs.reg); - if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); + + /* TAG & TAG = TAG, so we don't need to tag it again. */ + a.and_(dst.reg, lhs.reg, rhs.reg); + flush_var(dst); } else { Label next = a.newLabel(); - /* All other term types has at least one zero in the low 4 - * bits. Therefore, the result will be a small iff both operands - * are small. */ + /* TAG & TAG = TAG, so we don't need to tag it again. */ + a.and_(ARG1, lhs.reg, rhs.reg); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); - a.b_eq(next); + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operands since other types are " + "boxed"); + emit_is_boxed(next, ARG1); + } else { + /* All other term types has at least one zero in the low 4 + * bits. Therefore, the result will be a small iff both + * operands are small. */ + a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_eq(next); + } mov_var(ARG2, lhs); mov_var(ARG3, rhs); @@ -861,10 +951,9 @@ void BeamModuleAssembler::emit_i_band(const ArgLabel &Fail, } a.bind(next); + mov_var(dst, ARG1); + flush_var(dst); } - - mov_var(dst, ARG1); - flush_var(dst); } /* @@ -886,53 +975,58 @@ void BeamModuleAssembler::emit_i_bor(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { + if (always_small(LHS) && RHS.isSmall()) { + a64::Utils::LogicalImm ignore; + Uint64 rhs = RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_SMALL; + if (a64::Utils::encodeLogicalImm(rhs, 64, &ignore)) { + comment("skipped test for small operands since they are always " + "small"); + auto lhs = load_source(LHS, ARG2); + auto dst = init_destination(Dst, ARG1); + + a.orr(dst.reg, lhs.reg, rhs); + flush_var(dst); + return; + } + } + auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); auto dst = init_destination(Dst, ARG1); - /* TAG | TAG = TAG, so we don't need to tag it again. */ - a.orr(ARG1, lhs.reg, rhs.reg); - if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); + + /* TAG | TAG = TAG, so we don't need to tag it again. */ + a.orr(dst.reg, lhs.reg, rhs.reg); + flush_var(dst); } else { - Label generic = a.newLabel(), next = a.newLabel(); - if (always_small(RHS)) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (always_small(LHS)) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); - } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } + Label next = a.newLabel(); - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); - a.b_eq(next); + /* TAG | TAG = TAG, so we don't need to tag it again. */ + a.orr(ARG1, lhs.reg, rhs.reg); - a.bind(generic); - { - mov_var(ARG2, lhs); - mov_var(ARG3, rhs); - - if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); - a.mov(ARG1, c_p); - runtime_call<3>(erts_bor); - emit_leave_runtime(Live.get()); - emit_branch_if_not_value(ARG1, - resolve_beam_label(Fail, dispUnknown)); - } else { - emit_enter_runtime(Live.get()); - fragment_call(ga->get_i_bor_body_shared()); - emit_leave_runtime(Live.get()); - } + emit_are_both_small(LHS, lhs.reg, RHS, rhs.reg, next); + + mov_var(ARG2, lhs); + mov_var(ARG3, rhs); + + if (Fail.get() != 0) { + emit_enter_runtime(Live.get()); + a.mov(ARG1, c_p); + runtime_call<3>(erts_bor); + emit_leave_runtime(Live.get()); + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } else { + emit_enter_runtime(Live.get()); + fragment_call(ga->get_i_bor_body_shared()); + emit_leave_runtime(Live.get()); } a.bind(next); + mov_var(dst, ARG1); + flush_var(dst); } - - mov_var(dst, ARG1); - flush_var(dst); } /* @@ -955,9 +1049,9 @@ void BeamModuleAssembler::emit_i_bxor(const ArgLabel &Fail, const ArgSource &RHS, const ArgRegister &Dst) { auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); + auto dst = init_destination(Dst, ARG1); if (always_small(LHS) && always_small(RHS)) { - auto dst = init_destination(Dst, ARG1); comment("skipped test for small operands because they are always " "small"); @@ -969,24 +1063,12 @@ void BeamModuleAssembler::emit_i_bxor(const ArgLabel &Fail, } Label next = a.newLabel(); - auto dst = init_destination(Dst, ARG1); /* TAG ^ TAG = 0, so we'll need to tag it again. */ a.eor(ARG1, lhs.reg, rhs.reg); a.orr(ARG1, ARG1, imm(_TAG_IMMED1_SMALL)); - if (always_small(RHS)) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (always_small(LHS)) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); - } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } - - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); - a.b_eq(next); + emit_are_both_small(LHS, lhs.reg, RHS, rhs.reg, next); mov_var(ARG2, lhs); mov_var(ARG3, rhs); @@ -1221,18 +1303,26 @@ void BeamModuleAssembler::emit_i_bsl(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); auto dst = init_destination(Dst, ARG1); if (is_bsl_small(LHS, RHS)) { comment("skipped tests because operands and result are always small"); - a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK)); - a.lsl(TMP1, TMP1, imm(RHS.as<ArgSmall>().getSigned())); + if (RHS.isSmall()) { + auto lhs = load_source(LHS, ARG2); + a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK)); + a.lsl(TMP1, TMP1, imm(RHS.as<ArgSmall>().getSigned())); + } else { + auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); + a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK)); + a.lsr(TMP2, rhs.reg, imm(_TAG_IMMED1_SIZE)); + a.lsl(TMP1, TMP1, TMP2); + } a.orr(dst.reg, TMP1, imm(_TAG_IMMED1_SMALL)); flush_var(dst); return; } + auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3); Label generic = a.newLabel(), next = a.newLabel(); bool inline_shift = true; diff --git a/erts/emulator/beam/jit/arm/instr_bif.cpp b/erts/emulator/beam/jit/arm/instr_bif.cpp index 186343374a..66efc4beba 100644 --- a/erts/emulator/beam/jit/arm/instr_bif.cpp +++ b/erts/emulator/beam/jit/arm/instr_bif.cpp @@ -30,6 +30,15 @@ extern "C" #include "erl_msacc.h" } +void BeamModuleAssembler::ubif_comment(const ArgWord &Bif) { + if (logger.file()) { + ErtsCodeMFA *mfa = ubif2mfa((void *)Bif.get()); + if (mfa) { + comment("UBIF: %T/%d", mfa->function, mfa->arity); + } + } +} + /* ARG2 = argument vector, ARG4 (!) = bif function pointer * * Result is returned in ARG1 (will be THE_NON_VALUE if the BIF call failed). */ @@ -104,6 +113,7 @@ void BeamModuleAssembler::emit_i_bif1(const ArgSource &Src1, a.str(src1.reg, getXRef(0)); + ubif_comment(Bif); emit_i_bif(Fail, Bif, Dst); } @@ -116,6 +126,7 @@ void BeamModuleAssembler::emit_i_bif2(const ArgSource &Src1, a.stp(src1.reg, src2.reg, getXRef(0)); + ubif_comment(Bif); emit_i_bif(Fail, Bif, Dst); } @@ -131,6 +142,7 @@ void BeamModuleAssembler::emit_i_bif3(const ArgSource &Src1, a.stp(src1.reg, src2.reg, getXRef(0)); a.str(src3.reg, getXRef(2)); + ubif_comment(Bif); emit_i_bif(Fail, Bif, Dst); } @@ -161,6 +173,7 @@ void BeamModuleAssembler::emit_nofail_bif1(const ArgSource &Src1, a.str(src1.reg, getXRef(0)); + ubif_comment(Bif); mov_arg(ARG4, Bif); fragment_call(ga->get_i_bif_guard_shared()); mov_arg(Dst, ARG1); @@ -174,6 +187,7 @@ void BeamModuleAssembler::emit_nofail_bif2(const ArgSource &Src1, a.stp(src1.reg, src2.reg, getXRef(0)); + ubif_comment(Bif); mov_arg(ARG4, Bif); fragment_call(ga->get_i_bif_guard_shared()); mov_arg(Dst, ARG1); @@ -564,6 +578,10 @@ void BeamModuleAssembler::emit_call_light_bif(const ArgWord &Bif, mov_arg(ARG8, Bif); a.adr(ARG3, entry); + if (logger.file()) { + BeamFile_ImportEntry *e = &beam->imports.entries[Exp.get()]; + comment("BIF: %T:%T/%d", e->module, e->function, e->arity); + } fragment_call(ga->get_call_light_bif_shared()); } @@ -776,6 +794,10 @@ void BeamModuleAssembler::emit_call_bif_mfa(const ArgAtom &M, a.adr(ARG3, currLabel); a.sub(ARG2, ARG3, imm(sizeof(ErtsCodeMFA))); + comment("HBIF: %T:%T/%d", + e->info.mfa.module, + e->info.mfa.function, + A.get()); a.mov(ARG4, imm(func)); a.b(resolve_fragment(ga->get_call_bif_shared(), disp128MB)); diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp index 06873cd709..2f408fc178 100644 --- a/erts/emulator/beam/jit/arm/instr_bs.cpp +++ b/erts/emulator/beam/jit/arm/instr_bs.cpp @@ -19,6 +19,7 @@ */ #include "beam_asm.hpp" +#include <numeric> extern "C" { @@ -29,8 +30,6 @@ extern "C" /* Clobbers TMP1+TMP2 * - * If max_size > 0, we jump to the fail label when Size > max_size - * * Returns -1 when the field check always fails, 1 if it may fail, and 0 if it * never fails. */ int BeamModuleAssembler::emit_bs_get_field_size(const ArgSource &Size, @@ -55,18 +54,40 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgSource &Size, return -1; } else { auto size_reg = load_source(Size, TMP2); + bool can_fail = true; + + if (always_small(Size)) { + auto [min, max] = getClampedRange(Size); + can_fail = + !(0 <= min && (max >> (SMALL_BITS - ERL_UNIT_BITS)) == 0); + } /* Negating the tag bits lets us guard against non-smalls, negative * numbers, and overflow with a single `tst` instruction. */ ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); ASSERT(unit <= 1024); - a.eor(out, size_reg.reg, imm(_TAG_IMMED1_SMALL)); - a.tst(out, imm(0xFFF0000000000000UL | _TAG_IMMED1_MASK)); + if (!can_fail) { + comment("simplified segment size checks because " + "the types are known"); + } + + if (unit == 1 && !can_fail) { + a.lsr(out, size_reg.reg, imm(_TAG_IMMED1_SIZE)); + } else { + a.eor(out, size_reg.reg, imm(_TAG_IMMED1_SMALL)); + } + + if (can_fail) { + a.tst(out, imm(0xFFF0000000000000UL | _TAG_IMMED1_MASK)); + } if (unit == 0) { /* Silly but legal.*/ mov_imm(out, 0); + } else if (unit == 1 && !can_fail) { + /* The result is already in the out register. */ + ; } else if (Support::isPowerOf2(unit)) { int trailing_bits = Support::ctz<Eterm>(unit); @@ -88,9 +109,11 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgSource &Size, a.mul(out, out, TMP1); } - a.b_ne(fail); + if (can_fail) { + a.b_ne(fail); + } - return 1; + return can_fail; } } @@ -541,10 +564,9 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgRegister &Src, a.bind(is_binary); { - /* Src is not guaranteed to be inside the live range, so we need to - * stash it during GC. */ - emit_gc_test_preserve(ArgVal(ArgVal::Word, ERL_BIN_MATCHSTATE_SIZE(0)), + emit_gc_test_preserve(ArgWord(ERL_BIN_MATCHSTATE_SIZE(0)), Live, + Src, ARG2); emit_enter_runtime<Update::eStack | Update::eHeap>(Live.get()); @@ -585,9 +607,8 @@ void BeamModuleAssembler::emit_i_bs_match_string(const ArgRegister &Ctx, a.and_(ARG4, TMP2, imm(7)); /* ARG3 = mb->base + (mb->offset >> 3) */ - a.lsr(TMP2, TMP2, imm(3)); a.ldur(TMP1, emit_boxed_val(ctx_reg.reg, base_offset)); - a.add(ARG3, TMP1, TMP2); + a.add(ARG3, TMP1, TMP2, arm::lsr(3)); } emit_enter_runtime(); @@ -624,77 +645,90 @@ void BeamModuleAssembler::emit_i_bs_get_position(const ArgRegister &Ctx, flush_var(dst_reg); } -void BeamModuleAssembler::emit_i_bs_get_fixed_integer(const ArgRegister &Ctx, - const ArgLabel &Fail, - const ArgWord &Live, - const ArgWord &Flags, - const ArgWord &Bits, - const ArgRegister &Dst) { - auto ctx = load_source(Ctx, TMP1); - int flags, bits; - - flags = Flags.get(); - bits = Bits.get(); - - if (bits >= SMALL_BITS) { - emit_gc_test_preserve(ArgVal(ArgVal::Word, BIG_NEED_FOR_BITS(bits)), - Live, - ctx.reg); - } - - lea(ARG4, emit_boxed_val(ctx.reg, offsetof(ErlBinMatchState, mb))); +void BeamModuleAssembler::emit_bs_get_integer2(const ArgLabel &Fail, + const ArgRegister &Ctx, + const ArgWord &Live, + const ArgSource &Sz, + const ArgWord &Unit, + const ArgWord &Flags, + const ArgRegister &Dst) { + Uint size; + Uint flags = Flags.get(); - if (bits >= SMALL_BITS) { - emit_enter_runtime<Update::eHeap>(Live.get()); - } else { - emit_enter_runtime(Live.get()); + if (flags & BSF_NATIVE) { + flags &= ~BSF_NATIVE; + flags |= BSF_LITTLE; } - a.mov(ARG1, c_p); - a.mov(ARG2, bits); - a.mov(ARG3, flags); - /* ARG4 set above. */ - runtime_call<4>(erts_bs_get_integer_2); - - if (bits >= SMALL_BITS) { - emit_leave_runtime<Update::eHeap>(Live.get()); + if (Sz.isSmall() && + (size = Sz.as<ArgSmall>().getUnsigned()) < 8 * sizeof(Uint)) { + /* Segment of a fixed size supported by bs_match. */ + const ArgVal match[] = {ArgAtom(am_ensure_at_least), + ArgWord(size), + Unit, + ArgAtom(am_integer), + Live, + ArgWord(flags), + ArgWord(size), + Unit, + Dst}; + + const Span<ArgVal> args(match, sizeof(match) / sizeof(match[0])); + emit_i_bs_match(Fail, Ctx, args); } else { - emit_leave_runtime(Live.get()); - } - - emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); - mov_arg(Dst, ARG1); -} - -void BeamModuleAssembler::emit_i_bs_get_integer(const ArgRegister &Ctx, - const ArgLabel &Fail, - const ArgWord &Live, - const ArgWord &FlagsAndUnit, - const ArgSource &Sz, - const ArgRegister &Dst) { - Label fail; - int unit; - - fail = resolve_beam_label(Fail, dispUnknown); - unit = FlagsAndUnit.get() >> 3; - - if (emit_bs_get_field_size(Sz, unit, fail, ARG5) >= 0) { - mov_arg(ARG3, Ctx); - mov_arg(ARG4, FlagsAndUnit); - mov_arg(ARG6, Live); + Label fail = resolve_beam_label(Fail, dispUnknown); + int unit = Unit.get(); + + if (emit_bs_get_field_size(Sz, unit, fail, ARG5) >= 0) { + /* This operation can be expensive if a bignum can be + * created because there can be a garbage collection. */ + auto max = std::get<1>(getClampedRange(Sz)); + bool potentially_expensive = + max >= SMALL_BITS || (max * Unit.get()) >= SMALL_BITS; + + mov_arg(ARG3, Ctx); + mov_imm(ARG4, flags); + if (potentially_expensive) { + mov_arg(ARG6, Live); + } else { +#ifdef DEBUG + /* Never actually used. */ + mov_imm(ARG6, 1023); +#endif + } - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(Live.get()); + if (potentially_expensive) { + emit_enter_runtime<Update::eStack | Update::eHeap | + Update::eXRegs | Update::eReductions>( + Live.get()); + } else { + comment("simplified entering runtime because result is always " + "small"); + emit_enter_runtime(Live.get()); + } - a.mov(ARG1, c_p); - load_x_reg_array(ARG2); - runtime_call<6>(beam_jit_bs_get_integer); + a.mov(ARG1, c_p); + if (potentially_expensive) { + load_x_reg_array(ARG2); + } else { +#ifdef DEBUG + /* Never actually used. */ + mov_imm(ARG2, 0); +#endif + } + runtime_call<6>(beam_jit_bs_get_integer); - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(Live.get()); + if (potentially_expensive) { + emit_leave_runtime<Update::eStack | Update::eHeap | + Update::eXRegs | Update::eReductions>( + Live.get()); + } else { + emit_leave_runtime(Live.get()); + } - emit_branch_if_not_value(ARG1, fail); - mov_arg(Dst, ARG1); + emit_branch_if_not_value(ARG1, fail); + mov_arg(Dst, ARG1); + } } } @@ -738,11 +772,7 @@ void BeamModuleAssembler::emit_i_bs_get_binary_all2(const ArgRegister &Ctx, mov_arg(ARG1, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED), - Live, - ARG1); + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, Ctx, ARG1); /* Make field fetching slightly more compact by pre-loading the match * buffer into the right argument slot for `erts_bs_get_binary_all_2`. */ @@ -811,11 +841,7 @@ void BeamModuleAssembler::emit_bs_get_tail(const ArgRegister &Ctx, const ArgWord &Live) { mov_arg(ARG1, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED), - Live, - ARG1); + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, Ctx, ARG1); fragment_call(ga->get_bs_get_tail_shared()); @@ -841,12 +867,34 @@ void BeamModuleAssembler::emit_bs_skip_bits(const ArgLabel &Fail, } void BeamModuleAssembler::emit_i_bs_skip_bits2(const ArgRegister &Ctx, - const ArgRegister &Bits, + const ArgRegister &Size, const ArgLabel &Fail, const ArgWord &Unit) { Label fail = resolve_beam_label(Fail, dispUnknown); - if (emit_bs_get_field_size(Bits, Unit.get(), fail, ARG1) >= 0) { + bool can_fail = true; + + if (always_small(Size)) { + auto [min, max] = getClampedRange(Size); + can_fail = !(0 <= min && (max >> (SMALL_BITS - ERL_UNIT_BITS)) == 0); + } + + if (!can_fail && Unit.get() == 1) { + comment("simplified skipping because the types are known"); + + const int position_offset = offsetof(ErlBinMatchState, mb.offset); + const int size_offset = offsetof(ErlBinMatchState, mb.size); + auto [ctx, size] = load_sources(Ctx, TMP1, Size, TMP2); + + a.ldur(TMP3, emit_boxed_val(ctx.reg, position_offset)); + a.ldur(TMP4, emit_boxed_val(ctx.reg, size_offset)); + + a.add(TMP3, TMP3, size.reg, arm::lsr(_TAG_IMMED1_SIZE)); + a.cmp(TMP3, TMP4); + a.b_hi(resolve_beam_label(Fail, disp1MB)); + + a.stur(TMP3, emit_boxed_val(ctx.reg, position_offset)); + } else if (emit_bs_get_field_size(Size, Unit.get(), fail, ARG1) >= 0) { emit_bs_skip_bits(Fail, Ctx); } } @@ -875,10 +923,9 @@ void BeamModuleAssembler::emit_i_bs_get_binary2(const ArgRegister &Ctx, mov_arg(ARG4, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to - * stash it during GC. */ - emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED), + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, + Ctx, ARG4); lea(ARG4, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb))); @@ -912,9 +959,7 @@ void BeamModuleAssembler::emit_i_bs_get_float2(const ArgRegister &Ctx, mov_arg(ARG4, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgWord(FLOAT_SIZE_OBJECT), Live, ARG4); + emit_gc_test_preserve(ArgWord(FLOAT_SIZE_OBJECT), Live, Ctx, ARG4); if (emit_bs_get_field_size(Sz, unit, fail, ARG2) >= 0) { lea(ARG4, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb))); @@ -1429,10 +1474,49 @@ void BeamGlobalAssembler::emit_bs_bit_size_shared() { a.ret(a64::x30); } +/* + * ARG1 = tagged bignum term + */ +void BeamGlobalAssembler::emit_get_sint64_shared() { + Label success = a.newLabel(); + Label fail = a.newLabel(); + + emit_is_boxed(fail, ARG1); + arm::Gp boxed_ptr = emit_ptr_val(TMP3, ARG1); + a.ldr(TMP1, emit_boxed_val(boxed_ptr)); + a.ldr(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(POS_BIG_SUBTAG)); + a.b_eq(success); + + a.cmp(TMP1, imm(NEG_BIG_SUBTAG)); + a.b_ne(fail); + + a.neg(TMP2, TMP2); + + a.bind(success); + { + a.mov(ARG1, TMP2); + /* Clear Z flag. + * + * TMP1 is known to be POS_BIG_SUBTAG or NEG_BIG_SUBTAG at this point. + */ + ERTS_CT_ASSERT(POS_BIG_SUBTAG != 0 && NEG_BIG_SUBTAG != 0); + a.tst(TMP1, TMP1); + a.ret(a64::x30); + } + + a.bind(fail); + { + a.tst(ZERO, ZERO); + a.ret(a64::x30); + } +} + struct BscSegment { BscSegment() : type(am_false), unit(1), flags(0), src(ArgNil()), size(ArgNil()), - error_info(0), effectiveSize(-1) { + error_info(0), effectiveSize(-1), action(action::DIRECT) { } Eterm type; @@ -1443,19 +1527,443 @@ struct BscSegment { Uint error_info; Sint effectiveSize; + + /* Here are sub actions for storing integer segments. + * + * We use the ACCUMULATE_FIRST and ACCUMULATE actions to shift the + * values of segments with known, small sizes (no more than 64 bits) + * into an accumulator register. + * + * When no more segments can be accumulated, the STORE action is + * used to store the value of the accumulator into the binary. + * + * The DIRECT action is used when it is not possible to use the + * accumulator (for unknown or too large sizes). + */ + enum class action { DIRECT, ACCUMULATE_FIRST, ACCUMULATE, STORE } action; }; +static std::vector<BscSegment> bs_combine_segments( + const std::vector<BscSegment> segments) { + std::vector<BscSegment> segs; + + for (auto seg : segments) { + switch (seg.type) { + case am_integer: { + if (!(0 < seg.effectiveSize && seg.effectiveSize <= 64)) { + /* Unknown or too large size. Handle using the default + * DIRECT action. */ + segs.push_back(seg); + continue; + } + + if (seg.flags & BSF_LITTLE || segs.size() == 0 || + segs.back().action == BscSegment::action::DIRECT) { + /* There are no previous compatible ACCUMULATE / STORE + * actions. Create the first ones. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + continue; + } + + auto prev = segs.back(); + if (prev.flags & BSF_LITTLE) { + /* Little-endian segments cannot be combined with other + * segments. Create new ACCUMULATE_FIRST / STORE actions. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + continue; + } + + /* The current segment is compatible with the previous + * segment. Try combining them. */ + if (prev.effectiveSize + seg.effectiveSize <= 64) { + /* The combined values of the segments fit in the + * accumulator. Insert an ACCUMULATE action for the + * current segment before the pre-existing STORE + * action. */ + segs.pop_back(); + prev.effectiveSize += seg.effectiveSize; + seg.action = BscSegment::action::ACCUMULATE; + segs.push_back(seg); + segs.push_back(prev); + } else { + /* The size exceeds 64 bits. Can't combine. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + } + break; + } + default: + segs.push_back(seg); + break; + } + } + return segs; +} + +/* + * In: + * bin_offset = register to store the bit offset into the binary + * bit_offset = current bit offset into binary, or -1 if unknown + * size = size of segment to be constructed + * (ignored if size_reg is valid register) + * size_reg = if a valid register, it contains the size of + * the segment to be constructed + * + * Out: + * bin_offset register = if bit_offset is not byte aligned, the bit + * offset into the binary + * TMP1 = pointer to the current byte in the binary + * + * Preserves all other ARG* registers. + */ +void BeamModuleAssembler::update_bin_state(arm::Gp bin_offset, + Sint bit_offset, + Sint size, + arm::Gp size_reg) { + int cur_bin_offset = offsetof(ErtsSchedulerRegisters, + aux_regs.d.erl_bits_state.erts_current_bin_); + arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset); + arm::Mem mem_bin_offset = + arm::Mem(scheduler_registers, cur_bin_offset + sizeof(Eterm)); + + if (bit_offset % 8 != 0) { + /* The bit offset is unknown or not byte-aligned. */ + ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state, + erts_current_bin_, + erts_bin_offset_); + a.ldp(TMP2, bin_offset, mem_bin_base); + + if (size_reg.isValid()) { + a.add(TMP1, bin_offset, size_reg); + } else { + add(TMP1, bin_offset, size); + } + a.str(TMP1, mem_bin_offset); + + a.add(TMP1, TMP2, bin_offset, arm::lsr(3)); + } else { + comment("optimized updating of binary construction state"); + ASSERT(size >= 0 || size_reg.isValid()); + ASSERT(bit_offset % 8 == 0); + a.ldr(TMP1, mem_bin_base); + if (size_reg.isValid()) { + if (bit_offset == 0) { + a.str(size_reg, mem_bin_offset); + } else { + add(TMP2, size_reg, bit_offset); + a.str(TMP2, mem_bin_offset); + } + } else { + mov_imm(TMP2, bit_offset + size); + a.str(TMP2, mem_bin_offset); + } + if (bit_offset != 0) { + add(TMP1, TMP1, bit_offset >> 3); + } + } +} + +/* + * The size of the segment is assumed to be in ARG3. + */ +void BeamModuleAssembler::set_zero(Sint effectiveSize) { + Label store_units = a.newLabel(); + Label less_than_a_store_unit = a.newLabel(); + Sint store_unit = 1; + + update_bin_state(ARG2, -1, -1, ARG3); + + if (effectiveSize >= 256) { + /* Store four 64-bit words machine words when the size is + * known and at least 256 bits. */ + store_unit = 4; + a.movi(a64::d31, 0); + } else if (effectiveSize >= 128) { + /* Store two 64-bit words machine words when the size is + * known and at least 128 bits. */ + store_unit = 2; + } + + if (effectiveSize < Sint(store_unit * 8 * sizeof(Eterm))) { + /* The size is either not known or smaller than a word. */ + a.cmp(ARG3, imm(store_unit * 8 * sizeof(Eterm))); + a.b_lt(less_than_a_store_unit); + } + + a.bind(store_units); + if (store_unit == 4) { + a.stp(a64::q31, a64::q31, arm::Mem(TMP1).post(sizeof(Eterm[4]))); + } else if (store_unit == 2) { + a.stp(ZERO, ZERO, arm::Mem(TMP1).post(sizeof(Eterm[2]))); + } else { + a.str(ZERO, arm::Mem(TMP1).post(sizeof(Eterm))); + } + a.sub(ARG3, ARG3, imm(store_unit * 8 * sizeof(Eterm))); + + a.cmp(ARG3, imm(store_unit * 8 * sizeof(Eterm))); + a.b_ge(store_units); + + a.bind(less_than_a_store_unit); + if (effectiveSize < 0) { + /* Unknown size. */ + Label byte_loop = a.newLabel(); + Label done = a.newLabel(); + + ASSERT(store_unit = 1); + + a.cbz(ARG3, done); + + a.bind(byte_loop); + a.strb(ZERO.w(), arm::Mem(TMP1).post(1)); + a.subs(ARG3, ARG3, imm(8)); + a.b_gt(byte_loop); + + a.bind(done); + } else if (effectiveSize % (store_unit * 8 * sizeof(Eterm)) != 0) { + /* The size is known, and we know that there are less than + * 256 bits to initialize. */ + if (store_unit == 4 && (effectiveSize & 255) >= 128) { + a.stp(ZERO, ZERO, arm::Mem(TMP1).post(16)); + } + + if ((effectiveSize & 127) >= 64) { + a.str(ZERO, arm::Mem(TMP1).post(8)); + } + + if ((effectiveSize & 63) >= 32) { + a.str(ZERO.w(), arm::Mem(TMP1).post(4)); + } + + if ((effectiveSize & 31) >= 16) { + a.strh(ZERO.w(), arm::Mem(TMP1).post(2)); + } + + if ((effectiveSize & 15) >= 8) { + a.strb(ZERO.w(), arm::Mem(TMP1).post(1)); + } + + if ((effectiveSize & 7) > 0) { + a.strb(ZERO.w(), arm::Mem(TMP1)); + } + } +} + +/* + * In: + * + * ARG1 = valid unicode code point (=> 0x80) to encode + * + * Out: + * + * ARG1 = the code point encoded in UTF-8. + * ARG4 = number of bits of result (16, 24, or 32) + * + * Preserves other ARG* registers, clobbers TMP* registers + */ +void BeamGlobalAssembler::emit_construct_utf8_shared() { + Label more_than_two_bytes = a.newLabel(); + Label four_bytes = a.newLabel(); + const arm::Gp value = ARG1; + const arm::Gp num_bits = ARG4; + + a.cmp(value, imm(0x800)); + a.b_hs(more_than_two_bytes); + + /* Encode Unicode code point in two bytes. */ + a.ubfiz(TMP1, value, imm(8), imm(6)); + mov_imm(TMP2, 0x80c0); + a.orr(TMP1, TMP1, value, arm::lsr(6)); + mov_imm(num_bits, 16); + a.orr(value, TMP1, TMP2); + a.ret(a64::x30); + + /* Test whether the value should be encoded in four bytes. */ + a.bind(more_than_two_bytes); + a.lsr(TMP1, value, imm(16)); + a.cbnz(TMP1, four_bytes); + + /* Encode Unicode code point in three bytes. */ + a.lsl(TMP1, value, imm(2)); + a.ubfiz(TMP2, value, imm(16), imm(6)); + a.and_(TMP1, TMP1, imm(0x3f00)); + mov_imm(num_bits, 24); + a.orr(TMP1, TMP1, value, arm::lsr(12)); + a.orr(TMP1, TMP1, TMP2); + mov_imm(TMP2, 0x8080e0); + a.orr(value, TMP1, TMP2); + a.ret(a64::x30); + + /* Encode Unicode code point in four bytes. */ + a.bind(four_bytes); + a.lsl(TMP1, value, imm(10)); + a.lsr(TMP2, value, imm(4)); + a.and_(TMP1, TMP1, imm(0x3f0000)); + a.and_(TMP2, TMP2, imm(0x3f00)); + a.bfxil(TMP1, value, imm(18), imm(14)); + mov_imm(num_bits, 32); + a.bfi(TMP1, value, imm(24), imm(6)); + a.orr(TMP1, TMP1, TMP2); + mov_imm(TMP2, 0x808080f0); + a.orr(value, TMP1, TMP2); + a.ret(a64::x30); +} + +void BeamModuleAssembler::emit_construct_utf8(const ArgVal &Src, + Sint bit_offset, + bool is_byte_aligned) { + Label prepare_store = a.newLabel(); + Label store = a.newLabel(); + Label next = a.newLabel(); + + comment("construct utf8 segment"); + auto src = load_source(Src, ARG1); + + a.lsr(ARG1, src.reg, imm(_TAG_IMMED1_SIZE)); + mov_imm(ARG4, 8); + a.cmp(ARG1, imm(0x80)); + a.b_lo(prepare_store); + + fragment_call(ga->get_construct_utf8_shared()); + + a.bind(prepare_store); + arm::Gp bin_offset = ARG3; + update_bin_state(bin_offset, bit_offset, -1, ARG4); + + if (!is_byte_aligned) { + /* Not known to be byte-aligned. Must test alignment. */ + a.ands(TMP2, bin_offset, imm(7)); + a.b_eq(store); + + /* We must combine the last partial byte with the UTF-8 + * encoded code point. */ + a.ldrb(TMP5.w(), arm::Mem(TMP1)); + + a.rev64(TMP4, ARG1); + a.lsr(TMP4, TMP4, TMP2); + a.rev64(TMP4, TMP4); + + a.lsl(TMP5, TMP5, TMP2); + a.and_(TMP5, TMP5, imm(~0xff)); + a.lsr(TMP5, TMP5, TMP2); + + a.orr(ARG1, TMP4, TMP5); + + a.add(ARG4, ARG4, imm(8)); + } + + a.bind(store); + if (bit_offset % (4 * 8) == 0) { + /* This segment is aligned on a 4-byte boundary. This implies + * that a 4-byte write will be inside the allocated binary. */ + a.str(ARG1.w(), arm::Mem(TMP1)); + } else { + Label do_store_1 = a.newLabel(); + Label do_store_2 = a.newLabel(); + + /* Unsuitable or unknown alignment. We must be careful not + * to write beyound the allocated end of the binary. */ + a.cmp(ARG4, imm(8)); + a.b_ne(do_store_1); + + a.strb(ARG1.w(), arm::Mem(TMP1)); + a.b(next); + + a.bind(do_store_1); + a.cmp(ARG4, imm(24)); + a.b_hi(do_store_2); + + a.strh(ARG1.w(), arm::Mem(TMP1)); + a.cmp(ARG4, imm(16)); + a.b_eq(next); + + a.lsr(ARG1, ARG1, imm(16)); + a.strb(ARG1.w(), arm::Mem(TMP1, 2)); + a.b(next); + + a.bind(do_store_2); + a.str(ARG1.w(), arm::Mem(TMP1)); + + if (!is_byte_aligned) { + a.cmp(ARG4, imm(32)); + a.b_eq(next); + + a.lsr(ARG1, ARG1, imm(32)); + a.strb(ARG1.w(), arm::Mem(TMP1, 4)); + } + } + + a.bind(next); +} + +/* + * In: + * TMP1 = pointer to current byte + * ARG3 = bit offset + * ARG4 = number of bits to write + * ARG8 = data to write + */ +void BeamGlobalAssembler::emit_store_unaligned() { + Label loop = a.newLabel(); + Label done = a.newLabel(); + const arm::Gp left_bit_offset = ARG3; + const arm::Gp right_bit_offset = TMP6; + const arm::Gp num_bits = ARG4; + const arm::Gp bitdata = ARG8; + + a.ldrb(TMP5.w(), arm::Mem(TMP1)); + + a.and_(TMP4, bitdata, imm(0xff)); + a.lsr(TMP4, TMP4, left_bit_offset); + + a.lsl(TMP5, TMP5, left_bit_offset); + a.and_(TMP5, TMP5, imm(~0xff)); + a.lsr(TMP5, TMP5, left_bit_offset); + + a.orr(TMP5, TMP4, TMP5); + + a.strb(TMP5.w(), arm::Mem(TMP1).post(1)); + + mov_imm(right_bit_offset, 8); + a.sub(right_bit_offset, right_bit_offset, left_bit_offset); + + a.rev64(bitdata, bitdata); + a.lsl(bitdata, bitdata, right_bit_offset); + + a.subs(num_bits, num_bits, right_bit_offset); + a.b_le(done); + + a.bind(loop); + a.ror(bitdata, bitdata, imm(56)); + a.strb(bitdata.w(), arm::Mem(TMP1).post(1)); + a.subs(num_bits, num_bits, imm(8)); + a.b_gt(loop); + + a.bind(done); + a.ret(a64::x30); +} + void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, const ArgWord &Alloc, const ArgWord &Live0, const ArgRegister &Dst, const Span<ArgVal> &args) { Uint num_bits = 0; + Uint estimated_num_bits = 0; std::size_t n = args.size(); std::vector<BscSegment> segments; - Label error; + Label error; /* Intentionally uninitialized */ ArgWord Live = Live0; arm::Gp sizeReg; + Sint allocated_size = -1; + bool need_error_handler = false; /* * Collect information about each segment and calculate sizes of @@ -1501,17 +2009,65 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, seg.error_info = beam_jit_set_bsc_segment_op(bsc_segment, bsc_op); /* + * Test whether we can omit the code for the error handler. + */ + switch (seg.type) { + case am_append: + if (std::gcd(seg.unit, getSizeUnit(seg.src)) != seg.unit) { + need_error_handler = true; + } + break; + case am_binary: + if (!(seg.size.isAtom() && seg.size.as<ArgAtom>().get() == am_all && + std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit)) { + need_error_handler = true; + } + break; + case am_integer: + if (!always_one_of(seg.src, BEAM_TYPE_INTEGER)) { + need_error_handler = true; + } + break; + case am_private_append: + case am_string: + break; + default: + need_error_handler = true; + break; + } + + /* * Attempt to calculate the effective size of this segment. - * Give up is variable or invalid. + * Give up if variable or invalid. */ if (seg.size.isSmall() && seg.unit != 0) { Uint unsigned_size = seg.size.as<ArgSmall>().getUnsigned(); - if ((unsigned_size >> (sizeof(Eterm) - 1) * 8) == 0) { + if ((unsigned_size >> (sizeof(Eterm) - 1) * 8) != 0) { + /* Suppress creation of heap binary. */ + estimated_num_bits += (ERL_ONHEAP_BIN_LIMIT + 1) * 8; + } else { /* This multiplication cannot overflow. */ Uint seg_size = seg.unit * unsigned_size; seg.effectiveSize = seg_size; num_bits += seg_size; + estimated_num_bits += seg_size; + } + } else if (seg.unit > 0) { + auto max = std::min(std::get<1>(getClampedRange(seg.size)), + Sint((ERL_ONHEAP_BIN_LIMIT + 1) * 8)); + estimated_num_bits += max * seg.unit; + } else { + switch (seg.type) { + case am_utf8: + case am_utf16: + case am_utf32: + estimated_num_bits += 32; + break; + default: + /* Suppress creation of heap binary. */ + estimated_num_bits += (ERL_ONHEAP_BIN_LIMIT + 1) * 8; + break; } } @@ -1520,14 +2076,15 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, /* At least one segment will need a dynamic size * calculation. */ sizeReg = ARG8; + need_error_handler = true; } segments.insert(segments.end(), seg); } - if (Fail.get() != 0) { + if (need_error_handler && Fail.get() != 0) { error = resolve_beam_label(Fail, dispUnknown); - } else { + } else if (need_error_handler) { Label past_error = a.newLabel(); a.b(past_error); @@ -1550,6 +2107,8 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } a.bind(past_error); + } else { + comment("(cannot fail)"); } /* We count the total number of bits in an unsigned integer. To @@ -1575,13 +2134,49 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, if (seg.size.isAtom() && seg.size.as<ArgAtom>().get() == am_all && seg.type == am_binary) { comment("size of an entire binary"); - mov_arg(ARG1, seg.src); - a.mov(ARG3, ARG1); - fragment_call(ga->get_bs_bit_size_shared()); if (exact_type(seg.src, BEAM_TYPE_BITSTRING)) { - comment("skipped check for success since the source " - "is always a bit string"); + auto src = load_source(seg.src, ARG1); + arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); + auto unit = getSizeUnit(seg.src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + + if (is_bitstring) { + comment("inlined size code because the value is always " + "a bitstring"); + } else { + comment("inlined size code because the value is always " + "a binary"); + } + + a.ldur(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + + if (is_bitstring) { + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + } + + a.add(sizeReg, sizeReg, TMP2, arm::lsl(3)); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const int bit_number = 3; + ERTS_CT_ASSERT( + (_TAG_HEADER_SUB_BIN & (1 << bit_number)) != 0 && + (_TAG_HEADER_REFC_BIN & (1 << bit_number)) == 0 && + (_TAG_HEADER_HEAP_BIN & (1 << bit_number)) == 0); + + a.tbz(TMP1, imm(bit_number), not_sub_bin); + + a.ldurb(TMP2.w(), + emit_boxed_val(boxed_ptr, + offsetof(ErlSubBin, bitsize))); + a.add(sizeReg, sizeReg, TMP2); + + a.bind(not_sub_bin); + } } else { + mov_arg(ARG1, seg.src); + a.mov(ARG3, ARG1); + fragment_call(ga->get_bs_bit_size_shared()); if (Fail.get() == 0) { mov_imm(ARG4, beam_jit_update_bsc_reason_info(seg.error_info, @@ -1590,14 +2185,14 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, BSC_VALUE_ARG3)); } a.b_mi(resolve_label(error, disp1MB)); + a.add(sizeReg, sizeReg, ARG1); } - a.add(sizeReg, sizeReg, ARG1); } else if (seg.unit != 0) { bool can_fail = true; comment("size binary/integer/float/string"); if (always_small(seg.size)) { - auto [min, _] = getIntRange(seg.size); + auto min = std::get<0>(getClampedRange(seg.size)); if (min >= 0) { can_fail = false; } @@ -1627,10 +2222,10 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, if (can_fail) { a.tbnz(ARG3, 63, resolve_label(error, disp32K)); } - a.asr(TMP1, ARG3, imm(_TAG_IMMED1_SIZE)); if (seg.unit == 1) { - a.add(sizeReg, sizeReg, TMP1); + a.add(sizeReg, sizeReg, ARG3, arm::asr(_TAG_IMMED1_SIZE)); } else { + a.asr(TMP1, ARG3, imm(_TAG_IMMED1_SIZE)); if (Fail.get() == 0) { mov_imm(ARG4, beam_jit_update_bsc_reason_info( @@ -1639,7 +2234,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, BSC_INFO_SIZE, BSC_VALUE_ARG3)); } - a.tst(TMP1, imm(0xffful << 52)); + a.tst(TMP1, imm(0xffful << (SMALL_BITS - ERL_UNIT_BITS))); a.b_ne(resolve_label(error, disp1MB)); mov_imm(TMP2, seg.unit); a.madd(sizeReg, TMP1, TMP2, sizeReg); @@ -1649,24 +2244,61 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, case am_utf8: { comment("size utf8"); Label next = a.newLabel(); - auto src_reg = load_source(seg.src, TMP1); - a.lsr(TMP1, src_reg.reg, imm(_TAG_IMMED1_SIZE)); - mov_imm(TMP2, 1 * 8); + mov_arg(ARG3, seg.src); + + if (Fail.get() == 0) { + mov_imm(ARG4, + beam_jit_update_bsc_reason_info(seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG3)); + } + + if (always_small(seg.src)) { + comment("skipped test for small value since it is always " + "small"); + } else if (always_one_of(seg.src, + BEAM_TYPE_INTEGER | + BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operand since other " + "types are boxed"); + emit_is_not_boxed(resolve_label(error, dispUnknown), ARG3); + } else { + a.and_(TMP1, ARG3, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_ne(resolve_label(error, disp1MB)); + } + + a.asr(TMP1, ARG3, imm(_TAG_IMMED1_SIZE)); + mov_imm(TMP2, 1); a.cmp(TMP1, imm(0x7F)); a.b_ls(next); - mov_imm(TMP2, 2 * 8); + mov_imm(TMP2, 2); a.cmp(TMP1, imm(0x7FFUL)); a.b_ls(next); + /* Ensure that the value is not in the invalid range + * 0xD800 through 0xDFFF. */ + a.lsr(TMP3, TMP1, imm(11)); + a.cmp(TMP3, 0x1b); + a.b_eq(resolve_label(error, disp1MB)); + a.cmp(TMP1, imm(0x10000UL)); - mov_imm(TMP2, 3 * 8); - mov_imm(TMP3, 4 * 8); - a.csel(TMP2, TMP2, TMP3, arm::CondCode::kLO); + a.cset(TMP2, arm::CondCode::kHS); + a.add(TMP2, TMP2, imm(3)); + + auto [min, max] = getClampedRange(seg.src); + if (0 <= min && max < 0x110000) { + comment("skipped range check for unicode code point"); + } else { + a.cmp(TMP1, 0x110000); + a.b_hs(resolve_label(error, disp1MB)); + } a.bind(next); - a.add(sizeReg, sizeReg, TMP2); + a.add(sizeReg, sizeReg, TMP2, arm::lsl(3)); break; } case am_utf16: { @@ -1748,15 +2380,21 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | Update::eReductions>(Live.get() + 1); - if (Fail.get() == 0) { - mov_arg(ARG3, ArgXRegister(Live.get())); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_FVALUE, - BSC_VALUE_ARG3)); + if (std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit) { + /* There is no way the call can fail with a system_limit + * exception on a 64-bit architecture. */ + comment("skipped test for success because units are compatible"); + } else { + if (Fail.get() == 0) { + mov_arg(ARG3, ArgXRegister(Live.get())); + mov_imm(ARG4, + beam_jit_update_bsc_reason_info(seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_FVALUE, + BSC_VALUE_ARG3)); + } + emit_branch_if_not_value(ARG1, resolve_label(error, dispUnknown)); } - emit_branch_if_not_value(ARG1, resolve_label(error, dispUnknown)); } else if (segments[0].type == am_private_append) { BscSegment seg = segments[0]; comment("private append to binary"); @@ -1773,6 +2411,83 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, runtime_call<4>(erts_bs_private_append_checked); emit_leave_runtime(Live.get()); /* There is no way the call can fail on a 64-bit architecture. */ + } else if (estimated_num_bits % 8 == 0 && + estimated_num_bits / 8 <= ERL_ONHEAP_BIN_LIMIT) { + Uint need; + int cur_bin_offset = + offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + + offsetof(struct erl_bits_state, erts_current_bin_); + arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset); + arm::Mem mem_bin_offset = + arm::Mem(scheduler_registers, cur_bin_offset + sizeof(Eterm)); + ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state, + erts_current_bin_, + erts_bin_offset_); + + if (sizeReg.isValid()) { + Label after_gc_check = a.newLabel(); + + comment("allocate heap binary of dynamic size (=< %ld bits)", + estimated_num_bits); + + /* Calculate number of bytes to allocate. */ + need = (heap_bin_size(0) + Alloc.get() + S_RESERVED); + a.lsr(sizeReg, sizeReg, imm(3)); + a.add(TMP3, sizeReg, imm(7)); + a.and_(TMP3, TMP3, imm(-8)); + a.add(TMP1, TMP3, imm(need * sizeof(Eterm))); + + /* Do a GC test. */ + a.add(ARG3, HTOP, TMP1); + a.cmp(ARG3, E); + a.b_ls(after_gc_check); + + a.stp(sizeReg, TMP3, TMP_MEM1q); + + mov_imm(ARG4, Live.get()); + fragment_call(ga->get_garbage_collect()); + + a.ldp(sizeReg, TMP3, TMP_MEM1q); + + a.bind(after_gc_check); + + mov_imm(TMP1, header_heap_bin(0)); + a.lsr(TMP4, TMP3, imm(3)); + a.add(TMP1, TMP1, TMP4, arm::lsl(_HEADER_ARITY_OFFS)); + + /* Create the heap binary. */ + a.add(ARG1, HTOP, imm(TAG_PRIMARY_BOXED)); + a.stp(TMP1, sizeReg, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + + /* Initialize the erl_bin_state struct. */ + a.stp(HTOP, ZERO, mem_bin_base); + + /* Update HTOP. */ + a.add(HTOP, HTOP, TMP3); + } else { + Uint num_bytes = num_bits / 8; + + comment("allocate heap binary of static size"); + + allocated_size = (num_bytes + 7) & (-8); + + /* Ensure that there is sufficient room on the heap. */ + need = heap_bin_size(num_bytes) + Alloc.get(); + emit_gc_test(ArgWord(0), ArgWord(need), Live); + + mov_imm(TMP1, header_heap_bin(num_bytes)); + mov_imm(TMP2, num_bytes); + + /* Create the heap binary. */ + a.add(ARG1, HTOP, imm(TAG_PRIMARY_BOXED)); + a.stp(TMP1, TMP2, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + + /* Initialize the erl_bin_state struct. */ + a.stp(HTOP, ZERO, mem_bin_base); + + /* Update HTOP. */ + a.add(HTOP, HTOP, imm(allocated_size)); + } } else { comment("allocate binary"); mov_arg(ARG5, Alloc); @@ -1786,11 +2501,11 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, comment("(size in bits)"); a.mov(ARG4, sizeReg); runtime_call<6>(beam_jit_bs_init_bits); - } else if (num_bits % 8 == 0) { - comment("(size in bytes)"); - mov_imm(ARG4, num_bits / 8); - runtime_call<6>(beam_jit_bs_init); } else { + allocated_size = (num_bits + 7) / 8; + if (allocated_size <= ERL_ONHEAP_BIN_LIMIT) { + allocated_size = (allocated_size + 7) & (-8); + } mov_imm(ARG4, num_bits); runtime_call<6>(beam_jit_bs_init_bits); } @@ -1799,11 +2514,24 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } a.str(ARG1, TMP_MEM1q); + segments = bs_combine_segments(segments); + + /* Keep track of the bit offset from the being of the binary. + * Set to -1 if offset is not known (when a segment of unknown + * size has been seen). */ + Sint bit_offset = 0; + + /* Keep track of whether the current segment is byte-aligned. (A + * segment can be known to be byte-aligned even if the bit offset + * is unknown.) */ + bool is_byte_aligned = true; + /* Build each segment of the binary. */ for (auto seg : segments) { switch (seg.type) { case am_append: case am_private_append: + bit_offset = -1; break; case am_binary: { Uint error_info; @@ -1838,8 +2566,9 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, BSC_REASON_BADARG, BSC_INFO_UNIT, BSC_VALUE_FVALUE); - if (seg.unit == 1) { - comment("skipped test for success because unit =:= 1"); + if (std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit) { + comment("skipped test for success because units are " + "compatible"); can_fail = false; } } else { @@ -1847,8 +2576,8 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, * the value is a non-negative small in the * appropriate range. Multiply the size with the * unit. */ - mov_arg(ARG3, seg.size); - a.asr(ARG3, ARG3, imm(_TAG_IMMED1_SIZE)); + auto r = load_source(seg.size, ARG3); + a.asr(ARG3, r.reg, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(TMP1, seg.unit); a.mul(ARG3, ARG3, TMP1); @@ -1879,8 +2608,8 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, if (seg.effectiveSize >= 0) { mov_imm(ARG3, seg.effectiveSize); } else { - mov_arg(ARG3, seg.size); - a.asr(ARG3, ARG3, imm(_TAG_IMMED1_SIZE)); + auto r = load_source(seg.size, ARG3); + a.asr(ARG3, r.reg, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(TMP1, seg.unit); a.mul(ARG3, ARG3, TMP1); @@ -1904,38 +2633,282 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, emit_branch_if_value(ARG1, resolve_label(error, dispUnknown)); break; case am_integer: - comment("construct integer segment"); - if (seg.effectiveSize >= 0) { - mov_imm(ARG3, seg.effectiveSize); - } else { - mov_arg(ARG3, seg.size); - a.asr(ARG3, ARG3, imm(_TAG_IMMED1_SIZE)); - if (seg.unit != 1) { - mov_imm(TMP1, seg.unit); - a.mul(ARG3, ARG3, TMP1); + switch (seg.action) { + case BscSegment::action::ACCUMULATE_FIRST: + case BscSegment::action::ACCUMULATE: { + /* Shift an integer of known size (no more than 64 bits) + * into a word-size accumulator. */ + Label value_is_small = a.newLabel(); + Label done = a.newLabel(); + + comment("accumulate value for integer segment"); + auto src = load_source(seg.src, ARG1); + if (seg.effectiveSize < 64 && + seg.action == BscSegment::action::ACCUMULATE) { + a.lsl(ARG8, ARG8, imm(seg.effectiveSize)); } + + if (!always_small(seg.src)) { + if (always_one_of(seg.src, + BEAM_TYPE_INTEGER | + BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types " + "are boxed"); + emit_is_boxed(value_is_small, seg.src, src.reg); + } else { + a.and_(TMP1, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_eq(value_is_small); + } + + /* The value is boxed. If it is a bignum, extract the + * least significant 64 bits. */ + mov_var(ARG1, src); + fragment_call(ga->get_get_sint64_shared()); + if (seg.effectiveSize == 64) { + a.mov(ARG8, ARG1); + } else { + a.bfxil(ARG8, + ARG1, + arm::lsr(0), + imm(seg.effectiveSize)); + } + + if (always_one_of(seg.src, BEAM_TYPE_INTEGER)) { + a.b(done); + } else { + a.b_ne(done); + + /* Not a bignum. Signal error. */ + if (Fail.get() == 0) { + mov_imm(ARG4, + beam_jit_update_bsc_reason_info( + seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG1)); + } + a.b(resolve_label(error, disp128MB)); + } + } + + a.bind(value_is_small); + if (seg.effectiveSize == 64) { + a.asr(ARG8, src.reg, imm(_TAG_IMMED1_SIZE)); + } else if (seg.effectiveSize + _TAG_IMMED1_SIZE > 64) { + a.asr(TMP1, src.reg, imm(_TAG_IMMED1_SIZE)); + a.bfxil(ARG8, TMP1, arm::lsr(0), imm(seg.effectiveSize)); + } else { + a.bfxil(ARG8, + src.reg, + arm::lsr(_TAG_IMMED1_SIZE), + imm(seg.effectiveSize)); + } + + a.bind(done); + break; } - mov_arg(ARG2, seg.src); - mov_imm(ARG4, seg.flags); - load_erl_bits_state(ARG1); + case BscSegment::action::STORE: { + /* The accumulator is now full or the next segment is + * not possible to accumulate, so it's time to store + * the accumulator to the current position in the + * binary. */ + Label store = a.newLabel(); + Label done = a.newLabel(); + + comment("construct integer segment from accumulator"); + + /* First we'll need to ensure that the value in the + * accumulator is in little endian format. */ + ASSERT(seg.effectiveSize >= 0); + if (seg.effectiveSize % 8) { + Uint complete_bytes = 8 * (seg.effectiveSize / 8); + Uint num_partial = seg.effectiveSize % 8; + if (seg.flags & BSF_LITTLE) { + a.ubfx(TMP1, + ARG8, + imm(complete_bytes), + imm(num_partial)); + a.bfc(ARG8, + arm::lsr(complete_bytes), + imm(64 - complete_bytes)); + a.bfi(ARG8, + TMP1, + imm(complete_bytes + 8 - num_partial), + imm(num_partial)); + } else { + a.lsl(ARG8, ARG8, imm(64 - seg.effectiveSize)); + a.rev64(ARG8, ARG8); + } + } else if ((seg.flags & BSF_LITTLE) == 0) { + switch (seg.effectiveSize) { + case 8: + break; + case 16: + a.rev16(ARG8, ARG8); + break; + case 32: + a.rev32(ARG8, ARG8); + break; + case 64: + a.rev64(ARG8, ARG8); + break; + default: + a.rev64(ARG8, ARG8); + a.lsr(ARG8, ARG8, imm(64 - seg.effectiveSize)); + } + } - emit_enter_runtime(Live.get()); - runtime_call<4>(erts_new_bs_put_integer); - emit_leave_runtime(Live.get()); + arm::Gp bin_offset = ARG3; + arm::Gp bin_data = ARG8; + + update_bin_state(bin_offset, + bit_offset, + seg.effectiveSize, + arm::Gp()); + + if (!is_byte_aligned) { + if (bit_offset < 0) { + /* Bit offset is unknown. Must test alignment. */ + a.ands(bin_offset, bin_offset, imm(7)); + a.b_eq(store); + } else if (bit_offset >= 0) { + /* Alignment is known to be unaligned. */ + mov_imm(bin_offset, bit_offset & 7); + } + + /* Bit offset is tested or known to be unaligned. */ + mov_imm(ARG4, seg.effectiveSize); + fragment_call(ga->get_store_unaligned()); + + if (bit_offset < 0) { + /* The bit offset is unknown, which implies that + * there exists store code that we will need to + * branch past. */ + a.b(done); + } + } - if (exact_type(seg.src, BEAM_TYPE_INTEGER)) { - comment("skipped test for success because construction can't " - "fail"); - } else { - if (Fail.get() == 0) { - mov_arg(ARG3, seg.src); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_TYPE, - BSC_VALUE_ARG3)); + a.bind(store); + + if (bit_offset < 0 || is_byte_aligned) { + /* Bit offset is tested or known to be + * byte-aligned. Emit inline code to store the + * value of the accumulator into the binary. */ + int num_bytes = (seg.effectiveSize + 7) / 8; + + /* If more than one instruction is required for + * doing the store, test whether it would be safe + * to do a single 32 or 64 bit store. */ + switch (num_bytes) { + case 3: + if (bit_offset >= 0 && + allocated_size * 8 - bit_offset >= 32) { + comment("simplified complicated store"); + num_bytes = 4; + } + break; + case 5: + case 6: + case 7: + if (bit_offset >= 0 && + allocated_size * 8 - bit_offset >= 64) { + comment("simplified complicated store"); + num_bytes = 8; + } + break; + } + + do { + switch (num_bytes) { + case 1: + a.strb(bin_data.w(), arm::Mem(TMP1)); + break; + case 2: + a.strh(bin_data.w(), arm::Mem(TMP1)); + break; + case 3: + a.strh(bin_data.w(), arm::Mem(TMP1)); + a.lsr(bin_data, bin_data, imm(16)); + a.strb(bin_data.w(), arm::Mem(TMP1, 2)); + break; + case 4: + a.str(bin_data.w(), arm::Mem(TMP1)); + break; + case 5: + case 6: + case 7: + a.str(bin_data.w(), arm::Mem(TMP1).post(4)); + a.lsr(bin_data, bin_data, imm(32)); + break; + case 8: + a.str(bin_data, arm::Mem(TMP1)); + num_bytes = 0; + break; + } + num_bytes -= 4; + } while (num_bytes > 0); + } + + a.bind(done); + break; + } + case BscSegment::action::DIRECT: + /* This segment either has a size exceeding the maximum + * accumulator size of 64 bits or has a variable size. + * + * First load the effective size (size * unit) into ARG3. + */ + comment("construct integer segment"); + if (seg.effectiveSize >= 0) { + mov_imm(ARG3, seg.effectiveSize); + } else { + auto size = load_source(seg.size, TMP1); + a.lsr(ARG3, size.reg, imm(_TAG_IMMED1_SIZE)); + if (Support::isPowerOf2(seg.unit)) { + Uint trailing_bits = Support::ctz<Eterm>(seg.unit); + if (trailing_bits) { + a.lsl(ARG3, ARG3, imm(trailing_bits)); + } + } else { + mov_imm(TMP1, seg.unit); + a.mul(ARG3, ARG3, TMP1); + } + } + + if (is_byte_aligned && seg.src.isSmall() && + seg.src.as<ArgSmall>().getSigned() == 0) { + /* Optimize the special case of setting a known + * byte-aligned segment to zero. */ + comment("optimized setting segment to 0"); + set_zero(seg.effectiveSize); + } else { + /* Call the helper function to fetch and store the + * integer into the binary. */ + mov_arg(ARG2, seg.src); + mov_imm(ARG4, seg.flags); + load_erl_bits_state(ARG1); + + emit_enter_runtime(Live.get()); + runtime_call<4>(erts_new_bs_put_integer); + emit_leave_runtime(Live.get()); + + if (exact_type(seg.src, BEAM_TYPE_INTEGER)) { + comment("skipped test for success because construction " + "can't fail"); + } else { + if (Fail.get() == 0) { + mov_arg(ARG3, seg.src); + mov_imm(ARG4, + beam_jit_update_bsc_reason_info( + seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG3)); + } + a.cbz(ARG1, resolve_label(error, disp1MB)); + } } - a.cbz(ARG1, resolve_label(error, disp1MB)); } break; case am_string: { @@ -1953,27 +2926,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, emit_leave_runtime(Live.get()); break; } - case am_utf8: - comment("construct utf8 segment"); - mov_arg(ARG2, seg.src); - load_erl_bits_state(ARG1); - - emit_enter_runtime(Live.get()); - runtime_call<2>(erts_bs_put_utf8); - - emit_leave_runtime(Live.get()); - if (Fail.get() == 0) { - mov_arg(ARG3, seg.src); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_TYPE, - BSC_VALUE_ARG3)); - } - a.cbz(ARG1, resolve_label(error, disp1MB)); + case am_utf8: { + emit_construct_utf8(seg.src, bit_offset, is_byte_aligned); break; + } case am_utf16: - comment("construct utf8 segment"); + comment("construct utf16 segment"); mov_arg(ARG2, seg.src); a.mov(ARG3, seg.flags); load_erl_bits_state(ARG1); @@ -2016,8 +2974,918 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, ASSERT(0); break; } + + /* Try to keep track of the bit offset. */ + if (bit_offset >= 0 && (seg.action == BscSegment::action::DIRECT || + seg.action == BscSegment::action::STORE)) { + if (seg.effectiveSize >= 0) { + bit_offset += seg.effectiveSize; + } else { + bit_offset = -1; + } + } + + /* Try to keep track whether the next segment is byte + * aligned. */ + if (seg.type == am_append || seg.type == am_private_append) { + if (std::gcd(getSizeUnit(seg.src), 8) != 8) { + is_byte_aligned = false; + } + } else if (bit_offset % 8 == 0) { + is_byte_aligned = true; + } else if (seg.effectiveSize >= 0) { + if (seg.effectiveSize % 8 != 0) { + is_byte_aligned = false; + } + } else if (std::gcd(seg.unit, 8) != 8) { + is_byte_aligned = false; + } } comment("done"); mov_arg(Dst, TMP_MEM1q); } + +/* + * Here follows the bs_match instruction and friends. + */ + +struct BsmSegment { + BsmSegment() + : action(action::TEST_HEAP), live(ArgNil()), size(0), unit(1), + flags(0), dst(ArgXRegister(0)){}; + + enum class action { + TEST_HEAP, + ENSURE_AT_LEAST, + ENSURE_EXACTLY, + READ, + EXTRACT_BINARY, + EXTRACT_INTEGER, + GET_INTEGER, + GET_BINARY, + SKIP, + DROP, + GET_TAIL, + EQ + } action; + ArgVal live; + Uint size; + Uint unit; + Uint flags; + ArgRegister dst; +}; + +void BeamModuleAssembler::emit_read_bits(Uint bits, + const arm::Gp bin_base, + const arm::Gp bin_offset, + const arm::Gp bitdata) { + Label handle_partial = a.newLabel(); + Label rev64 = a.newLabel(); + Label shift = a.newLabel(); + Label read_done = a.newLabel(); + + bool need_rev64 = false; + + const arm::Gp bin_byte_ptr = TMP2; + const arm::Gp bit_offset = TMP4; + const arm::Gp tmp = TMP5; + + auto num_partial = bits % 8; + + ASSERT(1 <= bits && bits <= 64); + + a.add(bin_byte_ptr, bin_base, bin_offset, arm::lsr(3)); + + if (bits <= 8) { + a.ands(bit_offset, bin_offset, imm(7)); + + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, + * this segment always spans two bytes. */ + a.b_ne(handle_partial); + } else if (num_partial > 1) { + /* The segment is smaller than one byte but more than one + * bit. Test whether it fits within the current byte. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_gt(handle_partial); + } + + /* The segment fits in the current byte. */ + a.ldrb(bitdata.w(), arm::Mem(bin_byte_ptr)); + if (num_partial == 0) { + a.rev64(bitdata, bitdata); + a.b(read_done); + } else if (num_partial > 1) { + a.b(rev64); + } + + /* The segment is unaligned and spans two bytes. */ + a.bind(handle_partial); + if (num_partial != 1) { + a.ldrh(bitdata.w(), arm::Mem(bin_byte_ptr)); + } + need_rev64 = true; + } else if (bits <= 16) { + a.ands(bit_offset, bin_offset, imm(7)); + + /* We always need to read at least two bytes. */ + a.ldrh(bitdata.w(), arm::Mem(bin_byte_ptr)); + a.rev64(bitdata, bitdata); + a.b_eq(read_done); /* Done if segment is byte-aligned. */ + + /* The segment is unaligned. If its size is 9, it always fits + * in two bytes and we fall through to the shift instruction. */ + a.bind(handle_partial); + if (num_partial > 1) { + /* If segment size is less than 15 bits or less, it is + * possible that it fits into two bytes. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_le(shift); + } + + if (num_partial != 1) { + /* The segment spans three bytes. Read an additional byte and + * shift into place (right below the already read two bytes a + * the top of the word). */ + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 2)); + a.orr(bitdata, bitdata, tmp, arm::lsl(40)); + } + } else if (bits <= 24) { + a.ands(bit_offset, bin_offset, imm(7)); + + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, + * this segment always spans four bytes. */ + a.b_ne(handle_partial); + } else if (num_partial > 1) { + /* The segment is smaller than three bytes. Test whether + * it spans three or four bytes. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_gt(handle_partial); + } + + /* This segment spans three bytes. */ + a.ldrh(bitdata.w(), arm::Mem(bin_byte_ptr)); + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 2)); + a.orr(bitdata, bitdata, tmp, arm::lsl(16)); + if (num_partial == 0) { + a.rev64(bitdata, bitdata); + a.b(read_done); + } else if (num_partial > 1) { + a.b(rev64); + } + + /* This segment spans four bytes. */ + a.bind(handle_partial); + if (num_partial != 1) { + a.ldr(bitdata.w(), arm::Mem(bin_byte_ptr)); + } + need_rev64 = true; + } else if (bits <= 32) { + a.ands(bit_offset, bin_offset, imm(7)); + + /* We always need to read at least four bytes. */ + a.ldr(bitdata.w(), arm::Mem(bin_byte_ptr)); + a.rev64(bitdata, bitdata); + a.b_eq(read_done); + + a.bind(handle_partial); + if (num_partial > 0) { + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_le(shift); + } + + if (num_partial != 1) { + /* The segment spans five bytes. Read an additional byte and + * shift into place. */ + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 4)); + a.orr(bitdata, bitdata, tmp, arm::lsl(24)); + } + } else if (bits <= 40) { + a.ands(bit_offset, bin_offset, imm(7)); + + /* We always need to read four bytes. */ + a.ldr(bitdata.w(), arm::Mem(bin_byte_ptr)); + a.rev64(bitdata, bitdata); + + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, + * this segment always spans six bytes. */ + a.b_ne(handle_partial); + } else if (num_partial > 1) { + /* The segment is smaller than five bytes. Test whether it + * spans five or six bytes. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_gt(handle_partial); + } + + /* This segment spans five bytes. Read an additional byte. */ + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 4)); + a.orr(bitdata, bitdata, tmp, arm::lsl(24)); + if (num_partial == 0) { + a.b(read_done); + } else if (num_partial > 1) { + a.b(shift); + } + + a.bind(handle_partial); + if (num_partial != 1) { + /* This segment spans six bytes. Read two additional bytes. */ + a.ldrh(tmp.w(), arm::Mem(bin_byte_ptr, 4)); + a.rev16(tmp.w(), tmp.w()); + a.orr(bitdata, bitdata, tmp, arm::lsl(16)); + } + } else if (bits <= 48) { + a.ands(bit_offset, bin_offset, imm(7)); + a.ldr(bitdata.w(), arm::Mem(bin_byte_ptr)); + a.ldrh(tmp.w(), arm::Mem(bin_byte_ptr, 4)); + a.orr(bitdata, bitdata, tmp, arm::lsl(32)); + a.rev64(bitdata, bitdata); + a.b_eq(read_done); + + a.bind(handle_partial); + if (num_partial > 1) { + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_le(shift); + } + + if (num_partial != 1) { + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 6)); + a.orr(bitdata, bitdata, tmp, arm::lsl(8)); + } + } else if (bits <= 56) { + a.ands(bit_offset, bin_offset, imm(7)); + + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, + * this segment always spans 8 bytes. */ + a.b_ne(handle_partial); + } else if (num_partial > 1) { + /* The segment is smaller than 8 bytes. Test whether it + * spans 7 or 8 bytes. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_gt(handle_partial); + } + + /* This segment spans 7 bytes. */ + a.ldr(bitdata, arm::Mem(bin_byte_ptr, -1)); + a.lsr(bitdata, bitdata, imm(8)); + a.b(rev64); + + /* This segment spans 8 bytes. */ + a.bind(handle_partial); + if (num_partial != 1) { + a.ldr(bitdata, arm::Mem(bin_byte_ptr)); + } + need_rev64 = true; + } else if (bits <= 64) { + a.ands(bit_offset, bin_offset, imm(7)); + a.ldr(bitdata, arm::Mem(bin_byte_ptr)); + a.rev64(bitdata, bitdata); + + if (num_partial == 0) { + /* Byte-sized segment. If it is aligned it spans 8 bytes + * and we are done. */ + a.b_eq(read_done); + } else if (num_partial == 1) { + /* This segment is 57 bits wide. It always spans 8 bytes. */ + a.b(shift); + } else { + /* The segment is smaller than 8 bytes. Test whether it + * spans 8 or 9 bytes. */ + a.cmp(bit_offset, imm(8 - num_partial)); + a.b_le(shift); + } + + /* This segments spans 9 bytes. Read an additional byte. */ + a.bind(handle_partial); + if (num_partial != 1) { + a.ldrb(tmp.w(), arm::Mem(bin_byte_ptr, 8)); + a.lsl(bitdata, bitdata, bit_offset); + a.lsl(tmp, tmp, bit_offset); + a.orr(bitdata, bitdata, tmp, arm::lsr(8)); + a.b(read_done); + } + } + + a.bind(rev64); + if (need_rev64) { + a.rev64(bitdata, bitdata); + } + + /* Shift the read data into the most significant bits of the + * word. */ + a.bind(shift); + a.lsl(bitdata, bitdata, bit_offset); + + a.bind(read_done); +} + +void BeamModuleAssembler::emit_extract_integer(const arm::Gp bitdata, + Uint flags, + Uint bits, + const ArgRegister &Dst) { + Label big = a.newLabel(); + Label done = a.newLabel(); + arm::Gp data_reg; + auto dst = init_destination(Dst, TMP1); + Uint num_partial = bits % 8; + Uint num_complete = 8 * (bits / 8); + + if (bits <= 8) { + /* Endian does not matter for values that fit in a byte. */ + flags &= ~BSF_LITTLE; + } + + /* If this segment is little-endian, reverse endianness. */ + if ((flags & BSF_LITTLE) != 0) { + comment("reverse endian for a little-endian segment"); + } + data_reg = TMP2; + if ((flags & BSF_LITTLE) == 0) { + data_reg = bitdata; + } else if (bits == 16) { + a.rev16(TMP2, bitdata); + } else if (bits == 32) { + a.rev32(TMP2, bitdata); + } else if (num_partial == 0) { + a.rev64(TMP2, bitdata); + a.lsr(TMP2, TMP2, arm::lsr(64 - bits)); + } else { + a.ubfiz(TMP3, bitdata, imm(num_complete), imm(num_partial)); + a.ubfx(TMP2, bitdata, imm(num_partial), imm(num_complete)); + a.rev64(TMP2, TMP2); + a.orr(TMP2, TMP3, TMP2, arm::lsr(64 - num_complete)); + } + + /* Sign-extend the number if the segment is signed. */ + if ((flags & BSF_SIGNED) != 0) { + if (0 < bits && bits < 64) { + comment("sign extend extracted value"); + a.lsl(TMP2, data_reg, imm(64 - bits)); + a.asr(TMP2, TMP2, imm(64 - bits)); + data_reg = TMP2; + } + } + + /* Handle segments whose values might not fit in a small integer. */ + if (bits >= SMALL_BITS) { + comment("test whether it fits in a small"); + if (bits < 64 && (flags & BSF_SIGNED) == 0) { + a.and_(TMP2, data_reg, imm((1ull << bits) - 1)); + data_reg = TMP2; + } + if ((flags & BSF_SIGNED) != 0) { + /* Signed segment. */ + a.adds(TMP3, ZERO, data_reg, arm::lsr(SMALL_BITS - 1)); + a.ccmp(TMP3, + imm(_TAG_IMMED1_MASK << 1 | 1), + imm(NZCV::kEqual), + imm(arm::CondCode::kNE)); + a.b_ne(big); + } else { + /* Unsigned segment. */ + a.lsr(TMP3, data_reg, imm(SMALL_BITS - 1)); + a.cbnz(TMP3, big); + } + } + + /* Tag and store the extracted small integer. */ + comment("store extracted integer as a small"); + mov_imm(dst.reg, _TAG_IMMED1_SMALL); + if ((flags & BSF_SIGNED) != 0) { + a.orr(dst.reg, dst.reg, data_reg, arm::lsl(_TAG_IMMED1_SIZE)); + } else { + if (bits >= SMALL_BITS) { + a.bfi(dst.reg, + data_reg, + arm::lsl(_TAG_IMMED1_SIZE), + imm(SMALL_BITS)); + } else if (bits != 0) { + a.bfi(dst.reg, data_reg, arm::lsl(_TAG_IMMED1_SIZE), imm(bits)); + } + } + + if (bits >= SMALL_BITS) { + a.b(done); + } + + /* Handle a bignum (up to 64 bits). */ + a.bind(big); + if (bits >= SMALL_BITS) { + comment("store extracted integer as a bignum"); + a.add(dst.reg, HTOP, imm(TAG_PRIMARY_BOXED)); + mov_imm(TMP3, make_pos_bignum_header(1)); + if ((flags & BSF_SIGNED) == 0) { + /* Unsigned. */ + a.stp(TMP3, data_reg, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } else { + /* Signed. */ + Label store = a.newLabel(); + a.adds(TMP2, data_reg, ZERO); + a.b_pl(store); + + mov_imm(TMP3, make_neg_bignum_header(1)); + a.neg(TMP2, TMP2); + + a.bind(store); + a.stp(TMP3, TMP2, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } + } + + a.bind(done); + flush_var(dst); +} + +void BeamModuleAssembler::emit_extract_binary(const arm::Gp bitdata, + Uint bits, + const ArgRegister &Dst) { + auto dst = init_destination(Dst, TMP1); + Uint num_bytes = bits / 8; + + a.add(dst.reg, HTOP, imm(TAG_PRIMARY_BOXED)); + mov_imm(TMP2, header_heap_bin(num_bytes)); + mov_imm(TMP3, num_bytes); + a.rev64(TMP4, bitdata); + a.stp(TMP2, TMP3, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + a.str(TMP4, arm::Mem(HTOP).post(sizeof(Eterm[1]))); + flush_var(dst); +} + +static std::vector<BsmSegment> opt_bsm_segments( + const std::vector<BsmSegment> segments, + const ArgWord &Need, + const ArgWord &Live) { + std::vector<BsmSegment> segs; + + Uint heap_need = Need.get(); + + /* + * First calculate the total number of heap words needed for + * bignums and binaries. + */ + for (auto seg : segments) { + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + if (seg.size >= SMALL_BITS) { + heap_need += BIG_NEED_FOR_BITS(seg.size); + } + break; + case BsmSegment::action::GET_BINARY: + heap_need += heap_bin_size((seg.size + 7) / 8); + break; + case BsmSegment::action::GET_TAIL: + heap_need += EXTRACT_SUB_BIN_HEAP_NEED; + break; + default: + break; + } + } + + int index = 0; + int read_action_pos = -1; + + index = 0; + for (auto seg : segments) { + if (heap_need != 0 && seg.live.isWord()) { + BsmSegment s = seg; + + read_action_pos = -1; + s.action = BsmSegment::action::TEST_HEAP; + s.size = heap_need; + segs.push_back(s); + index++; + heap_need = 0; + } + + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + case BsmSegment::action::GET_BINARY: + if (seg.size > 64) { + read_action_pos = -1; + } else if (seg.action == BsmSegment::action::GET_BINARY && + seg.size % 8 != 0) { + read_action_pos = -1; + } else { + if ((seg.flags & BSF_LITTLE) != 0 || read_action_pos < 0 || + seg.size + segs.at(read_action_pos).size > 64) { + BsmSegment s; + + /* Create a new READ action. */ + read_action_pos = index; + s.action = BsmSegment::action::READ; + s.size = seg.size; + segs.push_back(s); + index++; + } else { + /* Reuse previous READ action. */ + segs.at(read_action_pos).size += seg.size; + } + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + seg.action = BsmSegment::action::EXTRACT_INTEGER; + break; + case BsmSegment::action::GET_BINARY: + seg.action = BsmSegment::action::EXTRACT_BINARY; + break; + default: + break; + } + } + segs.push_back(seg); + break; + case BsmSegment::action::EQ: { + if (read_action_pos < 0 || + seg.size + segs.at(read_action_pos).size > 64) { + BsmSegment s; + + /* Create a new READ action. */ + read_action_pos = index; + s.action = BsmSegment::action::READ; + s.size = seg.size; + segs.push_back(s); + index++; + } else { + /* Reuse previous READ action. */ + segs.at(read_action_pos).size += seg.size; + } + auto &prev = segs.back(); + if (prev.action == BsmSegment::action::EQ && + prev.size + seg.size <= 64) { + /* Coalesce with the previous EQ instruction. */ + prev.size += seg.size; + prev.unit = prev.unit << seg.size | seg.unit; + index--; + } else { + segs.push_back(seg); + } + break; + } + case BsmSegment::action::SKIP: + if (read_action_pos >= 0 && + seg.size + segs.at(read_action_pos).size <= 64) { + segs.at(read_action_pos).size += seg.size; + seg.action = BsmSegment::action::DROP; + } else { + read_action_pos = -1; + } + segs.push_back(seg); + break; + default: + read_action_pos = -1; + segs.push_back(seg); + break; + } + index++; + } + + /* Handle a trailing test_heap instruction (for the + * i_bs_match_test_heap instruction). */ + if (heap_need) { + BsmSegment seg; + + seg.action = BsmSegment::action::TEST_HEAP; + seg.size = heap_need; + seg.live = Live; + segs.push_back(seg); + } + return segs; +} + +UWord BeamModuleAssembler::bs_get_flags(const ArgVal &val) { + if (val.isNil()) { + return 0; + } else if (val.isLiteral()) { + Eterm term = beamfile_get_literal(beam, val.as<ArgLiteral>().get()); + UWord flags = 0; + + while (is_list(term)) { + Eterm *consp = list_val(term); + Eterm elem = CAR(consp); + switch (elem) { + case am_little: + case am_native: + flags |= BSF_LITTLE; + break; + case am_signed: + flags |= BSF_SIGNED; + break; + } + term = CDR(consp); + } + ASSERT(is_nil(term)); + return flags; + } else if (val.isWord()) { + /* Originates from bs_get_integer2 instruction. */ + return val.as<ArgWord>().get(); + } else { + ASSERT(0); /* Should not happen. */ + return 0; + } +} + +void BeamModuleAssembler::emit_i_bs_match(ArgLabel const &Fail, + ArgRegister const &Ctx, + Span<ArgVal> const &List) { + emit_i_bs_match_test_heap(Fail, Ctx, ArgWord(0), ArgWord(0), List); +} + +void BeamModuleAssembler::emit_i_bs_match_test_heap(ArgLabel const &Fail, + ArgRegister const &Ctx, + ArgWord const &Need, + ArgWord const &Live, + Span<ArgVal> const &List) { + const int orig_offset = offsetof(ErlBinMatchState, mb.orig); + const int base_offset = offsetof(ErlBinMatchState, mb.base); + const int position_offset = offsetof(ErlBinMatchState, mb.offset); + const int size_offset = offsetof(ErlBinMatchState, mb.size); + + std::vector<BsmSegment> segments; + + auto current = List.begin(); + auto end = List.begin() + List.size(); + + while (current < end) { + auto cmd = current++->as<ArgImmed>().get(); + BsmSegment seg; + + switch (cmd) { + case am_ensure_at_least: { + seg.action = BsmSegment::action::ENSURE_AT_LEAST; + seg.size = current[0].as<ArgWord>().get(); + seg.unit = current[1].as<ArgWord>().get(); + current += 2; + break; + } + case am_ensure_exactly: { + seg.action = BsmSegment::action::ENSURE_EXACTLY; + seg.size = current[0].as<ArgWord>().get(); + current += 1; + break; + } + case am_binary: + case am_integer: { + auto size = current[2].as<ArgWord>().get(); + auto unit = current[3].as<ArgWord>().get(); + + switch (cmd) { + case am_integer: + seg.action = BsmSegment::action::GET_INTEGER; + break; + case am_binary: + seg.action = BsmSegment::action::GET_BINARY; + break; + } + + seg.live = current[0]; + seg.size = size * unit; + seg.unit = unit; + seg.flags = bs_get_flags(current[1]); + seg.dst = current[4].as<ArgRegister>(); + current += 5; + break; + } + case am_get_tail: { + seg.action = BsmSegment::action::GET_TAIL; + seg.live = current[0].as<ArgWord>(); + seg.dst = current[2].as<ArgRegister>(); + current += 3; + break; + } + case am_skip: { + seg.action = BsmSegment::action::SKIP; + seg.size = current[0].as<ArgWord>().get(); + seg.flags = 0; + current += 1; + break; + } + case am_Eq: { + seg.action = BsmSegment::action::EQ; + seg.live = current[0]; + seg.size = current[1].as<ArgWord>().get(); + seg.unit = current[2].as<ArgWord>().get(); + current += 3; + break; + } + default: + abort(); + break; + } + segments.push_back(seg); + } + + segments = opt_bsm_segments(segments, Need, Live); + + const arm::Gp bin_base = ARG2; + const arm::Gp bin_position = ARG3; + const arm::Gp bin_size = ARG4; + const arm::Gp bitdata = ARG8; + bool position_is_valid = false; + + for (auto seg : segments) { + switch (seg.action) { + case BsmSegment::action::ENSURE_AT_LEAST: { + comment("ensure_at_least %ld %ld", seg.size, seg.unit); + auto ctx_reg = load_source(Ctx, TMP1); + auto stride = seg.size; + auto unit = seg.unit; + + a.ldur(bin_position, emit_boxed_val(ctx_reg.reg, position_offset)); + a.ldur(bin_size, emit_boxed_val(ctx_reg.reg, size_offset)); + a.sub(TMP5, bin_size, bin_position); + cmp(TMP5, stride); + a.b_lo(resolve_beam_label(Fail, disp1MB)); + + if (unit != 1) { + if (stride % unit != 0) { + sub(TMP5, TMP5, stride); + } + + if ((unit & (unit - 1)) != 0) { + mov_imm(TMP4, unit); + + a.udiv(TMP3, TMP5, TMP4); + a.msub(TMP5, TMP3, TMP4, TMP5); + + a.cbnz(TMP5, resolve_beam_label(Fail, disp1MB)); + } else { + a.tst(TMP5, imm(unit - 1)); + a.b_ne(resolve_beam_label(Fail, disp1MB)); + } + } + + position_is_valid = true; + break; + } + case BsmSegment::action::ENSURE_EXACTLY: { + comment("ensure_exactly %ld", seg.size); + auto ctx_reg = load_source(Ctx, TMP1); + auto size = seg.size; + + a.ldur(bin_position, emit_boxed_val(ctx_reg.reg, position_offset)); + a.ldur(TMP3, emit_boxed_val(ctx_reg.reg, size_offset)); + if (size != 0) { + a.sub(TMP1, TMP3, bin_position); + cmp(TMP1, size); + } else { + a.subs(TMP1, TMP3, bin_position); + } + a.b_ne(resolve_beam_label(Fail, disp1MB)); + position_is_valid = true; + break; + } + case BsmSegment::action::EQ: { + comment("=:= %ld %ld", seg.size, seg.unit); + if (seg.size != 0 && seg.size != 64) { + a.ror(bitdata, bitdata, imm(64 - seg.size)); + } + if (seg.size == 64) { + cmp(bitdata, seg.unit); + } else if (seg.size == 32) { + cmp(bitdata.w(), seg.unit); + } else if (seg.unit == 0) { + a.tst(bitdata, imm((1ull << seg.size) - 1)); + } else { + a.and_(TMP1, bitdata, imm((1ull << seg.size) - 1)); + cmp(TMP1, seg.unit); + } + a.b_ne(resolve_beam_label(Fail, disp1MB)); + break; + } + case BsmSegment::action::TEST_HEAP: { + comment("test_heap %ld", seg.size); + emit_gc_test(ArgWord(0), ArgWord(seg.size), seg.live); + position_is_valid = false; + break; + } + case BsmSegment::action::READ: { + comment("read %ld", seg.size); + if (seg.size == 0) { + comment("(nothing to do)"); + } else { + auto ctx = load_source(Ctx, ARG1); + + if (!position_is_valid) { + a.ldur(bin_position, + emit_boxed_val(ctx.reg, position_offset)); + position_is_valid = true; + } + a.ldur(bin_base, emit_boxed_val(ctx.reg, base_offset)); + + emit_read_bits(seg.size, bin_base, bin_position, bitdata); + + a.add(bin_position, bin_position, imm(seg.size)); + a.stur(bin_position, emit_boxed_val(ctx.reg, position_offset)); + } + break; + } + case BsmSegment::action::EXTRACT_BINARY: { + auto bits = seg.size; + auto Dst = seg.dst; + + comment("extract binary %ld", bits); + emit_extract_binary(bitdata, bits, Dst); + if (bits != 0 && bits != 64) { + a.ror(bitdata, bitdata, imm(64 - bits)); + } + break; + } + case BsmSegment::action::EXTRACT_INTEGER: { + auto bits = seg.size; + auto flags = seg.flags; + auto Dst = seg.dst; + + comment("extract integer %ld", bits); + if (bits != 0 && bits != 64) { + a.ror(bitdata, bitdata, imm(64 - bits)); + } + emit_extract_integer(bitdata, flags, bits, Dst); + break; + } + case BsmSegment::action::GET_INTEGER: { + Uint live = seg.live.as<ArgWord>().get(); + Uint flags = seg.flags; + auto bits = seg.size; + auto Dst = seg.dst; + + comment("get integer %ld", bits); + auto ctx = load_source(Ctx, TMP1); + + a.mov(ARG1, c_p); + a.mov(ARG2, bits); + a.mov(ARG3, flags); + lea(ARG4, emit_boxed_val(ctx.reg, offsetof(ErlBinMatchState, mb))); + + if (bits >= SMALL_BITS) { + emit_enter_runtime<Update::eHeap>(live); + } else { + emit_enter_runtime(live); + } + + runtime_call<4>(erts_bs_get_integer_2); + + if (bits >= SMALL_BITS) { + emit_leave_runtime<Update::eHeap>(live); + } else { + emit_leave_runtime(live); + } + + mov_arg(Dst, ARG1); + + position_is_valid = false; + break; + } + case BsmSegment::action::GET_BINARY: { + auto Live = seg.live; + comment("get binary %ld", seg.size); + auto ctx = load_source(Ctx, TMP1); + + lea(ARG1, arm::Mem(c_p, offsetof(Process, htop))); + a.ldur(ARG2, emit_boxed_val(ctx.reg, orig_offset)); + a.ldur(ARG3, emit_boxed_val(ctx.reg, base_offset)); + a.ldur(ARG4, emit_boxed_val(ctx.reg, position_offset)); + mov_imm(ARG5, seg.size); + a.add(TMP2, ARG4, ARG5); + a.stur(TMP2, emit_boxed_val(ctx.reg, position_offset)); + + emit_enter_runtime<Update::eHeap>(Live.as<ArgWord>().get()); + + runtime_call<5>(erts_extract_sub_binary); + + emit_leave_runtime<Update::eHeap>(Live.as<ArgWord>().get()); + + mov_arg(seg.dst, ARG1); + position_is_valid = false; + break; + } + case BsmSegment::action::GET_TAIL: { + comment("get_tail"); + + mov_arg(ARG1, Ctx); + fragment_call(ga->get_bs_get_tail_shared()); + mov_arg(seg.dst, ARG1); + position_is_valid = false; + break; + } + case BsmSegment::action::SKIP: { + comment("skip %ld", seg.size); + auto ctx = load_source(Ctx, TMP1); + if (!position_is_valid) { + a.ldur(bin_position, emit_boxed_val(ctx.reg, position_offset)); + position_is_valid = true; + } + add(bin_position, bin_position, seg.size); + a.stur(bin_position, emit_boxed_val(ctx.reg, position_offset)); + break; + } + case BsmSegment::action::DROP: + auto bits = seg.size; + comment("drop %ld", bits); + if (bits != 0 && bits != 64) { + a.ror(bitdata, bitdata, imm(64 - bits)); + } + break; + } + } +} diff --git a/erts/emulator/beam/jit/arm/instr_common.cpp b/erts/emulator/beam/jit/arm/instr_common.cpp index ea67b2c18e..639545ec68 100644 --- a/erts/emulator/beam/jit/arm/instr_common.cpp +++ b/erts/emulator/beam/jit/arm/instr_common.cpp @@ -52,6 +52,7 @@ */ #include <algorithm> +#include <numeric> #include "beam_asm.hpp" extern "C" @@ -61,6 +62,7 @@ extern "C" #include "beam_catches.h" #include "beam_common.h" #include "code_ix.h" +#include "erl_binary.h" } using namespace asmjit; @@ -75,21 +77,47 @@ void BeamModuleAssembler::emit_error(int reason) { void BeamModuleAssembler::emit_gc_test_preserve(const ArgWord &Need, const ArgWord &Live, - arm::Gp term) { + const ArgSource &Preserve, + arm::Gp preserve_reg) { const int32_t bytes_needed = (Need.get() + S_RESERVED) * sizeof(Eterm); Label after_gc_check = a.newLabel(); +#ifdef DEBUG + comment("(debug: fill dead X registers with garbage)"); + const arm::Gp garbage_reg = preserve_reg == ARG4 ? ARG3 : ARG4; + mov_imm(garbage_reg, ERTS_HOLE_MARKER); + if (!(Preserve.isXRegister() && + Preserve.as<ArgXRegister>().get() >= Live.get())) { + mov_arg(ArgXRegister(Live.get()), garbage_reg); + mov_arg(ArgXRegister(Live.get() + 1), garbage_reg); + } else { + mov_imm(garbage_reg, ERTS_HOLE_MARKER); + mov_arg(ArgXRegister(Live.get() + 1), garbage_reg); + mov_arg(ArgXRegister(Live.get() + 2), garbage_reg); + } +#endif + add(ARG3, HTOP, bytes_needed); a.cmp(ARG3, E); a.b_ls(after_gc_check); ASSERT(Live.get() < ERTS_X_REGS_ALLOCATED); - mov_arg(ArgVal(ArgVal::XReg, Live.get()), term); - mov_imm(ARG4, Live.get() + 1); - fragment_call(ga->get_garbage_collect()); + /* We don't need to stash the preserved term if it's currently live, making + * the code slightly shorter. */ + if (!(Preserve.isXRegister() && + Preserve.as<ArgXRegister>().get() >= Live.get())) { + mov_imm(ARG4, Live.get()); + fragment_call(ga->get_garbage_collect()); + mov_arg(preserve_reg, Preserve); + } else { + mov_arg(ArgXRegister(Live.get()), preserve_reg); - mov_arg(term, ArgVal(ArgVal::XReg, Live.get())); + mov_imm(ARG4, Live.get() + 1); + fragment_call(ga->get_garbage_collect()); + + mov_arg(preserve_reg, ArgXRegister(Live.get())); + } a.bind(after_gc_check); } @@ -100,6 +128,13 @@ void BeamModuleAssembler::emit_gc_test(const ArgWord &Ns, int32_t bytes_needed = (Ns.get() + Nh.get() + S_RESERVED) * sizeof(Eterm); Label after_gc_check = a.newLabel(); +#ifdef DEBUG + comment("(debug: fill dead X registers with garbage)"); + mov_imm(ARG4, ERTS_HOLE_MARKER); + mov_arg(ArgXRegister(Live.get()), ARG4); + mov_arg(ArgXRegister(Live.get() + 1), ARG4); +#endif + add(ARG3, HTOP, bytes_needed); a.cmp(ARG3, E); a.b_ls(after_gc_check); @@ -423,7 +458,10 @@ void BeamModuleAssembler::emit_store_two_xregs(const ArgXRegister &Src1, const ArgXRegister &Src2, const ArgYRegister &Dst2) { auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2); - safe_stp(src1.reg, src2.reg, Dst1, Dst2); + auto dst1 = init_destination(Dst1, src1.reg); + auto dst2 = init_destination(Dst2, src2.reg); + + flush_vars(dst1, dst2); } void BeamModuleAssembler::emit_load_two_xregs(const ArgYRegister &Src1, @@ -471,22 +509,11 @@ void BeamModuleAssembler::emit_swap(const ArgRegister &R1, } else if (isRegisterBacked(R2)) { return emit_swap(R2, R1); } else { - switch (ArgVal::memory_relation(R1, R2)) { - case ArgVal::Relation::consecutive: - safe_ldp(TMP1, TMP2, R1, R2); - safe_stp(TMP2, TMP1, R1, R2); - break; - case ArgVal::Relation::reverse_consecutive: - safe_ldp(TMP1, TMP2, R2, R1); - safe_stp(TMP2, TMP1, R2, R1); - break; - case ArgVal::Relation::none: - a.ldr(TMP1, getArgRef(R1)); - a.ldr(TMP2, getArgRef(R2)); - a.str(TMP1, getArgRef(R2)); - a.str(TMP2, getArgRef(R1)); - break; - } + /* Both BEAM registers are stored in memory. */ + auto [r1, r2] = load_sources(R1, TMP1, R2, TMP2); + auto dst1 = init_destination(R2, r1.reg); + auto dst2 = init_destination(R1, r2.reg); + flush_vars(dst1, dst2); } } @@ -629,6 +656,157 @@ void BeamModuleAssembler::emit_self(const ArgRegister &Dst) { mov_arg(Dst, arm::Mem(c_p, offsetof(Process, common.id))); } +void BeamModuleAssembler::emit_copy_words_increment(arm::Gp from, + arm::Gp to, + size_t count) { + check_pending_stubs(); + + /* Copy the words inline if we can, otherwise use a loop with the largest + * vector size we're capable of. */ + if (count <= 16) { + while (count >= 4) { + a.ldp(a64::q30, a64::q31, arm::Mem(from).post(sizeof(UWord[4]))); + a.stp(a64::q30, a64::q31, arm::Mem(to).post(sizeof(UWord[4]))); + count -= 4; + } + } else { + Label copy_next = a.newLabel(); + + ASSERT(Support::isUInt16(count / 4)); + mov_imm(SUPER_TMP, count / 4); + a.bind(copy_next); + { + a.ldp(a64::q30, a64::q31, arm::Mem(from).post(sizeof(UWord[4]))); + a.stp(a64::q30, a64::q31, arm::Mem(to).post(sizeof(UWord[4]))); + a.subs(SUPER_TMP, SUPER_TMP, imm(1)); + a.b_ne(copy_next); + } + + count = count % 4; + } + + if (count >= 2) { + a.ldr(a64::q30, arm::Mem(from).post(sizeof(UWord[2]))); + a.str(a64::q30, arm::Mem(to).post(sizeof(UWord[2]))); + count -= 2; + } + + if (count == 1) { + a.ldr(SUPER_TMP, arm::Mem(from).post(sizeof(UWord))); + a.str(SUPER_TMP, arm::Mem(to).post(sizeof(UWord))); + count -= 1; + } + + ASSERT(count == 0); + (void)count; +} + +void BeamModuleAssembler::emit_update_record(const ArgAtom &Hint, + const ArgWord &TupleSize, + const ArgSource &Src, + const ArgRegister &Dst, + const ArgWord &UpdateCount, + const Span<ArgVal> &updates) { + const size_t size_on_heap = TupleSize.get() + 1; + Label next = a.newLabel(); + + ASSERT(UpdateCount.get() == updates.size()); + ASSERT((UpdateCount.get() % 2) == 0); + + ASSERT(size_on_heap > 2); + + auto destination = init_destination(Dst, ARG1); + auto src = load_source(Src, ARG2); + + arm::Gp untagged_src = ARG3; + emit_untag_ptr(untagged_src, src.reg); + + /* Setting a field to the same value is pretty common, so we'll check for + * that since it's vastly cheaper than copying if we're right, and doesn't + * cost much if we're wrong. */ + if (Hint.get() == am_reuse && updates.size() == 2) { + const auto next_index = updates[0].as<ArgWord>().get(); + const auto &next_value = updates[1].as<ArgSource>(); + + safe_ldr(TMP1, arm::Mem(untagged_src, next_index * sizeof(Eterm))); + cmp_arg(TMP1, next_value); + + if (destination.reg != src.reg) { + a.csel(destination.reg, + destination.reg, + src.reg, + imm(arm::CondCode::kNE)); + } + a.b_eq(next); + } + + size_t copy_index = 0; + + for (size_t i = 0; i < updates.size(); i += 2) { + const auto next_index = updates[i].as<ArgWord>().get(); + const auto &next_value = updates[i + 1].as<ArgSource>(); + bool odd_copy; + + ASSERT(next_index > 0 && next_index >= copy_index); + + /* If we need to copy an odd number of elements, we'll do the last one + * ourselves to save us from having to increment `untagged_src` + * separately. */ + odd_copy = (next_index - copy_index) & 1; + emit_copy_words_increment(untagged_src, + HTOP, + (next_index - copy_index) & ~1); + + if ((i + 2) < updates.size()) { + const auto adjacent_index = updates[i + 2].as<ArgWord>().get(); + const auto &adjacent_value = updates[i + 3].as<ArgSource>(); + + if (adjacent_index == next_index + 1) { + auto [first, second] = + load_sources(next_value, TMP1, adjacent_value, TMP2); + + if (odd_copy) { + a.ldr(TMP3, arm::Mem(untagged_src).post(sizeof(Eterm[3]))); + a.stp(TMP3, + first.reg, + arm::Mem(HTOP).post(sizeof(Eterm[2]))); + a.str(second.reg, arm::Mem(HTOP).post(sizeof(Eterm))); + } else { + a.add(untagged_src, untagged_src, imm(sizeof(Eterm[2]))); + a.stp(first.reg, + second.reg, + arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } + + copy_index = next_index + 2; + i += 2; + continue; + } + } + + auto value = load_source(next_value, TMP1); + + if ((next_index - copy_index) & 1) { + a.ldr(TMP2, arm::Mem(untagged_src).post(sizeof(Eterm[2]))); + a.stp(TMP2, value.reg, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } else { + a.add(untagged_src, untagged_src, imm(sizeof(Eterm))); + a.str(value.reg, arm::Mem(HTOP).post(sizeof(Eterm))); + } + + copy_index = next_index + 1; + } + + emit_copy_words_increment(untagged_src, HTOP, size_on_heap - copy_index); + + sub(destination.reg, + HTOP, + (size_on_heap * sizeof(Eterm)) - TAG_PRIMARY_BOXED); + + a.bind(next); + flush_var(destination); +} + void BeamModuleAssembler::emit_set_tuple_element(const ArgSource &Element, const ArgRegister &Tuple, const ArgWord &Offset) { @@ -675,60 +853,71 @@ void BeamModuleAssembler::emit_is_boolean(const ArgLabel &Fail, a.b_ne(resolve_beam_label(Fail, disp1MB)); } -arm::Gp BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src, - Label next, - Label subbin) { +void BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, + const ArgSource &Src) { + Label is_binary = a.newLabel(), next = a.newLabel(); + auto src = load_source(Src, ARG1); emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); - a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN)); - a.b_eq(subbin); - if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_BITSTRING) { + const int bit_number = 3; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & (1 << bit_number)) != 0 && + (_TAG_HEADER_REFC_BIN & (1 << bit_number)) == 0 && + (_TAG_HEADER_HEAP_BIN & (1 << bit_number)) == 0); comment("simplified binary test since source is always a bitstring " "when boxed"); + a.tbz(TMP1, imm(bit_number), next); } else { + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN)); + a.b_ne(is_binary); + } + + /* This is a sub binary. */ + a.ldrb(TMP1.w(), emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize))); + a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) != BEAM_TYPE_BITSTRING) { + a.b(next); + } + + a.bind(is_binary); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) != BEAM_TYPE_BITSTRING) { ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); a.and_(TMP1, TMP1, imm(~4)); a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN)); a.b_ne(resolve_beam_label(Fail, disp1MB)); } - a.b(next); - - return boxed_ptr; -} - -void BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src) { - Label next = a.newLabel(), subbin = a.newLabel(); - - arm::Gp boxed_ptr = emit_is_binary(Fail, Src, next, subbin); - - a.bind(subbin); - { - /* emit_is_binary() has already removed the literal tag (if - * applicable) from the copy of Src. */ - a.ldrb(TMP1.w(), - emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize))); - a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB)); - } - a.bind(next); } void BeamModuleAssembler::emit_is_bitstring(const ArgLabel &Fail, const ArgSource &Src) { - Label next = a.newLabel(); + auto src = load_source(Src, ARG1); - (void)emit_is_binary(Fail, Src, next, next); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); - a.bind(next); + arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + + /* The header mask with the binary sub tag bits removed (0b110011) + * is not possible to use as an immediate operand for 'and'. (See + * the note at the beginning of the file.) Therefore, use a + * simpler mask (0b110000) that will also clear the primary tag + * bits. That works because we KNOW that a boxed pointer always + * points to a header word and that the primary tag for a header + * is 0. + */ + const auto mask = _HEADER_SUBTAG_MASK - _BINARY_XXX_MASK; + ERTS_CT_ASSERT(TAG_PRIMARY_HEADER == 0); + ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN == (_TAG_HEADER_REFC_BIN & mask)); + a.and_(TMP1, TMP1, imm(mask)); + a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN)); + a.b_ne(resolve_beam_label(Fail, disp1MB)); } void BeamModuleAssembler::emit_is_float(const ArgLabel &Fail, @@ -846,12 +1035,17 @@ void BeamModuleAssembler::emit_is_integer(const ArgLabel &Fail, arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - /* The following value (0b111011) is not possible to use as - * an immediate operand for 'and'. See the note at the beginning - * of the file. + /* The header mask with the sign bit removed (0b111011) is not + * possible to use as an immediate operand for 'and'. (See the + * note at the beginning of the file.) Therefore, use a + * simpler mask (0b111000) that will also clear the primary + * tag bits. That works because we KNOW that a boxed pointer + * always points to a header word and that the primary tag for + * a header is 0. */ - mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); - a.and_(TMP1, TMP1, TMP2); + auto mask = _HEADER_SUBTAG_MASK - _BIG_SIGN_BIT; + ERTS_CT_ASSERT(TAG_PRIMARY_HEADER == 0); + a.and_(TMP1, TMP1, imm(mask)); a.cmp(TMP1, imm(_TAG_HEADER_POS_BIG)); a.b_ne(resolve_beam_label(Fail, disp1MB)); } @@ -891,8 +1085,17 @@ void BeamModuleAssembler::emit_is_map(const ArgLabel &Fail, void BeamModuleAssembler::emit_is_nil(const ArgLabel &Fail, const ArgRegister &Src) { auto src = load_source(Src, TMP1); - a.cmp(src.reg, imm(NIL)); - a.b_ne(resolve_beam_label(Fail, disp1MB)); + + if (always_one_of(Src, (BEAM_TYPE_CONS | BEAM_TYPE_NIL))) { + const int bitNumber = 1; + ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST == + (1 << bitNumber)); + comment("simplified is_nil test because its argument is always a list"); + a.tbz(src.reg, imm(bitNumber), resolve_beam_label(Fail, disp32K)); + } else { + a.cmp(src.reg, imm(NIL)); + a.b_ne(resolve_beam_label(Fail, disp1MB)); + } } void BeamModuleAssembler::emit_is_number(const ArgLabel &Fail, @@ -918,12 +1121,17 @@ void BeamModuleAssembler::emit_is_number(const ArgLabel &Fail, arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - /* The following value (0b111011) is not possible to use as - * an immediate operand for 'and'. See the note at the beginning - * of the file. + /* The header mask with the sign bit removed (0b111011) is not + * possible to use as an immediate operand for 'and'. (See the + * note at the beginning of the file.) Therefore, use a + * simpler mask (0b111000) that will also clear the primary + * tag bits. That works because we KNOW that a boxed pointer + * always points to a header word and that the primary tag for + * a header is 0. */ - mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); - a.and_(TMP2, TMP1, TMP2); + auto mask = _HEADER_SUBTAG_MASK - _BIG_SIGN_BIT; + ERTS_CT_ASSERT(TAG_PRIMARY_HEADER == 0); + a.and_(TMP2, TMP1, imm(mask)); a.cmp(TMP2, imm(_TAG_HEADER_POS_BIG)); a.mov(TMP3, imm(HEADER_FLONUM)); @@ -1139,6 +1347,43 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, const ArgSource &Y) { auto x = load_source(X, ARG1); + bool is_empty_binary = false; + if (exact_type(X, BEAM_TYPE_BITSTRING) && Y.isLiteral()) { + Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + is_empty_binary = is_binary(literal) && binary_size(literal) == 0; + } + + if (is_empty_binary) { + auto unit = getSizeUnit(X); + + comment("simplified equality test with empty binary"); + if (unit != 0 && std::gcd(unit, 8) == 8) { + arm::Gp boxed_ptr = emit_ptr_val(ARG1, x.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB)); + } else { + Label next = a.newLabel(); + + emit_untag_ptr(ARG1, x.reg); + ERTS_CT_ASSERT_FIELD_PAIR(ErlHeapBin, thing_word, size); + a.ldp(TMP1, TMP2, arm::Mem(ARG1)); + a.cbnz(TMP2, resolve_beam_label(Fail, disp1MB)); + + const int bit_number = 3; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & (1 << bit_number)) != 0 && + (_TAG_HEADER_REFC_BIN & (1 << bit_number)) == 0 && + (_TAG_HEADER_HEAP_BIN & (1 << bit_number)) == 0); + a.tbz(TMP1, imm(bit_number), next); + + a.ldrb(TMP1.w(), arm::Mem(ARG1, offsetof(ErlSubBin, bitsize))); + a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB)); + + a.bind(next); + } + + return; + } + /* If either argument is known to be an immediate, we can fail immediately * if they're not equal. */ if (always_immediate(X) || always_immediate(Y)) { @@ -1152,7 +1397,7 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, return; } - /* Both operands are registers. */ + /* Both operands are registers or literals. */ Label next = a.newLabel(); auto y = load_source(Y, ARG2); @@ -1161,9 +1406,16 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, if (always_same_types(X, Y)) { comment("skipped tag test since they are always equal"); + } else if (Y.isLiteral()) { + /* Fail immediately unless X is the same type of pointer as + * the literal Y. + */ + Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK); + int bitNumber = Support::ctz<Eterm>(tag_test); + a.tbnz(x.reg, imm(bitNumber), resolve_beam_label(Fail, disp32K)); } else { - /* The terms could still be equal if both operands are pointers - * having the same tag. */ + /* Fail immediately if the pointer tags are not equal. */ emit_is_unequal_based_on_tags(x.reg, y.reg); a.b_eq(resolve_beam_label(Fail, disp1MB)); } @@ -1174,9 +1426,7 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, mov_var(ARG2, y); emit_enter_runtime(); - runtime_call<2>(eq); - emit_leave_runtime(); a.cbz(ARG1, resolve_beam_label(Fail, disp1MB)); @@ -1189,6 +1439,26 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, const ArgSource &Y) { auto x = load_source(X, ARG1); + bool is_empty_binary = false; + if (exact_type(X, BEAM_TYPE_BITSTRING) && Y.isLiteral()) { + auto unit = getSizeUnit(X); + if (unit != 0 && std::gcd(unit, 8) == 8) { + Eterm literal = + beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + is_empty_binary = is_binary(literal) && binary_size(literal) == 0; + } + } + + if (is_empty_binary) { + arm::Gp boxed_ptr = emit_ptr_val(ARG1, x.reg); + + comment("simplified non-equality test with empty binary"); + a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.cbz(TMP1, resolve_beam_label(Fail, disp1MB)); + + return; + } + /* If either argument is known to be an immediate, we can fail immediately * if they're equal. */ if (always_immediate(X) || always_immediate(Y)) { @@ -1202,7 +1472,7 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, return; } - /* Both operands are registers. */ + /* Both operands are registers or literals. */ Label next = a.newLabel(); auto y = load_source(Y, ARG2); @@ -1211,6 +1481,14 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, if (always_same_types(X, Y)) { comment("skipped tag test since they are always equal"); + } else if (Y.isLiteral()) { + /* Succeed immediately if X is not the same type of pointer as + * the literal Y. + */ + Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK); + int bitNumber = Support::ctz<Eterm>(tag_test); + a.tbnz(x.reg, imm(bitNumber), next); } else { /* Test whether the terms are definitely unequal based on the tags * alone. */ @@ -1355,25 +1633,56 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS) { - mov_arg(ARG1, LHS); - mov_arg(ARG2, RHS); + auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); + a.b_ge(resolve_beam_label(Fail, disp1MB)); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasLowerBound(RHS)) { + Label next = a.newLabel(); + comment("simplified test because it always succeeds when RHS is a " + "bignum"); + emit_is_not_boxed(next, rhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_ge(resolve_beam_label(Fail, disp1MB)); + a.bind(next); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasUpperBound(RHS)) { + comment("simplified test because it always fails when RHS is a bignum"); + emit_is_not_boxed(resolve_beam_label(Fail, dispUnknown), rhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_ge(resolve_beam_label(Fail, disp1MB)); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasLowerBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always fails when LHS is a bignum"); + emit_is_not_boxed(resolve_beam_label(Fail, dispUnknown), lhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_ge(resolve_beam_label(Fail, disp1MB)); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasUpperBound(LHS) && + always_small(RHS)) { + Label next = a.newLabel(); + comment("simplified test because it always succeeds when LHS is a " + "bignum"); + emit_is_not_boxed(next, rhs.reg); + a.cmp(lhs.reg, rhs.reg); a.b_ge(resolve_beam_label(Fail, disp1MB)); + a.bind(next); } else if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { Label branch_compare = a.newLabel(); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); /* The only possible kind of immediate is a small and all other values * are boxed, so we can test for smalls by testing boxed. */ comment("simplified small test since all other types are boxed"); - ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); - a.and_(TMP1, ARG1, ARG2); - a.tbnz(TMP1, imm(0), branch_compare); + a.and_(TMP1, lhs.reg, rhs.reg); + emit_is_boxed(branch_compare, TMP1); + + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); fragment_call(ga->get_arith_compare_shared()); /* The flags will either be from the initial comparison, or from the @@ -1386,22 +1695,28 @@ void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, /* Relative comparisons are overwhelmingly likely to be used on smalls, * so we'll specialize those and keep the rest in a shared fragment. */ if (RHS.isSmall()) { - a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); + a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); } else if (LHS.isSmall()) { - a.and_(TMP1, ARG2, imm(_TAG_IMMED1_MASK)); + a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); } else { + /* Avoid the expensive generic comparison for equal terms. */ + a.cmp(lhs.reg, rhs.reg); + a.b_eq(next); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, ARG1, ARG2); + a.and_(TMP1, lhs.reg, rhs.reg); a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); } a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); a.b_ne(generic); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); a.b(next); a.bind(generic); + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); fragment_call(ga->get_arith_compare_shared()); a.bind(next); @@ -1412,26 +1727,70 @@ void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS) { - mov_arg(ARG1, LHS); - mov_arg(ARG2, RHS); + if (always_small(LHS) && RHS.isSmall() && RHS.isImmed()) { + auto lhs = load_source(LHS, ARG1); + comment("simplified compare because one operand is an immediate small"); + cmp(lhs.reg, RHS.as<ArgImmed>().get()); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + return; + } else if (LHS.isSmall() && LHS.isImmed() && always_small(RHS)) { + auto rhs = load_source(RHS, ARG1); + comment("simplified compare because one operand is an immediate small"); + cmp(rhs.reg, LHS.as<ArgImmed>().get()); + a.b_gt(resolve_beam_label(Fail, disp1MB)); + return; + } + + auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasLowerBound(RHS)) { + comment("simplified test because it always fails when RHS is a bignum"); + emit_is_not_boxed(resolve_beam_label(Fail, dispUnknown), rhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasUpperBound(RHS)) { + Label next = a.newLabel(); + comment("simplified test because it always succeeds when RHS is a " + "bignum"); + emit_is_not_boxed(next, rhs.reg); + a.cmp(lhs.reg, rhs.reg); a.b_lt(resolve_beam_label(Fail, disp1MB)); + a.bind(next); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasUpperBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always fails when LHS is a bignum"); + emit_is_not_boxed(resolve_beam_label(Fail, dispUnknown), lhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasLowerBound(LHS) && + always_small(RHS)) { + Label next = a.newLabel(); + comment("simplified test because it always succeeds when LHS is a " + "bignum"); + emit_is_not_boxed(next, lhs.reg); + a.cmp(lhs.reg, rhs.reg); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + a.bind(next); } else if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { Label branch_compare = a.newLabel(); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); /* The only possible kind of immediate is a small and all other values * are boxed, so we can test for smalls by testing boxed. */ comment("simplified small test since all other types are boxed"); - ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); - a.and_(TMP1, ARG1, ARG2); - a.tbnz(TMP1, imm(0), branch_compare); + a.and_(TMP1, lhs.reg, rhs.reg); + emit_is_boxed(branch_compare, TMP1); + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); fragment_call(ga->get_arith_compare_shared()); /* The flags will either be from the initial comparison, or from the @@ -1444,22 +1803,28 @@ void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, /* Relative comparisons are overwhelmingly likely to be used on smalls, * so we'll specialize those and keep the rest in a shared fragment. */ if (RHS.isSmall()) { - a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); + a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); } else if (LHS.isSmall()) { - a.and_(TMP1, ARG2, imm(_TAG_IMMED1_MASK)); + a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); } else { + /* Avoid the expensive generic comparison for equal terms. */ + a.cmp(lhs.reg, rhs.reg); + a.b_eq(next); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, ARG1, ARG2); + a.and_(TMP1, lhs.reg, rhs.reg); a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); } a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); a.b_ne(generic); - a.cmp(ARG1, ARG2); + a.cmp(lhs.reg, rhs.reg); a.b(next); a.bind(generic); + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); fragment_call(ga->get_arith_compare_shared()); a.bind(next); @@ -1467,6 +1832,358 @@ void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, } } +/* + * ARG1 = Src + * ARG2 = Min + * ARG3 = Max + * + * Result is returned in the flags. + */ +void BeamGlobalAssembler::emit_is_in_range_shared() { + Label immediate = a.newLabel(), generic_compare = a.newLabel(), + float_done = a.newLabel(), done = a.newLabel(); + + /* Is the source a float? */ + emit_is_boxed(immediate, ARG1); + + arm::Gp boxed_ptr = emit_ptr_val(TMP1, ARG1); + a.ldur(TMP2, emit_boxed_val(boxed_ptr)); + + mov_imm(TMP3, HEADER_FLONUM); + a.cmp(TMP2, TMP3); + a.b_ne(generic_compare); + + a.ldur(a64::d0, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.asr(TMP1, ARG2, imm(_TAG_IMMED1_SIZE)); + a.scvtf(a64::d1, TMP1); + + a.fcmpe(a64::d0, a64::d1); + a.b_mi(float_done); + + a.asr(TMP1, ARG3, imm(_TAG_IMMED1_SIZE)); + a.scvtf(a64::d1, TMP1); + a.fcmpe(a64::d0, a64::d1); + a.b_gt(float_done); + a.tst(ZERO, ZERO); + + a.bind(float_done); + a.ret(a64::x30); + + a.bind(immediate); + { + /* + * Src is an immediate (such as ATOM) but not SMALL. + * That means that Src must be greater than the upper + * limit. + */ + mov_imm(TMP1, 1); + a.cmp(TMP1, imm(0)); + a.ret(a64::x30); + } + + a.bind(generic_compare); + { + emit_enter_runtime_frame(); + emit_enter_runtime(); + + a.stp(ARG1, ARG3, TMP_MEM1q); + + comment("erts_cmp_compound(X, Y, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.tst(ARG1, ARG1); + a.b_mi(done); + + a.ldp(ARG1, ARG2, TMP_MEM1q); + + comment("erts_cmp_compound(X, Y, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.tst(ARG1, ARG1); + + a.bind(done); + emit_leave_runtime(); + emit_leave_runtime_frame(); + + a.ret(a64::x30); + } +} + +/* + * 1121 occurrences in OTP at the time of writing. + */ +void BeamModuleAssembler::emit_is_in_range(ArgLabel const &Small, + ArgLabel const &Large, + ArgRegister const &Src, + ArgConstant const &Min, + ArgConstant const &Max) { + Label next = a.newLabel(), generic = a.newLabel(); + bool need_generic = true; + auto src = load_source(Src, ARG1); + + if (always_small(Src)) { + need_generic = false; + comment("skipped test for small operand since it always small"); + } else if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all + * other values are boxed, so we can test for smalls by + * testing boxed. */ + comment("simplified small test since all other types are boxed"); + ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); + if (Small == Large && + always_one_of(Src, BEAM_TYPE_MASK_BOXED - BEAM_TYPE_FLOAT)) { + /* Src is never a float and the failure labels are + * equal. Therefore, since a bignum will never be within + * the range, we can fail immediately if Src is not a + * small. */ + need_generic = false; + a.tbz(src.reg, imm(0), resolve_beam_label(Small, disp32K)); + } else { + /* Src can be a float or the failures labels are distinct. + * We need to call the generic routine if Src is not a small. */ + a.tbz(src.reg, imm(0), generic); + } + } else if (Small == Large) { + /* We can save one instruction if we incorporate the test for + * small into the range check. */ + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + comment("simplified small & range tests since failure labels are " + "equal"); + sub(TMP1, src.reg, Min.as<ArgImmed>().get()); + + /* Since we have subtracted the (tagged) lower bound, the + * tag bits of the difference is 0 if and only if Src is + * a small. Testing for a tag of 0 can be done in two + * instructions. */ + a.tst(TMP1, imm(_TAG_IMMED1_MASK)); + a.b_ne(generic); + + /* Now do the range check. */ + cmp(TMP1, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get()); + a.b_hi(resolve_beam_label(Small, disp1MB)); + + /* Bypass the test code. */ + goto test_done; + } else { + /* We have no applicable type information and the failure + * labels are distinct. Emit the standard test for small + * and call the generic routine if Src is not a small. */ + a.and_(TMP1, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_ne(generic); + } + + /* We have now established that the operand is small. */ + if (Small == Large) { + comment("simplified range test since failure labels are equal"); + sub(TMP1, src.reg, Min.as<ArgImmed>().get()); + cmp(TMP1, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get()); + a.b_hi(resolve_beam_label(Small, disp1MB)); + } else { + cmp(src.reg, Min.as<ArgImmed>().get()); + a.b_lt(resolve_beam_label(Small, disp1MB)); + cmp(src.reg, Max.as<ArgImmed>().get()); + a.b_gt(resolve_beam_label(Large, disp1MB)); + } + +test_done: + if (need_generic) { + a.b(next); + } + + a.bind(generic); + if (!need_generic) { + comment("skipped generic comparison because it is not needed"); + } else { + mov_var(ARG1, src); + mov_arg(ARG2, Min); + mov_arg(ARG3, Max); + fragment_call(ga->get_is_in_range_shared()); + if (Small == Large) { + a.b_ne(resolve_beam_label(Small, disp1MB)); + } else { + a.b_lt(resolve_beam_label(Small, disp1MB)); + a.b_gt(resolve_beam_label(Large, disp1MB)); + } + } + + a.bind(next); +} + +/* + * ARG1 = Src + * ARG2 = A + * ARG3 = B + * + * Result is returned in the flags. + */ +void BeamGlobalAssembler::emit_is_ge_lt_shared() { + Label done = a.newLabel(); + + emit_enter_runtime_frame(); + emit_enter_runtime(); + + a.stp(ARG1, ARG3, TMP_MEM1q); + + comment("erts_cmp_compound(Src, A, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.tst(ARG1, ARG1); + a.b_mi(done); + + comment("erts_cmp_compound(B, Src, 0, 0);"); + a.ldp(ARG2, ARG1, TMP_MEM1q); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.cmp(ARG1, imm(0)); + + /* Make sure that ARG1 is -1, 0, or 1. */ + a.cset(ARG1, imm(arm::CondCode::kNE)); + a.csinv(ARG1, ARG1, ZERO, imm(arm::CondCode::kGE)); + + /* Prepare return value and flags. */ + a.adds(ARG1, ARG1, imm(1)); + + /* We now have: + * ARG1 == 0 if B < SRC + * ARG1 > 0 if B => SRC + * and flags set accordingly. */ + + a.bind(done); + emit_leave_runtime(); + emit_leave_runtime_frame(); + + a.ret(a64::x30); +} + +/* + * The instruction sequence: + * + * is_ge Fail1 Src A + * is_lt Fail1 B Src + * + * is common (1841 occurrences in OTP at the time of writing). + * + * is_ge + is_lt is 18 instructions, while is_ge_lt is + * 14 instructions. + */ +void BeamModuleAssembler::emit_is_ge_lt(ArgLabel const &Fail1, + ArgLabel const &Fail2, + ArgRegister const &Src, + ArgConstant const &A, + ArgConstant const &B) { + Label generic = a.newLabel(), next = a.newLabel(); + auto src = load_source(Src, ARG1); + + mov_arg(ARG2, A); + mov_arg(ARG3, B); + + a.and_(TMP1, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_ne(generic); + + a.cmp(src.reg, ARG2); + a.b_lt(resolve_beam_label(Fail1, disp1MB)); + a.cmp(ARG3, src.reg); + a.b_ge(resolve_beam_label(Fail2, disp1MB)); + a.b(next); + + a.bind(generic); + mov_var(ARG1, src); + fragment_call(ga->get_is_ge_lt_shared()); + a.b_lt(resolve_beam_label(Fail1, disp1MB)); + a.b_gt(resolve_beam_label(Fail2, disp1MB)); + + a.bind(next); +} + +/* + * 1190 occurrences in OTP at the time of writing. + */ +void BeamModuleAssembler::emit_is_ge_ge(ArgLabel const &Fail1, + ArgLabel const &Fail2, + ArgRegister const &Src, + ArgConstant const &A, + ArgConstant const &B) { + if (!always_small(Src)) { + /* In practice, it is uncommon that Src is not a known small + * integer, so we will not bother optimizing that case. */ + emit_is_ge(Fail1, Src, A); + emit_is_ge(Fail2, Src, B); + return; + } + + auto src = load_source(Src, ARG1); + subs(TMP1, src.reg, A.as<ArgImmed>().get()); + a.b_lt(resolve_beam_label(Fail1, disp1MB)); + cmp(TMP1, B.as<ArgImmed>().get() - A.as<ArgImmed>().get()); + a.b_lo(resolve_beam_label(Fail2, disp1MB)); +} + +/* + * 60 occurrences in OTP at the time of writing. Seems to be common in + * Elixir code. + * + * Currently not very frequent in OTP but very nice reduction in code + * size when it happens. We expect this combination of instructions + * to become more common in the future. + */ +void BeamModuleAssembler::emit_is_int_in_range(ArgLabel const &Fail, + ArgRegister const &Src, + ArgConstant const &Min, + ArgConstant const &Max) { + auto src = load_source(Src, ARG1); + + sub(TMP1, src.reg, Min.as<ArgImmed>().get()); + + /* Since we have subtracted the (tagged) lower bound, the + * tag bits of the difference is 0 if and only if Src is + * a small. */ + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.tst(TMP1, imm(_TAG_IMMED1_MASK)); + a.b_ne(resolve_beam_label(Fail, disp1MB)); + cmp(TMP1, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get()); + a.b_hi(resolve_beam_label(Fail, disp1MB)); +} + +/* + * 428 occurrences in OTP at the time of writing. + */ +void BeamModuleAssembler::emit_is_int_ge(ArgLabel const &Fail, + ArgRegister const &Src, + ArgConstant const &Min) { + auto src = load_source(Src, ARG1); + Label small = a.newLabel(), next = a.newLabel(); + + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types are boxed"); + emit_is_boxed(small, Src, src.reg); + } else { + a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP2, imm(_TAG_IMMED1_SMALL)); + a.b_eq(small); + + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, TMP2); + } + + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(_TAG_HEADER_POS_BIG)); + a.b_ne(resolve_beam_label(Fail, disp1MB)); + a.b(next); + + a.bind(small); + cmp(src.reg, Min.as<ArgImmed>().get()); + a.b_lt(resolve_beam_label(Fail, disp1MB)); + + a.bind(next); +} + void BeamModuleAssembler::emit_badmatch(const ArgSource &Src) { mov_arg(arm::Mem(c_p, offsetof(Process, fvalue)), Src); emit_error(BADMATCH); @@ -1632,11 +2349,11 @@ void BeamModuleAssembler::emit_try_case_end(const ArgSource &Src) { void BeamModuleAssembler::emit_raise(const ArgSource &Trace, const ArgSource &Value) { - mov_arg(TMP1, Value); + auto value = load_source(Value, TMP1); mov_arg(ARG2, Trace); /* This is an error, attach a stacktrace to the reason. */ - a.str(TMP1, arm::Mem(c_p, offsetof(Process, fvalue))); + a.str(value.reg, arm::Mem(c_p, offsetof(Process, fvalue))); a.str(ARG2, arm::Mem(c_p, offsetof(Process, ftrace))); emit_enter_runtime(0); diff --git a/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp index 1c7df4e6f6..24e17f3358 100644 --- a/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp @@ -31,6 +31,7 @@ */ #include <algorithm> +#include <numeric> #include "beam_asm.hpp" extern "C" @@ -56,6 +57,8 @@ void BeamGlobalAssembler::emit_raise_badarg(const ErtsCodeMFA *mfa) { /* ================================================================ * '=:='/2 * '=/='/2 + * '>='/2 + * '<'/2 * ================================================================ */ @@ -125,13 +128,20 @@ void BeamGlobalAssembler::emit_bif_is_ne_exact_shared() { } } -void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgSource &LHS, - const ArgSource &RHS, - const ArgRegister &Dst, - Eterm fail_value, - Eterm succ_value) { +void BeamModuleAssembler::emit_cond_to_bool(arm::CondCode cc, + const ArgRegister &Dst) { auto dst = init_destination(Dst, TMP2); + mov_imm(TMP3, am_true); + mov_imm(TMP4, am_false); + a.csel(dst.reg, TMP3, TMP4, cc); + flush_var(dst); +} + +void BeamModuleAssembler::emit_cmp_immed_to_bool(arm::CondCode cc, + const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { if (RHS.isImmed()) { auto lhs = load_source(LHS, TMP1); cmp_arg(lhs.reg, RHS); @@ -139,11 +149,7 @@ void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgSource &LHS, auto [lhs, rhs] = load_sources(LHS, TMP1, RHS, TMP2); a.cmp(lhs.reg, rhs.reg); } - - mov_imm(TMP3, succ_value); - mov_imm(TMP4, fail_value); - a.csel(dst.reg, TMP3, TMP4, arm::CondCode::kEQ); - flush_var(dst); + emit_cond_to_bool(cc, Dst); } void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgRegister &LHS, @@ -153,7 +159,7 @@ void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgRegister &LHS, if (!LHS.isImmed() && !RHS.isImmed()) { comment("simplified check since one argument is an immediate"); } - emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_false, am_true); + emit_cmp_immed_to_bool(arm::CondCode::kEQ, LHS, RHS, Dst); } else { auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); auto dst = init_destination(Dst, ARG1); @@ -173,7 +179,7 @@ void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgRegister &LHS, if (!LHS.isImmed() && !RHS.isImmed()) { comment("simplified check since one argument is an immediate"); } - emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_true, am_false); + emit_cmp_immed_to_bool(arm::CondCode::kNE, LHS, RHS, Dst); } else { auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); auto dst = init_destination(Dst, ARG1); @@ -186,6 +192,107 @@ void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgRegister &LHS, } } +void BeamModuleAssembler::emit_bif_is_ge(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + if (always_small(LHS) && RHS.isSmall() && RHS.isImmed()) { + auto lhs = load_source(LHS, ARG1); + + comment("simplified compare because one operand is an immediate small"); + cmp(lhs.reg, RHS.as<ArgImmed>().get()); + emit_cond_to_bool(arm::CondCode::kGE, Dst); + + return; + } else if (LHS.isSmall() && LHS.isImmed() && always_small(RHS)) { + auto rhs = load_source(RHS, ARG1); + + comment("simplified compare because one operand is an immediate small"); + cmp(rhs.reg, LHS.as<ArgImmed>().get()); + emit_cond_to_bool(arm::CondCode::kLE, Dst); + + return; + } + + auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); + + Label generic = a.newLabel(), next = a.newLabel(); + + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all + * other values are boxed, so we can test for smalls by + * testing boxed. */ + comment("simplified small test since all other types are boxed"); + a.and_(TMP1, lhs.reg, rhs.reg); + emit_is_not_boxed(generic, TMP1); + } else { + /* Relative comparisons are overwhelmingly likely to be used + * on smalls, so we'll specialize those and keep the rest in a + * shared fragment. */ + if (RHS.isSmall()) { + a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); + } else if (LHS.isSmall()) { + a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, lhs.reg, rhs.reg); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + } + + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_ne(generic); + } + + a.cmp(lhs.reg, rhs.reg); + a.b(next); + + a.bind(generic); + { + a.cmp(lhs.reg, rhs.reg); + a.b_eq(next); + + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); + fragment_call(ga->get_arith_compare_shared()); + } + + a.bind(next); + emit_cond_to_bool(arm::CondCode::kGE, Dst); +} + +void BeamModuleAssembler::emit_bif_is_lt(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); + Label generic = a.newLabel(), next = a.newLabel(); + + /* Relative comparisons are overwhelmingly likely to be used on smalls, + * so we'll specialize those and keep the rest in a shared fragment. */ + if (RHS.isSmall()) { + a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); + } else if (LHS.isSmall()) { + a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, lhs.reg, rhs.reg); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + } + + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.b_ne(generic); + + a.cmp(lhs.reg, rhs.reg); + a.b(next); + + a.bind(generic); + mov_var(ARG1, lhs); + mov_var(ARG2, rhs); + fragment_call(ga->get_arith_compare_shared()); + + a.bind(next); + emit_cond_to_bool(arm::CondCode::kLT, Dst); +} + /* ================================================================ * and/2 * ================================================================ @@ -298,16 +405,54 @@ void BeamModuleAssembler::emit_bif_bit_size(const ArgLabel &Fail, auto src = load_source(Src, ARG1); auto dst = init_destination(Dst, ARG1); - mov_var(ARG1, src); + if (exact_type(Src, BEAM_TYPE_BITSTRING)) { + Label not_sub_bin = a.newLabel(); + arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + + if (is_bitstring) { + comment("inlined bit_size/1 because " + "its argument is a bitstring"); + } else { + comment("inlined and simplified bit_size/1 because " + "its argument is a binary"); + } - if (Fail.get() == 0) { - fragment_call(ga->get_bif_bit_size_body()); + if (is_bitstring) { + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + } + + a.ldur(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.lsl(TMP2, TMP2, imm(_TAG_IMMED1_SIZE + 3)); + + if (is_bitstring) { + const int bit_number = 3; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & (1 << bit_number)) != 0 && + (_TAG_HEADER_REFC_BIN & (1 << bit_number)) == 0 && + (_TAG_HEADER_HEAP_BIN & (1 << bit_number)) == 0); + a.tbz(TMP1, imm(bit_number), not_sub_bin); + + a.ldurb(TMP1.w(), + emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize))); + a.add(TMP2, TMP2, TMP1, imm(_TAG_IMMED1_SIZE)); + } + + a.bind(not_sub_bin); + a.orr(dst.reg, TMP2, _TAG_IMMED1_SMALL); } else { - fragment_call(ga->get_bif_bit_size_guard()); - emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); + mov_var(ARG1, src); + + if (Fail.get() == 0) { + fragment_call(ga->get_bif_bit_size_body()); + } else { + fragment_call(ga->get_bif_bit_size_guard()); + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } + mov_var(dst, ARG1); } - mov_var(dst, ARG1); flush_var(dst); } @@ -380,16 +525,55 @@ void BeamModuleAssembler::emit_bif_byte_size(const ArgLabel &Fail, auto src = load_source(Src, ARG1); auto dst = init_destination(Dst, ARG1); - mov_var(ARG1, src); + if (exact_type(Src, BEAM_TYPE_BITSTRING)) { + Label not_sub_bin = a.newLabel(); + arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + + if (is_bitstring) { + comment("inlined byte_size/1 because " + "its argument is a bitstring"); + } else { + comment("inlined and simplified byte_size/1 because " + "its argument is a binary"); + } - if (Fail.get() == 0) { - fragment_call(ga->get_bif_byte_size_body()); + if (is_bitstring) { + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + } + + a.ldur(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + + if (is_bitstring) { + const int bit_number = 3; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & (1 << bit_number)) != 0 && + (_TAG_HEADER_REFC_BIN & (1 << bit_number)) == 0 && + (_TAG_HEADER_HEAP_BIN & (1 << bit_number)) == 0); + a.tbz(TMP1, imm(bit_number), not_sub_bin); + + a.ldurb(TMP1.w(), + emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize))); + a.cmp(TMP1.w(), imm(0)); + a.cinc(TMP2, TMP2, arm::CondCode::kNE); + } + + a.bind(not_sub_bin); + mov_imm(dst.reg, _TAG_IMMED1_SMALL); + a.bfi(dst.reg, TMP2, imm(_TAG_IMMED1_SIZE), imm(SMALL_BITS)); } else { - fragment_call(ga->get_bif_byte_size_guard()); - emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); + mov_var(ARG1, src); + + if (Fail.get() == 0) { + fragment_call(ga->get_bif_byte_size_body()); + } else { + fragment_call(ga->get_bif_byte_size_guard()); + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } + mov_var(dst, ARG1); } - mov_var(dst, ARG1); flush_var(dst); } @@ -477,9 +661,8 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, if (is_tuple(tuple_literal)) { Label next = a.newLabel(), fail = a.newLabel(); Sint size = Sint(arityval(*tuple_val(tuple_literal))); - auto [min, max] = getIntRange(Pos); - bool is_bounded = min <= max; - bool can_fail = !is_bounded || min < 1 || size < max; + auto [min, max] = getClampedRange(Pos); + bool can_fail = min < 1 || size < max; auto [pos, tuple] = load_sources(Pos, ARG3, Tuple, ARG4); auto dst = init_destination(Dst, ARG1); @@ -499,14 +682,14 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, a.asr(TMP3, pos.reg, imm(_TAG_IMMED1_SIZE)); - if (is_bounded && min >= 1) { + if (min >= 1) { comment("skipped check for position >= 1"); } else { a.cmp(TMP3, imm(1)); a.b_mi(fail); } - if (is_bounded && size >= max) { + if (size >= max) { comment("skipped check for position beyond tuple"); } else { mov_imm(TMP2, size); @@ -538,19 +721,51 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, } } - mov_arg(ARG1, Pos); - mov_arg(ARG2, Tuple); + bool const_position; + + const_position = Pos.isSmall() && Pos.as<ArgSmall>().getSigned() > 0 && + Pos.as<ArgSmall>().getSigned() <= (Sint)MAX_ARITYVAL; - if (Fail.get() != 0) { - fragment_call(ga->get_bif_element_guard_shared()); - emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); + if (const_position && exact_type(Tuple, BEAM_TYPE_TUPLE)) { + comment("simplified element/2 because arguments are known types"); + auto tuple = load_source(Tuple, ARG2); + auto dst = init_destination(Dst, ARG1); + Uint position = Pos.as<ArgSmall>().getUnsigned(); + arm::Gp boxed_ptr = emit_ptr_val(TMP1, tuple.reg); + + a.ldur(TMP2, emit_boxed_val(boxed_ptr)); + ERTS_CT_ASSERT(make_arityval_zero() == 0); + cmp(TMP2, position << _HEADER_ARITY_OFFS); + if (Fail.get() != 0) { + a.b_lo(resolve_beam_label(Fail, disp1MB)); + } else { + Label good = a.newLabel(); + a.b_hs(good); + mov_arg(ARG1, Pos); + mov_var(ARG2, tuple); + fragment_call(ga->get_handle_element_error_shared()); + a.bind(good); + } + + safe_ldur(dst.reg, emit_boxed_val(boxed_ptr, position << 3)); + flush_var(dst); } else { - fragment_call(ga->get_bif_element_body_shared()); - } + /* Too much code to inline. Call a helper fragment. */ + mov_arg(ARG1, Pos); + mov_arg(ARG2, Tuple); + + if (Fail.get() != 0) { + fragment_call(ga->get_bif_element_guard_shared()); + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } else { + fragment_call(ga->get_bif_element_body_shared()); + } - auto dst = init_destination(Dst, ARG1); - mov_var(dst, ARG1); - flush_var(dst); + auto dst = init_destination(Dst, ARG1); + mov_var(dst, ARG1); + flush_var(dst); + } } /* ================================================================ @@ -585,6 +800,142 @@ void BeamModuleAssembler::emit_bif_hd(const ArgSource &Src, } /* ================================================================ + * is_map_key/2 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_is_map_key(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type(Src, BEAM_TYPE_MAP)) { + emit_i_bif2(Key, Src, Fail, Bif, Dst); + return; + } + + comment("inlined BIF is_map_key/2"); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (masked_types(Key, BEAM_TYPE_MASK_IMMEDIATE) != BEAM_TYPE_NONE) { + fragment_call(ga->get_i_get_map_element_shared()); + emit_cond_to_bool(arm::CondCode::kEQ, Dst); + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + cmp(ARG1, THE_NON_VALUE); + emit_cond_to_bool(arm::CondCode::kNE, Dst); + } +} + +/* ================================================================ + * map_get/2 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_map_get_badmap() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + mov_imm(TMP1, BADMAP); + a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason))); + a.str(ARG1, arm::Mem(c_p, offsetof(Process, fvalue))); + a.mov(XREG0, ARG2); + a.mov(XREG1, ARG1); + mov_imm(ARG4, &mfa); + a.b(labels[raise_exception]); +} + +void BeamGlobalAssembler::emit_handle_map_get_badkey() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + mov_imm(TMP1, BADKEY); + a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason))); + a.str(ARG2, arm::Mem(c_p, offsetof(Process, fvalue))); + a.mov(XREG0, ARG2); + a.mov(XREG1, ARG1); + mov_imm(ARG4, &mfa); + a.b(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_map_get(const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + Label good_key = a.newLabel(); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (exact_type(Src, BEAM_TYPE_MAP)) { + comment("skipped test for map for known map argument"); + } else { + Label bad_map = a.newLabel(); + Label good_map = a.newLabel(); + + if (Fail.get() == 0) { + emit_is_boxed(bad_map, Src, ARG1); + } else { + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, ARG1); + } + + /* As an optimization for the `error | #{}` case, skip checking the + * header word when we know that the only possible boxed type + * is a map. */ + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_MAP) { + comment("skipped header test since we know it's a map when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, ARG1); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(_TAG_HEADER_MAP)); + if (Fail.get() == 0) { + a.b_eq(good_map); + } else { + a.b_ne(resolve_beam_label(Fail, disp1MB)); + } + } + + a.bind(bad_map); + if (Fail.get() == 0) { + fragment_call(ga->get_handle_map_get_badmap()); + } + + a.bind(good_map); + } + + if (masked_types(Key, BEAM_TYPE_MASK_IMMEDIATE) != BEAM_TYPE_NONE) { + fragment_call(ga->get_i_get_map_element_shared()); + if (Fail.get() == 0) { + a.b_eq(good_key); + } else { + a.b_ne(resolve_beam_label(Fail, disp1MB)); + } + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + if (Fail.get() == 0) { + emit_branch_if_value(ARG1, good_key); + } else { + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } + } + + if (Fail.get() == 0) { + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + fragment_call(ga->get_handle_map_get_badkey()); + } + + a.bind(good_key); + mov_arg(Dst, ARG1); +} + +/* ================================================================ * map_size/1 * ================================================================ */ @@ -612,20 +963,26 @@ void BeamModuleAssembler::emit_bif_map_size(const ArgLabel &Fail, } arm::Gp boxed_ptr = emit_ptr_val(TMP3, src.reg); - a.ldur(TMP4, emit_boxed_val(boxed_ptr)); - a.and_(TMP4, TMP4, imm(_TAG_HEADER_MASK)); - a.cmp(TMP4, imm(_TAG_HEADER_MAP)); - if (Fail.get() == 0) { - a.b_eq(good_map); - a.bind(error); - { - mov_var(XREG0, src); - fragment_call(ga->get_handle_map_size_error()); - } - } else { - a.b_ne(resolve_beam_label(Fail, disp1MB)); + if (exact_type(Src, BEAM_TYPE_MAP)) { + comment("skipped type check because the argument is always a map"); a.bind(error); /* Never referenced. */ + } else { + a.ldur(TMP4, emit_boxed_val(boxed_ptr)); + a.and_(TMP4, TMP4, imm(_TAG_HEADER_MASK)); + a.cmp(TMP4, imm(_TAG_HEADER_MAP)); + + if (Fail.get() == 0) { + a.b_eq(good_map); + a.bind(error); + { + mov_var(XREG0, src); + fragment_call(ga->get_handle_map_size_error()); + } + } else { + a.b_ne(resolve_beam_label(Fail, disp1MB)); + a.bind(error); /* Never referenced. */ + } } a.bind(good_map); @@ -805,15 +1162,27 @@ void BeamModuleAssembler::emit_bif_tuple_size(const ArgLabel &Fail, auto src = load_source(Src, ARG1); auto dst = init_destination(Dst, ARG1); - mov_var(ARG1, src); - - if (Fail.get() == 0) { - fragment_call(ga->get_bif_tuple_size_body()); + if (exact_type(Src, BEAM_TYPE_TUPLE)) { + comment("simplifed tuple_size/1 because the argument is always a " + "tuple"); + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + ERTS_CT_ASSERT(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE > 0); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.lsr(TMP1, TMP1, _HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE); + a.orr(dst.reg, TMP1, imm(_TAG_IMMED1_SMALL)); } else { - fragment_call(ga->get_bif_tuple_size_guard()); - emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); - } + mov_var(ARG1, src); + + if (Fail.get() == 0) { + fragment_call(ga->get_bif_tuple_size_body()); + } else { + fragment_call(ga->get_bif_tuple_size_guard()); + emit_branch_if_not_value(ARG1, + resolve_beam_label(Fail, dispUnknown)); + } - mov_var(dst, ARG1); + mov_var(dst, ARG1); + } flush_var(dst); } diff --git a/erts/emulator/beam/jit/arm/instr_map.cpp b/erts/emulator/beam/jit/arm/instr_map.cpp index 36d0d95433..fb3db42431 100644 --- a/erts/emulator/beam/jit/arm/instr_map.cpp +++ b/erts/emulator/beam/jit/arm/instr_map.cpp @@ -25,6 +25,7 @@ using namespace asmjit; extern "C" { #include "erl_map.h" +#include "erl_term_hashing.h" #include "beam_common.h" } @@ -52,6 +53,11 @@ void BeamGlobalAssembler::emit_internal_hash_helper() { a.add(lower, lower, constant); a.add(upper, upper, constant); +#if defined(ERL_INTERNAL_HASH_CRC32C) + a.crc32cw(lower, hash, lower); + a.add(hash, hash, lower); + a.crc32cw(hash, hash, upper); +#else using rounds = std::initializer_list<std::tuple<a64::Gp, a64::Gp, a64::Gp, int>>; for (const auto &round : rounds{{lower, upper, hash, 13}, @@ -74,6 +80,7 @@ void BeamGlobalAssembler::emit_internal_hash_helper() { a.eor(r_a, r_a, r_c, arm::lsl(-shift)); } } +#endif a.ret(a64::x30); } @@ -256,22 +263,6 @@ void BeamModuleAssembler::emit_new_map(const ArgRegister &Dst, mov_arg(Dst, ARG1); } -void BeamGlobalAssembler::emit_i_new_small_map_lit_shared() { - emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(); - - a.mov(ARG1, c_p); - load_x_reg_array(ARG2); - runtime_call<5>(erts_gc_new_small_map_lit); - - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(); - emit_leave_runtime_frame(); - - a.ret(a64::x30); -} - void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgRegister &Dst, const ArgWord &Live, const ArgLiteral &Keys, @@ -279,15 +270,50 @@ void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgRegister &Dst, const Span<ArgVal> &args) { ASSERT(Size.get() == args.size()); - embed_vararg_rodata(args, ARG5); + emit_gc_test(ArgWord(0), + ArgWord(args.size() + MAP_HEADER_FLATMAP_SZ + 1), + Live); - ASSERT(Keys.isLiteral()); - mov_arg(ARG3, Keys); - mov_arg(ARG4, Live); + std::vector<ArgVal> data; + data.reserve(args.size() + MAP_HEADER_FLATMAP_SZ + 1); + data.push_back(ArgWord(MAP_HEADER_FLATMAP)); + data.push_back(Size); + data.push_back(Keys); - fragment_call(ga->get_i_new_small_map_lit_shared()); + bool dst_is_src = false; + for (auto arg : args) { + data.push_back(arg); + dst_is_src |= (arg == Dst); + } - mov_arg(Dst, ARG1); + if (dst_is_src) { + a.add(TMP1, HTOP, TAG_PRIMARY_BOXED); + } else { + auto ptr = init_destination(Dst, TMP1); + a.add(ptr.reg, HTOP, TAG_PRIMARY_BOXED); + flush_var(ptr); + } + + size_t size = data.size(); + unsigned i; + for (i = 0; i < size - 1; i += 2) { + if ((i % 128) == 0) { + check_pending_stubs(); + } + + auto [first, second] = load_sources(data[i], TMP2, data[i + 1], TMP3); + a.stp(first.reg, second.reg, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } + + if (i < size) { + mov_arg(arm::Mem(HTOP).post(sizeof(Eterm)), data[i]); + } + + if (dst_is_src) { + auto ptr = init_destination(Dst, TMP1); + mov_var(ptr, TMP1); + flush_var(ptr); + } } /* ARG1 = map diff --git a/erts/emulator/beam/jit/arm/instr_select.cpp b/erts/emulator/beam/jit/arm/instr_select.cpp index 41c9b0a95c..789d9dd7d4 100644 --- a/erts/emulator/beam/jit/arm/instr_select.cpp +++ b/erts/emulator/beam/jit/arm/instr_select.cpp @@ -102,6 +102,7 @@ static std::pair<UWord, int> plan_untag(const Span<ArgVal> &args) { } const std::vector<ArgVal> BeamModuleAssembler::emit_select_untag( + const ArgSource &Src, const Span<ArgVal> &args, a64::Gp comparand, Label fail, @@ -112,16 +113,21 @@ const std::vector<ArgVal> BeamModuleAssembler::emit_select_untag( /* Emit code to test that the source value has the correct type and * untag it. */ comment("(comparing untagged+rebased values)"); - if (args.front().isSmall()) { - a.and_(TMP1, comparand, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + if ((args.front().isSmall() && always_small(Src)) || + (args.front().isAtom() && exact_type(Src, BEAM_TYPE_ATOM))) { + comment("(skipped type test)"); } else { - ASSERT(args.front().isAtom()); - a.and_(TMP1, comparand, imm(_TAG_IMMED2_MASK)); - a.cmp(TMP1, imm(_TAG_IMMED2_ATOM)); - } + if (args.front().isSmall()) { + a.and_(TMP1, comparand, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + } else { + ASSERT(args.front().isAtom()); + a.and_(TMP1, comparand, imm(_TAG_IMMED2_MASK)); + a.cmp(TMP1, imm(_TAG_IMMED2_ATOM)); + } - a.b_ne(resolve_label(fail, disp1MB)); + a.b_ne(resolve_label(fail, disp1MB)); + } if (shift != 0) { a.lsr(ARG1, comparand, imm(shift)); @@ -281,7 +287,8 @@ void BeamModuleAssembler::emit_i_select_val_lins(const ArgSource &Src, emit_linear_search(src.reg, fail, args); } } else { - auto untagged = emit_select_untag(args, src.reg, next, base, shift); + auto untagged = + emit_select_untag(Src, args, src.reg, next, base, shift); if (!emit_optimized_three_way_select(ARG1, fail, untagged)) { emit_linear_search(ARG1, fail, untagged); @@ -322,7 +329,8 @@ void BeamModuleAssembler::emit_i_select_val_bins(const ArgSource &Src, if (base == 0 && shift == 0) { emit_binsearch_nodes(src.reg, 0, count - 1, fail, args); } else { - auto untagged = emit_select_untag(args, src.reg, fail, base, shift); + auto untagged = + emit_select_untag(Src, args, src.reg, fail, base, shift); emit_binsearch_nodes(ARG1, 0, count - 1, fail, untagged); } @@ -407,17 +415,21 @@ void BeamModuleAssembler::emit_i_jump_on_val(const ArgSource &Src, ASSERT(Size.get() == args.size()); - a.and_(TMP3, src.reg, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP3, imm(_TAG_IMMED1_SMALL)); - - if (Fail.isLabel()) { - a.b_ne(resolve_beam_label(Fail, disp1MB)); + if (always_small(Src)) { + comment("(skipped type test)"); } else { - /* NIL means fallthrough to the next instruction. */ - ASSERT(Fail.isNil()); + a.and_(TMP3, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP3, imm(_TAG_IMMED1_SMALL)); - fail = a.newLabel(); - a.b_ne(fail); + if (Fail.isLabel()) { + a.b_ne(resolve_beam_label(Fail, disp1MB)); + } else { + /* NIL means fallthrough to the next instruction. */ + ASSERT(Fail.isNil()); + + fail = a.newLabel(); + a.b_ne(fail); + } } a.asr(TMP1, src.reg, imm(_TAG_IMMED1_SIZE)); diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab index cc0bfeccf0..03465519b4 100644 --- a/erts/emulator/beam/jit/arm/ops.tab +++ b/erts/emulator/beam/jit/arm/ops.tab @@ -275,11 +275,12 @@ load_tuple_ptr s # If positions are in consecutive memory, fetch and store two words at # once. +## FIXME: Fix this bug in maint, too. i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple2 | get_tuple_element Tuple3 Pos2 Dst2 | equal(Tuple, Tuple2) | equal(Tuple, Tuple3) | - consecutive_words(Pos1, Pos2) => + consecutive_words(Pos1, Pos2) | distinct(Dst1, Dst2) => get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2 @@ -455,6 +456,41 @@ is_eq_exact f s s is_ne_exact f s s +is_integer NotInt N0 | is_ge Small N1=xy Min=i | is_ge Large Max=i N2=xy | + equal(N0, N1) | equal(N1, N2) | + equal(NotInt, Small) | equal(Small, Large) => + is_int_in_range NotInt N0 Min Max + +is_integer NotInt N0 | is_ge Large Max=i N2=xy | is_ge Small N1=xy Min=i | + equal(N0, N1) | equal(N1, N2) | + equal(NotInt, Small) | equal(Small, Large) => + is_int_in_range NotInt N0 Min Max + +is_integer NotInt N0 | is_ge Fail N1=xy Min=i | + equal(N0, N1) | equal(NotInt, Fail) => + is_int_ge NotInt N0 Min + +is_int_in_range f S c c +is_int_ge f S c + +is_ge Small N1=xy Min=i | is_ge Large Max=i N2=xy | equal(N1, N2) => + is_in_range Small Large N1 Min Max + +is_ge Large Max=i N2=xy | is_ge Small N1=xy Min=i | equal(N1, N2) => + is_in_range Small Large N2 Min Max + +is_in_range f f S c c + +is_ge Small N1=xy A=i | is_lt Large B=i N2=xy | equal(N1, N2) => + is_ge_lt Small Large N1 A B + +is_ge_lt f f S c c + +is_ge Fail1 N1=xy A=i | is_ge Fail2 N2=xy B=i | equal(N1, N2) => + is_ge_ge Fail1 Fail2 N1 A B + +is_ge_ge f f S c c + is_lt f s s is_ge f s s @@ -710,6 +746,14 @@ bif1 Fail Bif=u$bif:erlang:tuple_size/1 Src=d Dst=d => bif_tuple_size Fail Src Dst bif_tuple_size j S d +bif2 Fail Bif=u$bif:erlang:map_get/2 Src1 Src2=xy Dst=d => + bif_map_get Fail Src1 Src2 Dst +bif_map_get j s s d + +bif2 Fail Bif=u$bif:erlang:is_map_key/2 Key Map=xy Dst=d => + bif_is_map_key Bif Fail Key Map Dst +bif_is_map_key b j s s d + bif1 Fail Bif S1 Dst | never_fails(Bif) => nofail_bif1 S1 Bif Dst bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst @@ -718,6 +762,8 @@ bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst nofail_bif2 S1=d S2 Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact S1 S2 Dst nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst +nofail_bif2 S1 S2 Bif Dst | is_ge_bif(Bif) => bif_is_ge S1 S2 Dst +nofail_bif2 S1 S2 Bif Dst | is_lt_bif(Bif) => bif_is_lt S1 S2 Dst i_get_hash c I d i_get s d @@ -735,6 +781,8 @@ i_bif3 s s s j b d bif_is_eq_exact S s d bif_is_ne_exact S s d +bif_is_ge s s d +bif_is_lt s s d # # Internal calls. @@ -846,22 +894,30 @@ i_flush_stubs i_breakpoint_trampoline # ================================================================ -# New bit syntax matching (R11B). +# New bit syntax matching for fixed sizes (from OTP 26). +# ================================================================ + +bs_match Fail Ctx Size Rest=* => bs_match(Fail, Ctx, Size, Rest) + +i_bs_match Fail Ctx Rest=* | test_heap Need Live => + i_bs_match_test_heap Fail Ctx Need Live Rest + +i_bs_match f S * +i_bs_match_test_heap f S I t * + +# ================================================================ +# Bit syntax matching (from R11B). # ================================================================ %warm -# Matching integers +# Matching integers. bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val i_bs_match_string S f W M # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => - get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) - -i_bs_get_integer S f t t s d -i_bs_get_fixed_integer S f t t t d +bs_get_integer2 f S t s t t d # Fetching binaries from binaries. bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => @@ -1393,3 +1449,9 @@ recv_marker_use S # Landing pad for fun calls/apply where we set up arguments and check errors i_lambda_trampoline F f W W + +# +# OTP 26 +# + +update_record a I s d I * diff --git a/erts/emulator/beam/jit/arm/predicates.tab b/erts/emulator/beam/jit/arm/predicates.tab index fc55441c28..5c0af8b3bc 100644 --- a/erts/emulator/beam/jit/arm/predicates.tab +++ b/erts/emulator/beam/jit/arm/predicates.tab @@ -35,7 +35,7 @@ pred.is_mfa_bif(M, F, A) { pred.never_fails(Bif) { static Eterm nofail_bifs[] = {am_Neqeq, - am_Le, + am_Lt, am_Neq, am_Eq, am_Le, @@ -84,6 +84,7 @@ pred.never_fails(Bif) { pred.consecutive_words(A1, A2) { return A1.type == A2.type && A1.val + 1 == A2.val; } + pred.is_eq_exact_bif(Bif) { Uint index = Bif.val; @@ -105,3 +106,25 @@ pred.is_ne_exact_bif(Bif) { } return 0; } + +pred.is_ge_bif(Bif) { + Uint index = Bif.val; + + if (Bif.type == TAG_u && index < S->beam.imports.count) { + BeamFile_ImportEntry *entry = &S->beam.imports.entries[index]; + + return entry->module == am_erlang && entry->function == am_Ge && entry->arity == 2; + } + return 0; +} + +pred.is_lt_bif(Bif) { + Uint index = Bif.val; + + if (Bif.type == TAG_u && index < S->beam.imports.count) { + BeamFile_ImportEntry *entry = &S->beam.imports.entries[index]; + + return entry->module == am_erlang && entry->function == am_Lt && entry->arity == 2; + } + return 0; +} diff --git a/erts/emulator/beam/jit/beam_asm.h b/erts/emulator/beam/jit/beam_asm.h index 13f5142b7b..8745db9e1e 100644 --- a/erts/emulator/beam/jit/beam_asm.h +++ b/erts/emulator/beam/jit/beam_asm.h @@ -37,6 +37,7 @@ # define BEAMASM_PERF_MAP (1 << 1) extern int erts_jit_perf_support; # endif +extern int erts_jit_single_map; void beamasm_init(void); void *beamasm_new_assembler(Eterm mod, diff --git a/erts/emulator/beam/jit/beam_jit_args.hpp b/erts/emulator/beam/jit/beam_jit_args.hpp index 8e1e0359ee..4dba1b3f4f 100644 --- a/erts/emulator/beam/jit/beam_jit_args.hpp +++ b/erts/emulator/beam/jit/beam_jit_args.hpp @@ -246,6 +246,11 @@ struct ArgRegister : public ArgSource { constexpr int typeIndex() const { return (int)(val >> 10); } + + template<typename T> + constexpr T copy(int n) const { + return T(n | (val & ~REG_MASK)); + } }; struct ArgXRegister : public ArgRegister { diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp index 71b38d2946..fe76ca5f35 100644 --- a/erts/emulator/beam/jit/beam_jit_common.cpp +++ b/erts/emulator/beam/jit/beam_jit_common.cpp @@ -735,7 +735,6 @@ Eterm beam_jit_bs_init(Process *c_p, Uint alloc, unsigned Live) { erts_bin_offset = 0; - erts_writable_bin = 0; if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin *hb; Uint bin_need; @@ -802,7 +801,6 @@ Eterm beam_jit_bs_init_bits(Process *c_p, } erts_bin_offset = 0; - erts_writable_bin = 0; /* num_bits = Number of bits to build * num_bytes = Number of bytes to allocate in the binary diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp index a478b5625d..84c396b3fe 100644 --- a/erts/emulator/beam/jit/beam_jit_main.cpp +++ b/erts/emulator/beam/jit/beam_jit_main.cpp @@ -42,6 +42,8 @@ ErtsFrameLayout ERTS_WRITE_UNLIKELY(erts_frame_layout); #ifdef HAVE_LINUX_PERF_SUPPORT int erts_jit_perf_support; #endif +/* Force use of single-mapped RWX memory for JIT code */ +int erts_jit_single_map = 0; /* * Special Beam instructions. @@ -120,6 +122,13 @@ static JitAllocator *create_allocator(JitAllocator::CreateParams *params) { void *test_ro, *test_rw; Error err; +#if defined(__APPLE__) && defined(__aarch64__) + /* Using a single map will not work on Apple Silicon. */ + if (params->options == JitAllocatorOptions::kNone) { + return nullptr; + } +#endif + auto *allocator = new JitAllocator(params); err = allocator->alloc(&test_ro, &test_rw, 1); @@ -140,16 +149,19 @@ static JitAllocator *pick_allocator() { #if defined(HAVE_LINUX_PERF_SUPPORT) /* `perf` has a hard time showing symbols for dual-mapped memory, so we'll * use single-mapped memory when enabled. */ - if (erts_jit_perf_support & (BEAMASM_PERF_DUMP | BEAMASM_PERF_MAP)) { + if (erts_jit_perf_support & (BEAMASM_PERF_DUMP | BEAMASM_PERF_MAP)) + erts_jit_single_map = 1; + +#endif + if (erts_jit_single_map) { if (auto *alloc = create_allocator(&single_params)) { return alloc; } ERTS_INTERNAL_ERROR("jit: Failed to allocate executable+writable " - "memory. Either allow this or disable the " - "'+JPperf' option."); + "memory. Either allow this or disable both the " + "'+JPperf' and '+JMsingle' options."); } -#endif #if !defined(VALGRIND) /* Default to dual-mapped memory with separate executable and writable diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp index cd353714ef..c53bf8de28 100644 --- a/erts/emulator/beam/jit/x86/beam_asm.hpp +++ b/erts/emulator/beam/jit/x86/beam_asm.hpp @@ -747,7 +747,7 @@ protected: #endif } - void emit_is_boxed(Label Fail, x86::Gp Src, Distance dist = dLong) { + void emit_test_boxed(x86::Gp Src) { /* Use the shortest possible instruction depending on the source * register. */ if (Src == x86::rax || Src == x86::rdi || Src == x86::rsi || @@ -756,6 +756,10 @@ protected: } else { a.test(Src.r32(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); } + } + + void emit_is_boxed(Label Fail, x86::Gp Src, Distance dist = dLong) { + emit_test_boxed(Src); if (dist == dShort) { a.short_().jne(Fail); } else { @@ -763,6 +767,15 @@ protected: } } + void emit_is_not_boxed(Label Fail, x86::Gp Src, Distance dist = dLong) { + emit_test_boxed(Src); + if (dist == dShort) { + a.short_().je(Fail); + } else { + a.je(Fail); + } + } + x86::Gp emit_ptr_val(x86::Gp Dst, x86::Gp Src) { #if !defined(TAG_LITERAL_PTR) return Src; @@ -864,6 +877,100 @@ protected: mov_imm(to, 0); } + /* Copies `count` words from `from` to `to`. + * + * Clobbers `spill` and the first vector register (xmm0, ymm0 etc). */ + void emit_copy_words(x86::Mem from, + x86::Mem to, + Sint32 count, + x86::Gp spill) { + ASSERT(!from.hasIndex() && !to.hasIndex()); + ASSERT(count >= 0 && count < (ERTS_SINT32_MAX / (Sint32)sizeof(UWord))); + ASSERT(from.offset() < ERTS_SINT32_MAX - count * (Sint32)sizeof(UWord)); + ASSERT(to.offset() < ERTS_SINT32_MAX - count * (Sint32)sizeof(UWord)); + + /* We're going to mix sizes pretty wildly below, so it's easiest to + * turn off size validation. */ + from.setSize(0); + to.setSize(0); + + using vectors = std::initializer_list<std::tuple<x86::Vec, + Sint32, + x86::Inst::Id, + CpuFeatures::X86::Id>>; + for (const auto &spec : vectors{{x86::zmm0, + 8, + x86::Inst::kIdVmovups, + CpuFeatures::X86::kAVX512_VL}, + {x86::zmm0, + 8, + x86::Inst::kIdVmovups, + CpuFeatures::X86::kAVX512_F}, + {x86::ymm0, + 4, + x86::Inst::kIdVmovups, + CpuFeatures::X86::kAVX}, + {x86::xmm0, + 2, + x86::Inst::kIdMovups, + CpuFeatures::X86::kSSE}}) { + const auto &[vector_reg, vector_size, vector_inst, feature] = spec; + + if (!hasCpuFeature(feature)) { + continue; + } + + /* Copy the words inline if we can, otherwise use a loop with the + * largest vector size we're capable of. */ + if (count <= vector_size * 4) { + while (count >= vector_size) { + a.emit(vector_inst, vector_reg, from); + a.emit(vector_inst, to, vector_reg); + + from.addOffset(sizeof(UWord) * vector_size); + to.addOffset(sizeof(UWord) * vector_size); + count -= vector_size; + } + } else { + Sint32 loop_iterations, loop_size; + Label copy_next = a.newLabel(); + + loop_iterations = count / vector_size; + loop_size = loop_iterations * vector_size * sizeof(UWord); + + from.addOffset(loop_size); + to.addOffset(loop_size); + from.setIndex(spill); + to.setIndex(spill); + + mov_imm(spill, -loop_size); + a.bind(copy_next); + { + a.emit(vector_inst, vector_reg, from); + a.emit(vector_inst, to, vector_reg); + + a.add(spill, imm(vector_size * sizeof(UWord))); + a.short_().jne(copy_next); + } + + from.resetIndex(); + to.resetIndex(); + + count %= vector_size; + } + } + + if (count == 1) { + a.mov(spill, from); + a.mov(to, spill); + + count -= 1; + } + + ASSERT(count == 0); + (void)count; + } + public: void embed_rodata(const char *labelName, const char *buff, size_t size); void embed_bss(const char *labelName, size_t size); @@ -979,6 +1086,99 @@ class BeamModuleAssembler : public BeamAssembler { /* Save the last PC for an error. */ size_t last_error_offset = 0; + /* Skip unnecessary moves in mov_arg() and cmp_arg(). */ + size_t last_movarg_offset = 0; + x86::Gp last_movarg_from1, last_movarg_from2; + x86::Mem last_movarg_to1, last_movarg_to2; + + /* Private helper. */ + void preserve__cache(x86::Gp dst) { + last_movarg_offset = a.offset(); + invalidate_cache(dst); + } + + bool is_cache_valid() { + return a.offset() == last_movarg_offset; + } + + void preserve_cache(x86::Gp dst, bool cache_valid) { + if (cache_valid) { + preserve__cache(dst); + } + } + + /* Store CPU register into memory and update the cache. */ + void store_cache(x86::Gp src, x86::Mem dst) { + if (is_cache_valid() && dst != last_movarg_to1) { + /* Something is already cached in the first slot. Use the + * second slot. */ + a.mov(dst, src); + + last_movarg_offset = a.offset(); + last_movarg_to2 = dst; + last_movarg_from2 = src; + } else { + /* Nothing cached yet, or the first slot has the same + * memory address as we will store into. Use the first + * slot and invalidate the second slot. */ + a.mov(dst, src); + + last_movarg_offset = a.offset(); + last_movarg_to1 = dst; + last_movarg_from1 = src; + + last_movarg_to2 = x86::Mem(); + } + } + + void invalidate_cache(x86::Gp dst) { + if (dst == last_movarg_from1) { + last_movarg_to1 = x86::Mem(); + last_movarg_from1 = x86::Gp(); + } + if (dst == last_movarg_from2) { + last_movarg_to2 = x86::Mem(); + last_movarg_from2 = x86::Gp(); + } + } + + x86::Gp cached_reg(x86::Mem mem) { + if (is_cache_valid()) { + if (mem == last_movarg_to1) { + return last_movarg_from1; + } + if (mem == last_movarg_to2) { + return last_movarg_from2; + } + } + return x86::Gp(); + } + + void load_cached(x86::Gp dst, x86::Mem mem) { + if (a.offset() == last_movarg_offset) { + x86::Gp reg = cached_reg(mem); + + if (reg.isValid()) { + /* This memory location is cached. */ + if (reg != dst) { + comment("simplified fetching of BEAM register"); + a.mov(dst, reg); + preserve__cache(dst); + } else { + comment("skipped fetching of BEAM register"); + invalidate_cache(dst); + } + } else { + /* Not cached. Load and preserve the cache. */ + a.mov(dst, mem); + preserve__cache(dst); + } + } else { + /* The cache is invalid. */ + a.mov(dst, mem); + } + } + public: BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, @@ -1043,7 +1243,27 @@ protected: return beam->types.entries[typeIndex].type_union; } - auto getIntRange(const ArgSource &arg) const { + int getExtendedTypeUnion(const ArgSource &arg) const { + if (arg.isLiteral()) { + Eterm literal = + beamfile_get_literal(beam, arg.as<ArgLiteral>().get()); + if (is_binary(literal)) { + return BEAM_TYPE_BITSTRING; + } else if (is_list(literal)) { + return BEAM_TYPE_CONS; + } else if (is_tuple(literal)) { + return BEAM_TYPE_TUPLE; + } else if (is_map(literal)) { + return BEAM_TYPE_MAP; + } else { + return BEAM_TYPE_ANY; + } + } else { + return getTypeUnion(arg); + } + } + + auto getClampedRange(const ArgSource &arg) const { if (arg.isSmall()) { Sint value = arg.as<ArgSmall>().getSigned(); return std::make_pair(value, value); @@ -1053,23 +1273,52 @@ protected: ASSERT(typeIndex < beam->types.count); const auto &entry = beam->types.entries[typeIndex]; - ASSERT(entry.type_union & BEAM_TYPE_INTEGER); - return std::make_pair(entry.min, entry.max); + if (entry.min <= entry.max) { + return std::make_pair(entry.min, entry.max); + } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) { + return std::make_pair(entry.min, MAX_SMALL); + } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) { + return std::make_pair(MIN_SMALL, entry.max); + } else { + return std::make_pair(MIN_SMALL, MAX_SMALL); + } } } + int getSizeUnit(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + + ASSERT(typeIndex < beam->types.count); + return beam->types.entries[typeIndex].size_unit; + } + + bool hasLowerBound(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max); + } + + bool hasUpperBound(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max); + } + bool always_small(const ArgSource &arg) const { if (arg.isSmall()) { return true; } - int type_union = getTypeUnion(arg); - if (type_union == BEAM_TYPE_INTEGER) { - auto [min, max] = getIntRange(arg); - return min <= max; - } else { - return false; - } + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + const auto &entry = beam->types.entries[typeIndex]; + return entry.type_union == BEAM_TYPE_INTEGER && entry.min <= entry.max; } bool always_immediate(const ArgSource &arg) const { @@ -1082,8 +1331,8 @@ protected: } bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const { - int lhs_types = getTypeUnion(lhs); - int rhs_types = getTypeUnion(rhs); + int lhs_types = getExtendedTypeUnion(lhs); + int rhs_types = getExtendedTypeUnion(rhs); /* We can only be certain that the types are the same when there's * one possible type. For example, if one is a number and the other @@ -1132,67 +1381,53 @@ protected: return always_one_of(arg, type_id); } - bool is_sum_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - Sint min, max; - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - min = min1 + min2; - max = max1 + max2; - return IS_SSMALL(min) && IS_SSMALL(max); - } + bool is_sum_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 + min2; + max = max1 + max2; + return IS_SSMALL(min) && IS_SSMALL(max); } - bool is_difference_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - Sint min, max; - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - min = min1 - max2; - max = max1 - min2; - return IS_SSMALL(min) && IS_SSMALL(max); - } + bool is_diff_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 - max2; + max = max1 - min2; + return IS_SSMALL(min) && IS_SSMALL(max); } - bool is_product_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - auto [min1, max1] = getIntRange(LHS); - auto [min2, max2] = getIntRange(RHS); - auto mag1 = std::max(std::abs(min1), std::abs(max1)); - auto mag2 = std::max(std::abs(min2), std::abs(max2)); + bool is_product_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) { + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + auto mag1 = std::max(std::abs(min1), std::abs(max1)); + auto mag2 = std::max(std::abs(min2), std::abs(max2)); - /* - * mag1 * mag2 <= MAX_SMALL - * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) - */ - ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); - return mag2 == 0 || mag1 <= MAX_SMALL / mag2; - } + /* + * mag1 * mag2 <= MAX_SMALL + * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) + */ + ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); + return mag2 == 0 || mag1 <= MAX_SMALL / mag2; } bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) { - /* - * In the code compiled by scripts/diffable, there never - * seems to be any range information for the RHS. Therefore, - * don't bother unless RHS is an immediate small. - */ - if (!(always_small(LHS) && RHS.isSmall())) { + if (!(always_small(LHS) && always_small(RHS))) { return false; } else { - auto [min1, max1] = getIntRange(LHS); - auto rhs_val = RHS.as<ArgSmall>().getSigned(); + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); - if (min1 < 0 || max1 == 0 || rhs_val < 0) { + if (min1 < 0 || max1 == 0 || min2 < 0) { return false; } - return rhs_val < Support::clz(max1) - _TAG_IMMED1_SIZE; + return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE; } } @@ -1202,7 +1437,8 @@ protected: const ArgWord &Live); void emit_gc_test_preserve(const ArgWord &Need, const ArgWord &Live, - x86::Gp term); + const ArgSource &Preserve, + x86::Gp preserve_reg); x86::Mem emit_variable_apply(bool includeI); x86::Mem emit_fixed_apply(const ArgWord &arity, bool includeI); @@ -1211,11 +1447,6 @@ protected: bool skip_fun_test = false, bool skip_arity_test = false); - x86::Gp emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src, - Label next, - Label subbin); - void emit_is_boxed(Label Fail, x86::Gp Src, Distance dist = dLong) { BeamAssembler::emit_is_boxed(Fail, Src, dist); } @@ -1248,10 +1479,12 @@ protected: void emit_error(int code); - x86::Mem emit_bs_get_integer_prologue(Label next, - Label fail, - int flags, - int size); + void emit_bs_get_integer(const ArgRegister &Ctx, + const ArgLabel &Fail, + const ArgWord &Live, + const ArgWord Flags, + int bits, + const ArgRegister &Dst); int emit_bs_get_field_size(const ArgSource &Size, int unit, @@ -1263,6 +1496,40 @@ protected: void emit_bs_get_utf16(const ArgRegister &Ctx, const ArgLabel &Fail, const ArgWord &Flags); + void update_bin_state(x86::Gp bin_offset, + x86::Gp current_byte, + Sint bit_offset, + Sint size, + x86::Gp size_reg); + bool need_mask(const ArgVal Val, Sint size); + void set_zero(Sint effectiveSize); + bool bs_maybe_enter_runtime(bool entered); + void bs_maybe_leave_runtime(bool entered); + void emit_construct_utf8_shared(); + void emit_construct_utf8(const ArgVal &Src, + Sint bit_offset, + bool is_byte_aligned); + + void emit_read_bits(Uint bits, + const x86::Gp bin_base, + const x86::Gp bin_offset, + const x86::Gp bitdata); + void emit_extract_integer(const x86::Gp bitdata, + const x86::Gp tmp, + Uint flags, + Uint bits, + const ArgRegister &Dst); + void emit_extract_binary(const x86::Gp bitdata, + Uint bits, + const ArgRegister &Dst); + void emit_read_integer(const x86::Gp bin_base, + const x86::Gp bin_position, + const x86::Gp tmp, + Uint flags, + Uint bits, + const ArgRegister &Dst); + + UWord bs_get_flags(const ArgVal &val); void emit_raise_exception(); void emit_raise_exception(const ErtsCodeMFA *exp); @@ -1296,6 +1563,8 @@ protected: Eterm fail_value, Eterm succ_value); + void emit_cond_to_bool(uint32_t instId, const ArgRegister &Dst); + void emit_proc_lc_unrequire(void); void emit_proc_lc_require(void); @@ -1358,13 +1627,33 @@ protected: } void cmp_arg(x86::Mem mem, const ArgVal &val, const x86::Gp &spill) { - /* Note that the cast to Sint is necessary to handle negative numbers - * such as NIL. */ - if (val.isImmed() && Support::isInt32((Sint)val.as<ArgImmed>().get())) { - a.cmp(mem, imm(val.as<ArgImmed>().get())); + x86::Gp reg = cached_reg(mem); + + if (reg.isValid()) { + /* Note that the cast to Sint is necessary to handle + * negative numbers such as NIL. */ + if (val.isImmed() && + Support::isInt32((Sint)val.as<ArgImmed>().get())) { + comment("simplified compare of BEAM register"); + a.cmp(reg, imm(val.as<ArgImmed>().get())); + } else if (reg != spill) { + comment("simplified compare of BEAM register"); + mov_arg(spill, val); + a.cmp(reg, spill); + } else { + mov_arg(spill, val); + a.cmp(mem, spill); + } } else { - mov_arg(spill, val); - a.cmp(mem, spill); + /* Note that the cast to Sint is necessary to handle + * negative numbers such as NIL. */ + if (val.isImmed() && + Support::isInt32((Sint)val.as<ArgImmed>().get())) { + a.cmp(mem, imm(val.as<ArgImmed>().get())); + } else { + mov_arg(spill, val); + a.cmp(mem, spill); + } } } @@ -1377,8 +1666,31 @@ protected: } } + void cmp(x86::Gp gp, int64_t val, const x86::Gp &spill) { + if (Support::isInt32(val)) { + a.cmp(gp, imm(val)); + } else if (gp.isGpd()) { + mov_imm(spill, val); + a.cmp(gp, spill.r32()); + } else { + mov_imm(spill, val); + a.cmp(gp, spill); + } + } + + void sub(x86::Gp gp, int64_t val, const x86::Gp &spill) { + if (Support::isInt32(val)) { + a.sub(gp, imm(val)); + } else { + mov_imm(spill, val); + a.sub(gp, spill); + } + } + /* Note: May clear flags. */ void mov_arg(x86::Gp to, const ArgVal &from, const x86::Gp &spill) { + bool valid_cache = is_cache_valid(); + if (from.isBytePtr()) { make_move_patch(to, strings, from.as<ArgBytePtr>().get()); } else if (from.isExport()) { @@ -1390,13 +1702,15 @@ protected: } else if (from.isLiteral()) { make_move_patch(to, literals[from.as<ArgLiteral>().get()].patches); } else if (from.isRegister()) { - a.mov(to, getArgRef(from.as<ArgRegister>())); + auto mem = getArgRef(from.as<ArgRegister>()); + load_cached(to, mem); } else if (from.isWord()) { mov_imm(to, from.as<ArgWord>().get()); } else { ASSERT(!"mov_arg with incompatible type"); } + preserve_cache(to, valid_cache); #ifdef DEBUG /* Explicitly clear flags to catch bugs quicker, it may be very rare * for a certain instruction to load values that would otherwise cause @@ -1415,6 +1729,15 @@ protected: a.mov(spill, imm(val)); a.mov(to, spill); } + } else if (from.isWord()) { + auto val = from.as<ArgWord>().get(); + + if (Support::isInt32((Sint)val)) { + a.mov(to, imm(val)); + } else { + a.mov(spill, imm(val)); + a.mov(to, spill); + } } else { mov_arg(spill, from); a.mov(to, spill); @@ -1424,7 +1747,8 @@ protected: void mov_arg(const ArgVal &to, x86::Gp from, const x86::Gp &spill) { (void)spill; - a.mov(getArgRef(to), from); + auto mem = getArgRef(to); + store_cache(from, mem); } void mov_arg(const ArgVal &to, x86::Mem from, const x86::Gp &spill) { diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl index 9ea270c4f4..61978275ef 100755 --- a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl +++ b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl @@ -31,7 +31,6 @@ my @beam_global_funcs = qw( bs_add_shared bs_create_bin_error_shared bs_size_check_shared - bs_fixed_integer_shared bs_get_tail_shared call_bif_shared call_light_bif_shared @@ -40,6 +39,7 @@ my @beam_global_funcs = qw( call_nif_yield_helper catch_end_shared check_float_error + construct_utf8_shared dispatch_bif dispatch_nif dispatch_return @@ -48,11 +48,15 @@ my @beam_global_funcs = qw( garbage_collect generic_bp_global generic_bp_local + get_sint64_shared debug_bp fconv_shared handle_call_fun_error handle_element_error handle_hd_error + handle_map_get_badkey + handle_map_get_badmap + handle_map_size_error i_band_body_shared i_band_guard_shared i_bif_body_shared @@ -74,11 +78,12 @@ my @beam_global_funcs = qw( i_length_guard_shared i_length_body_shared i_loop_rec_shared - i_new_small_map_lit_shared i_test_yield_shared increment_body_shared int_div_rem_body_shared int_div_rem_guard_shared + is_in_range_shared + is_ge_lt_shared internal_hash_helper minus_body_shared minus_guard_shared @@ -89,6 +94,7 @@ my @beam_global_funcs = qw( process_main raise_exception raise_exception_shared + store_unaligned times_body_shared times_guard_shared unary_minus_body_shared diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index fbde1bd565..83e11dc8ba 100644 --- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp @@ -326,6 +326,8 @@ void BeamModuleAssembler::emit_label(const ArgLabel &Label) { currLabel = rawLabels[Label.get()]; a.bind(currLabel); + + last_movarg_offset = ~0; } void BeamModuleAssembler::emit_aligned_label(const ArgLabel &Label, diff --git a/erts/emulator/beam/jit/x86/generators.tab b/erts/emulator/beam/jit/x86/generators.tab index 4b0b2ad043..52d84b3691 100644 --- a/erts/emulator/beam/jit/x86/generators.tab +++ b/erts/emulator/beam/jit/x86/generators.tab @@ -19,60 +19,6 @@ // %CopyrightEnd% // -// Generate the fastest instruction to fetch an integer from a binary. -gen.get_integer2(Fail, Ms, Live, Size, Unit, Flags, Dst) { - BeamOp* op; - UWord bits; - - $NewBeamOp(S, op); - $NativeEndian(Flags); - if (Size.type == TAG_i) { - if (!beam_load_safe_mul(Size.val, Unit.val, &bits)) { - $BeamOpNameArity(op, jump, 1); - op->a[0] = Fail; - } else if (bits == 8) { - $BeamOpNameArity(op, i_bs_get_integer_8, 4); - op->a[0] = Ms; - op->a[1] = Flags; - op->a[2] = Fail; - op->a[3] = Dst; - } else if (bits == 16) { - $BeamOpNameArity(op, i_bs_get_integer_16, 4); - op->a[0] = Ms; - op->a[1] = Flags; - op->a[2] = Fail; - op->a[3] = Dst; - } else if (bits == 32) { - $BeamOpNameArity(op, i_bs_get_integer_32, 4); - op->a[0] = Ms; - op->a[1] = Flags; - op->a[2] = Fail; - op->a[3] = Dst; - } else if (bits == 64) { - $BeamOpNameArity(op, i_bs_get_integer_64, 5); - op->a[0] = Ms; - op->a[1] = Flags; - op->a[2] = Fail; - op->a[3] = Live; - op->a[4] = Dst; - } else { - goto generic; - } - } else { - generic: - $BeamOpNameArity(op, i_bs_get_integer, 6); - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Live; - op->a[3].type = TAG_u; - op->a[3].val = (Unit.val << 3) | Flags.val; - op->a[4] = Size; - op->a[5] = Dst; - return op; - } - return op; -} - gen.select_tuple_arity(Src, Fail, Size, Rest) { BeamOp* op; BeamOpArg *tmp; @@ -487,27 +433,6 @@ gen.combine_conses(Len, Dst, Hd) { return cons; } -gen.is_eq_exact_literal(Fail, R, C) { - BeamOp* op; - Eterm literal; - Uint tag_test; - - ASSERT(C.type == TAG_q); - literal = beamfile_get_literal(&S->beam, C.val); - ASSERT(is_boxed(literal) || is_list(literal)); - tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK); - - $NewBeamOp(S, op); - $BeamOpNameArity(op, i_is_eq_exact_literal, 4); - op->a[0] = Fail; - op->a[1] = R; - op->a[2] = C; - op->a[3].type = TAG_u; - op->a[3].val = tag_test; - - return op; -} - gen.allocate_heap_zero(Ns, Nh, Live) { BeamOp* alloc; BeamOp* init; @@ -635,6 +560,28 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { Flags.val = flags; $NativeEndian(Flags); op->a[i+fixed_args+3] = Flags; + + /* + * Replace short string segments with integer segments. + * Integer segments can be combined with adjacent integer + * segments for better performance. + */ + if (op->a[i+fixed_args+0].val == am_string) { + Sint num_chars = op->a[i+fixed_args+5].val; + if (num_chars <= 4) { + Sint index = op->a[i+fixed_args+4].val; + const byte* s = S->beam.strings.data + index; + Uint num = 0; + op->a[i+fixed_args+0].val = am_integer; + op->a[i+fixed_args+2].val = 8; + op->a[i+fixed_args+5].val = num_chars; + while (num_chars-- > 0) { + num = num << 8 | *s++; + } + op->a[i+fixed_args+4].type = TAG_i; + op->a[i+fixed_args+4].val = num; + } + } } if (op->a[4].val == am_private_append && Alloc.val != 0) { @@ -652,3 +599,33 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { return op; } + +gen.bs_match(Fail, Ctx, N, List) { + BeamOp* op; + int fixed_args; + int i; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, i_bs_match, 2); + fixed_args = op->arity; + $BeamOpArity(op, (N.val + fixed_args)); + + op->a[0] = Fail; + op->a[1] = Ctx; + + for (i = 0; i < N.val; i++) { + BeamOpArg current; + + current = List[i]; + if (current.type == TAG_o) { + /* An overflow tag (in ensure_at_least or ensure_exactly) + * means that the match will always fail. */ + $BeamOpNameArity(op, jump, 1); + op->a[0] = Fail; + return op; + } + op->a[i+fixed_args] = current; + } + + return op; +} diff --git a/erts/emulator/beam/jit/x86/instr_arith.cpp b/erts/emulator/beam/jit/x86/instr_arith.cpp index 31d6ff2631..b013a00e11 100644 --- a/erts/emulator/beam/jit/x86/instr_arith.cpp +++ b/erts/emulator/beam/jit/x86/instr_arith.cpp @@ -31,6 +31,9 @@ extern "C" #include "erl_bif_table.h" } +/* + * Clobbers ARG1. + */ void BeamModuleAssembler::emit_is_small(Label fail, const ArgSource &Arg, x86::Gp Reg) { @@ -40,7 +43,7 @@ void BeamModuleAssembler::emit_is_small(Label fail, comment("skipped test for small operand since it is always small"); } else if (always_one_of(Arg, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { comment("simplified test for small operand since it is a number"); - a.test(Reg.r32(), imm(TAG_PRIMARY_LIST)); + a.test(Reg.r8(), imm(TAG_PRIMARY_LIST)); a.short_().je(fail); } else { comment("is the operand small?"); @@ -51,6 +54,9 @@ void BeamModuleAssembler::emit_is_small(Label fail, } } +/* + * Clobbers RET, ARG1. + */ void BeamModuleAssembler::emit_are_both_small(Label fail, const ArgSource &LHS, x86::Gp A, @@ -63,9 +69,9 @@ void BeamModuleAssembler::emit_are_both_small(Label fail, always_one_of(RHS, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { comment("simplified test for small operands since both are numbers"); if (always_small(RHS)) { - a.test(A.r32(), imm(TAG_PRIMARY_LIST)); + a.test(A.r8(), imm(TAG_PRIMARY_LIST)); } else if (always_small(LHS)) { - a.test(B.r32(), imm(TAG_PRIMARY_LIST)); + a.test(B.r8(), imm(TAG_PRIMARY_LIST)); } else if (A != RET && B != RET) { a.mov(RETd, A.r32()); a.and_(RETd, B.r32()); @@ -73,9 +79,29 @@ void BeamModuleAssembler::emit_are_both_small(Label fail, } else { a.mov(ARG1d, A.r32()); a.and_(ARG1d, B.r32()); - a.test(ARG1d, imm(TAG_PRIMARY_LIST)); + a.test(ARG1.r8(), imm(TAG_PRIMARY_LIST)); } a.short_().je(fail); + } else if (always_small(LHS)) { + if (A == RET || B == RET) { + emit_is_small(fail, RHS, B); + } else { + comment("is the operand small?"); + a.mov(RETd, B.r32()); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(fail); + } + } else if (always_small(RHS)) { + if (A == RET || B == RET) { + emit_is_small(fail, LHS, A); + } else { + comment("is the operand small?"); + a.mov(RETd, A.r32()); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(fail); + } } else { comment("are both operands small?"); if (A != RET && B != RET) { @@ -121,8 +147,9 @@ void BeamModuleAssembler::emit_i_increment(const ArgRegister &Src, const ArgWord &Val, const ArgRegister &Dst) { ArgVal tagged_val = ArgVal(ArgVal::Immediate, make_small(Val.get())); + bool small_result = is_sum_small_if_args_are_small(Src, tagged_val); - if (is_sum_small(Src, tagged_val)) { + if (always_small(Src) && small_result) { Uint shifted_val = Val.get() << _TAG_IMMED1_SIZE; comment("skipped operand and overflow checks"); @@ -145,13 +172,11 @@ void BeamModuleAssembler::emit_i_increment(const ArgRegister &Src, mov_arg(ARG2, Src); mov_imm(ARG3, Val.get() << _TAG_IMMED1_SIZE); - if (always_one_of(Src, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { comment("simplified test for small operand since it is a number"); a.mov(RET, ARG2); - a.test(RETb, imm(TAG_PRIMARY_LIST)); - a.short_().je(mixed); + emit_is_not_boxed(mixed, RET, dShort); a.add(RET, ARG3); - a.short_().jno(next); } else { a.mov(RETd, ARG2d); a.and_(RETb, imm(_TAG_IMMED1_MASK)); @@ -159,6 +184,12 @@ void BeamModuleAssembler::emit_i_increment(const ArgRegister &Src, a.short_().jne(mixed); a.mov(RET, ARG2); a.add(RET, ARG3); + } + + if (small_result) { + comment("skipped overflow test because the result is always small"); + a.short_().jmp(next); + } else { a.short_().jno(next); } @@ -225,7 +256,9 @@ void BeamModuleAssembler::emit_i_plus(const ArgSource &LHS, const ArgSource &RHS, const ArgLabel &Fail, const ArgRegister &Dst) { - if (is_sum_small(LHS, RHS)) { + bool small_result = is_sum_small_if_args_are_small(LHS, RHS); + + if (always_small(LHS) && always_small(RHS) && small_result) { comment("add without overflow check"); mov_arg(RET, LHS); mov_arg(ARG2, RHS); @@ -242,12 +275,15 @@ void BeamModuleAssembler::emit_i_plus(const ArgSource &LHS, mov_arg(ARG3, RHS); /* Used by erts_mixed_plus in this slot */ emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); - comment("add with overflow check"); a.mov(RET, ARG2); - a.mov(ARG4, ARG3); - a.and_(ARG4, imm(~_TAG_IMMED1_MASK)); - a.add(RET, ARG4); - a.short_().jno(next); + a.and_(RET, imm(~_TAG_IMMED1_MASK)); + a.add(RET, ARG3); + if (small_result) { + comment("skipped overflow test because the result is always small"); + a.short_().jmp(next); + } else { + a.short_().jno(next); + } /* Call mixed addition. */ a.bind(mixed); @@ -319,7 +355,9 @@ void BeamModuleAssembler::emit_i_minus(const ArgSource &LHS, const ArgSource &RHS, const ArgLabel &Fail, const ArgRegister &Dst) { - if (is_difference_small(LHS, RHS)) { + bool small_result = is_diff_small_if_args_are_small(LHS, RHS); + + if (always_small(LHS) && always_small(RHS) && small_result) { comment("subtract without overflow check"); mov_arg(RET, LHS); mov_arg(ARG2, RHS); @@ -337,12 +375,19 @@ void BeamModuleAssembler::emit_i_minus(const ArgSource &LHS, emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); - comment("sub with overflow check"); - a.mov(RET, ARG2); - a.mov(ARG4, ARG3); - a.and_(ARG4, imm(~_TAG_IMMED1_MASK)); - a.sub(RET, ARG4); - a.short_().jno(next); + if (small_result) { + comment("skipped overflow test because the result is always small"); + a.mov(RET, ARG2); + a.and_(ARG3, imm(~_TAG_IMMED1_MASK)); + a.sub(RET, ARG3); + a.short_().jmp(next); + } else { + a.mov(RET, ARG2); + a.mov(ARG4, ARG3); + a.and_(ARG4, imm(~_TAG_IMMED1_MASK)); + a.sub(RET, ARG4); + a.short_().jno(next); + } a.bind(mixed); if (Fail.get() != 0) { @@ -408,8 +453,9 @@ void BeamModuleAssembler::emit_i_unary_minus(const ArgSource &Src, const ArgLabel &Fail, const ArgRegister &Dst) { ArgVal zero = ArgVal(ArgVal::Immediate, make_small(0)); + bool small_result = is_diff_small_if_args_are_small(zero, Src); - if (is_difference_small(zero, Src)) { + if (always_small(Src) && small_result) { comment("negation without overflow test"); mov_arg(ARG2, Src); a.mov(RETd, imm(_TAG_IMMED1_SMALL)); @@ -428,13 +474,18 @@ void BeamModuleAssembler::emit_i_unary_minus(const ArgSource &Src, a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); a.short_().jne(mixed); - comment("negation with overflow test"); /* RETb is now equal to _TAG_IMMED1_SMALL. */ a.movzx(RET, RETb); /* Set RET to make_small(0). */ a.mov(ARG3, ARG2); a.and_(ARG3, imm(~_TAG_IMMED1_MASK)); a.sub(RET, ARG3); - a.short_().jno(next); + + if (small_result) { + comment("skipped overflow test because the result is always small"); + a.short_().jmp(next); + } else { + a.short_().jno(next); + } a.bind(mixed); if (Fail.get() != 0) { @@ -660,8 +711,7 @@ void BeamModuleAssembler::emit_div_rem(const ArgLabel &Fail, } else if (always_one_of(LHS, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { comment("simplified test for small dividend since it is an " "integer"); - a.mov(ARG2d, x86::eax); - a.test(ARG2d, imm(TAG_PRIMARY_LIST)); + a.test(x86::al, imm(TAG_PRIMARY_LIST)); a.short_().je(generic_div); } else { comment("testing for a small dividend"); @@ -673,25 +723,45 @@ void BeamModuleAssembler::emit_div_rem(const ArgLabel &Fail, /* Sign-extend and divide. The result is implicitly placed in * RAX and the remainder in RDX (ARG3). */ - comment("divide with inlined code"); - a.sar(x86::rax, imm(_TAG_IMMED1_SIZE)); - a.cqo(); - a.idiv(ARG6); + if (Support::isPowerOf2(divisor) && + std::get<0>(getClampedRange(LHS)) >= 0) { + int trailing_bits = Support::ctz<Eterm>(divisor); + + if (need_rem) { + Uint mask = Support::lsbMask<Uint>(trailing_bits + + _TAG_IMMED1_SIZE); + mask = (1ULL << (trailing_bits + _TAG_IMMED1_SIZE)) - 1; + comment("optimized rem by replacing with masking"); + mov_imm(x86::rdx, mask); + a.and_(x86::rdx, x86::rax); + } + if (need_div) { + comment("optimized div by replacing with right shift"); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.shr(x86::rax, imm(trailing_bits)); + a.or_(x86::rax, imm(_TAG_IMMED1_SMALL)); + } + } else { + comment("divide with inlined code"); + a.sar(x86::rax, imm(_TAG_IMMED1_SIZE)); + a.cqo(); + a.idiv(ARG6); - if (need_div) { - a.sal(x86::rax, imm(_TAG_IMMED1_SIZE)); - } + if (need_div) { + a.sal(x86::rax, imm(_TAG_IMMED1_SIZE)); + } - if (need_rem) { - a.sal(x86::rdx, imm(_TAG_IMMED1_SIZE)); - } + if (need_rem) { + a.sal(x86::rdx, imm(_TAG_IMMED1_SIZE)); + } - if (need_div) { - a.or_(x86::rax, imm(_TAG_IMMED1_SMALL)); - } + if (need_div) { + a.or_(x86::rax, imm(_TAG_IMMED1_SMALL)); + } - if (need_rem) { - a.or_(x86::rdx, imm(_TAG_IMMED1_SMALL)); + if (need_rem) { + a.or_(x86::rdx, imm(_TAG_IMMED1_SMALL)); + } } if (need_generic) { @@ -862,15 +932,35 @@ void BeamModuleAssembler::emit_i_times(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - if (is_product_small(LHS, RHS)) { + bool small_result = is_product_small_if_args_are_small(LHS, RHS); + + if (always_small(LHS) && always_small(RHS) && small_result) { comment("multiplication without overflow check"); - mov_arg(RET, LHS); - mov_arg(ARG2, RHS); - a.and_(RET, imm(~_TAG_IMMED1_MASK)); - a.sar(ARG2, imm(_TAG_IMMED1_SIZE)); - a.imul(RET, ARG2); + if (RHS.isSmall()) { + Sint factor = RHS.as<ArgSmall>().getSigned(); + + mov_arg(RET, LHS); + a.and_(RET, imm(~_TAG_IMMED1_MASK)); + if (Support::isPowerOf2(factor)) { + int trailing_bits = Support::ctz<Eterm>(factor); + comment("optimized multiplication by replacing with left " + "shift"); + a.shl(RET, imm(trailing_bits)); + } else { + mov_imm(ARG2, factor); + a.imul(RET, ARG2); + } + } else { + mov_arg(RET, LHS); + mov_arg(ARG2, RHS); + a.and_(RET, imm(~_TAG_IMMED1_MASK)); + a.sar(ARG2, imm(_TAG_IMMED1_SIZE)); + a.imul(RET, ARG2); + } + a.or_(RET, imm(_TAG_IMMED1_SMALL)); mov_arg(Dst, RET); + return; } @@ -882,18 +972,10 @@ void BeamModuleAssembler::emit_i_times(const ArgLabel &Fail, if (RHS.isSmall()) { Sint val = RHS.as<ArgSmall>().getSigned(); emit_is_small(mixed, LHS, ARG2); - comment("mul with overflow check, imm RHS"); a.mov(RET, ARG2); a.mov(ARG4, imm(val)); - } else if (LHS.isSmall()) { - Sint val = LHS.as<ArgSmall>().getSigned(); - emit_is_small(mixed, RHS, ARG3); - comment("mul with overflow check, imm LHS"); - a.mov(RET, ARG3); - a.mov(ARG4, imm(val)); } else { emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); - comment("mul with overflow check"); a.mov(RET, ARG2); a.mov(ARG4, ARG3); a.sar(ARG4, imm(_TAG_IMMED1_SIZE)); @@ -901,7 +983,11 @@ void BeamModuleAssembler::emit_i_times(const ArgLabel &Fail, a.and_(RET, imm(~_TAG_IMMED1_MASK)); a.imul(RET, ARG4); - a.short_().jo(mixed); + if (small_result) { + comment("skipped overflow check because the result is always small"); + } else { + a.short_().jo(mixed); + } a.or_(RET, imm(_TAG_IMMED1_SMALL)); a.short_().jmp(next); @@ -993,23 +1079,28 @@ void BeamModuleAssembler::emit_i_band(const ArgSource &LHS, const ArgSource &RHS, const ArgLabel &Fail, const ArgRegister &Dst) { - mov_arg(ARG2, LHS); - mov_arg(RET, RHS); - if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); - a.and_(RET, ARG2); + mov_arg(RET, LHS); + if (RHS.isSmall() && Support::isInt32(RHS.as<ArgSmall>().get())) { + a.and_(RETd, imm(RHS.as<ArgSmall>().get())); + } else if (RHS.isSmall() && + Support::isInt32((Sint)RHS.as<ArgSmall>().get())) { + a.and_(RET, imm(RHS.as<ArgSmall>().get())); + } else { + mov_arg(ARG2, RHS); + a.and_(RET, ARG2); + } mov_arg(Dst, RET); return; } + mov_arg(ARG2, LHS); + mov_arg(RET, RHS); + Label generic = a.newLabel(), next = a.newLabel(); - if (always_small(RHS)) { - emit_is_small(generic, LHS, ARG2); - } else { - emit_are_both_small(generic, LHS, ARG2, RHS, RET); - } + emit_are_both_small(generic, LHS, ARG2, RHS, RET); /* TAG & TAG = TAG, so we don't need to tag it again. */ a.and_(RET, ARG2); @@ -1048,23 +1139,25 @@ void BeamModuleAssembler::emit_i_bor(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - mov_arg(ARG2, LHS); - mov_arg(RET, RHS); - if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); - a.or_(RET, ARG2); + mov_arg(RET, LHS); + if (RHS.isImmed() && Support::isInt32((Sint)RHS.as<ArgSmall>().get())) { + a.or_(RET, imm(RHS.as<ArgSmall>().get())); + } else { + mov_arg(ARG2, RHS); + a.or_(RET, ARG2); + } mov_arg(Dst, RET); return; } + mov_arg(ARG2, LHS); + mov_arg(RET, RHS); + Label generic = a.newLabel(), next = a.newLabel(); - if (always_small(RHS)) { - emit_is_small(generic, LHS, ARG2); - } else { - emit_are_both_small(generic, LHS, ARG2, RHS, RET); - } + emit_are_both_small(generic, LHS, ARG2, RHS, RET); /* TAG | TAG = TAG, so we don't need to tag it again. */ a.or_(RET, ARG2); @@ -1103,25 +1196,27 @@ void BeamModuleAssembler::emit_i_bxor(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - mov_arg(ARG2, LHS); - mov_arg(RET, RHS); - if (always_small(LHS) && always_small(RHS)) { comment("skipped test for small operands since they are always small"); - /* TAG ^ TAG = 0, so we need to tag it again. */ - a.xor_(RET, ARG2); - a.or_(RET, imm(_TAG_IMMED1_SMALL)); + mov_arg(RET, LHS); + if (RHS.isImmed() && Support::isInt32((Sint)RHS.as<ArgSmall>().get())) { + a.xor_(RET, imm(RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_SMALL)); + } else { + /* TAG ^ TAG = 0, so we need to tag it again. */ + mov_arg(ARG2, RHS); + a.xor_(RET, ARG2); + a.or_(RET, imm(_TAG_IMMED1_SMALL)); + } mov_arg(Dst, RET); return; } + mov_arg(ARG2, LHS); + mov_arg(RET, RHS); + Label generic = a.newLabel(), next = a.newLabel(); - if (always_small(RHS)) { - emit_is_small(generic, LHS, ARG2); - } else { - emit_are_both_small(generic, LHS, ARG2, RHS, RET); - } + emit_are_both_small(generic, LHS, ARG2, RHS, RET); /* TAG ^ TAG = 0, so we need to tag it again. */ a.xor_(RET, ARG2); @@ -1341,7 +1436,13 @@ void BeamModuleAssembler::emit_i_bsl(const ArgSource &LHS, mov_arg(RET, LHS); ERTS_CT_ASSERT(_TAG_IMMED1_MASK == _TAG_IMMED1_SMALL); a.xor_(RET, imm(_TAG_IMMED1_MASK)); - a.sal(RET, imm(RHS.as<ArgSmall>().getSigned())); + if (RHS.isSmall()) { + a.shl(RET, imm(RHS.as<ArgSmall>().getSigned())); + } else { + mov_arg(x86::rcx, RHS); + a.shr(x86::rcx, imm(_TAG_IMMED1_SIZE)); + a.shl(RET, x86::cl); + } a.or_(RET, imm(_TAG_IMMED1_SMALL)); mov_arg(Dst, RET); return; diff --git a/erts/emulator/beam/jit/x86/instr_bif.cpp b/erts/emulator/beam/jit/x86/instr_bif.cpp index 470c9a5a02..33e80c6b90 100644 --- a/erts/emulator/beam/jit/x86/instr_bif.cpp +++ b/erts/emulator/beam/jit/x86/instr_bif.cpp @@ -99,7 +99,7 @@ void BeamGlobalAssembler::emit_i_bif_body_shared() { } void BeamModuleAssembler::emit_setup_guard_bif(const std::vector<ArgVal> &args, - const ArgWord &bif) { + const ArgWord &Bif) { bool is_contiguous_mem = false; ASSERT(args.size() > 0 && args.size() <= 3); @@ -125,7 +125,13 @@ void BeamModuleAssembler::emit_setup_guard_bif(const std::vector<ArgVal> &args, } } - mov_arg(ARG4, bif); + if (logger.file()) { + ErtsCodeMFA *mfa = ubif2mfa((void *)Bif.get()); + if (mfa) { + comment("UBIF: %T/%d", mfa->function, mfa->arity); + } + } + mov_arg(ARG4, Bif); } void BeamModuleAssembler::emit_i_bif1(const ArgSource &Src1, @@ -626,6 +632,10 @@ void BeamModuleAssembler::emit_call_light_bif(const ArgWord &Bif, a.mov(RET, imm(Bif.get())); a.lea(ARG3, x86::qword_ptr(entry)); + if (logger.file()) { + BeamFile_ImportEntry *e = &beam->imports.entries[Exp.get()]; + comment("BIF: %T:%T/%d", e->module, e->function, e->arity); + } fragment_call(ga->get_call_light_bif_shared()); } @@ -842,6 +852,10 @@ void BeamModuleAssembler::emit_call_bif_mfa(const ArgAtom &M, e = erts_active_export_entry(M.get(), F.get(), A.get()); ASSERT(e != NULL && e->bif_number != -1); + comment("HBIF: %T:%T/%d", + e->info.mfa.module, + e->info.mfa.function, + A.get()); func = (BeamInstr)bif_table[e->bif_number].f; emit_call_bif(ArgWord(func)); } diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index ab6abff6cc..9a67f17635 100644 --- a/erts/emulator/beam/jit/x86/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -19,6 +19,7 @@ */ #include "beam_asm.hpp" +#include <numeric> extern "C" { @@ -57,12 +58,22 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgSource &Size, a.jmp(fail); return -1; } else { + bool can_fail = true; + mov_arg(RET, Size); - a.mov(ARG3d, RETd); - a.and_(ARG3d, imm(_TAG_IMMED1_MASK)); - a.cmp(ARG3d, imm(_TAG_IMMED1_SMALL)); - a.jne(fail); + if (always_small(Size)) { + auto [min, max] = getClampedRange(Size); + can_fail = + !(0 <= min && (max >> (SMALL_BITS - ERL_UNIT_BITS)) == 0); + comment("simplified segment size checks because " + "the types are known"); + } else { + a.mov(ARG3d, RETd); + a.and_(ARG3d, imm(_TAG_IMMED1_MASK)); + a.cmp(ARG3d, imm(_TAG_IMMED1_SMALL)); + a.jne(fail); + } if (max_size) { ASSERT(Support::isInt32((Sint)make_small(max_size))); @@ -70,19 +81,35 @@ int BeamModuleAssembler::emit_bs_get_field_size(const ArgSource &Size, a.ja(fail); } - if (unit == 1) { + if (unit == 0) { + mov_imm(RET, 0); + } else if (unit == 1) { a.sar(RET, imm(_TAG_IMMED1_SIZE)); - a.js(fail); + if (can_fail) { + a.js(fail); + } + } else if (!can_fail && Support::isPowerOf2(unit)) { + int trailing_bits = Support::ctz<Eterm>(unit); + a.and_(RET, imm(~_TAG_IMMED1_MASK)); + if (trailing_bits < _TAG_IMMED1_SIZE) { + a.sar(RET, imm(_TAG_IMMED1_SIZE - trailing_bits)); + } else if (trailing_bits > _TAG_IMMED1_SIZE) { + a.shl(RET, imm(trailing_bits - _TAG_IMMED1_SIZE)); + } } else { /* Untag the size but don't shift it just yet, we want to fail on * overflow if the final result doesn't fit into a small. */ a.and_(RET, imm(~_TAG_IMMED1_MASK)); - a.js(fail); + if (can_fail) { + a.js(fail); + } /* Size = (Size) * (Unit) */ mov_imm(ARG3, unit); a.mul(ARG3); /* CLOBBERS ARG3! */ - a.jo(fail); + if (can_fail) { + a.jo(fail); + } a.sar(RET, imm(_TAG_IMMED1_SIZE)); } @@ -576,9 +603,10 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgRegister &Src, a.bind(is_binary); { - /* Src is not guaranteed to be inside the live range, so we need to - * stash it during GC. */ - emit_gc_test_preserve(ArgWord(ERL_BIN_MATCHSTATE_SIZE(0)), Live, ARG2); + emit_gc_test_preserve(ArgWord(ERL_BIN_MATCHSTATE_SIZE(0)), + Live, + Src, + ARG2); emit_enter_runtime<Update::eStack | Update::eHeap>(); @@ -650,278 +678,93 @@ void BeamModuleAssembler::emit_i_bs_get_position(const ArgRegister &Ctx, mov_arg(Dst, ARG1); } -/* ARG3 = flags | (size << 3), - * ARG4 = tagged match context */ -void BeamGlobalAssembler::emit_bs_fixed_integer_shared() { - emit_enter_runtime<Update::eStack | Update::eHeap>(); - - a.mov(ARG1, c_p); - /* Unpack size ... */ - a.mov(ARG2, ARG3); - a.shr(ARG2, imm(3)); - /* ... flags. */ - a.and_(ARG3, imm(BSF_ALIGNED | BSF_LITTLE | BSF_SIGNED)); - a.lea(ARG4, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb))); - runtime_call<4>(erts_bs_get_integer_2); - - emit_leave_runtime<Update::eStack | Update::eHeap>(); - - a.ret(); -} - -x86::Mem BeamModuleAssembler::emit_bs_get_integer_prologue(Label next, - Label fail, - int flags, - int size) { - Label aligned = a.newLabel(); - - a.mov(ARG2, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb.offset))); - a.lea(ARG3, x86::qword_ptr(ARG2, size)); - a.cmp(ARG3, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb.size))); - a.ja(fail); - - a.test(ARG2.r8(), imm(CHAR_BIT - 1)); - a.short_().je(aligned); - - /* Actually unaligned reads are quite rare, so we handle everything in a - * shared fragment. */ - mov_imm(ARG3, flags | (size << 3)); - safe_fragment_call(ga->get_bs_fixed_integer_shared()); - - /* The above call can't fail since we work on small numbers and - * bounds-tested above. */ -#ifdef JIT_HARD_DEBUG - a.jmp(next); -#else - a.short_().jmp(next); -#endif - - a.bind(aligned); - { - /* Read base address and convert offset to bytes. */ - a.mov(ARG1, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb.base))); - a.shr(ARG2, imm(3)); - - /* We cannot fail from here on; bump the match context's position. */ - a.mov(emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb.offset)), - ARG3); - - return x86::Mem(ARG1, ARG2, 0, 0, size / 8); - } -} - -void BeamModuleAssembler::emit_i_bs_get_integer_8(const ArgRegister &Ctx, - const ArgWord &Flags, - const ArgLabel &Fail, - const ArgRegister &Dst) { - int flags = Flags.get(); - Label next = a.newLabel(); - x86::Mem address; - - mov_arg(ARG4, Ctx); - - address = emit_bs_get_integer_prologue(next, - resolve_beam_label(Fail), - flags, - 8); - - if (flags & BSF_SIGNED) { - a.movsx(RET, address); - } else { - a.movzx(RET, address); - } - - a.shl(RET, imm(_TAG_IMMED1_SIZE)); - a.or_(RET, imm(_TAG_IMMED1_SMALL)); - - a.bind(next); - mov_arg(Dst, RET); -} - -void BeamModuleAssembler::emit_i_bs_get_integer_16(const ArgRegister &Ctx, - const ArgWord &Flags, - const ArgLabel &Fail, - const ArgRegister &Dst) { - int flags = Flags.get(); - Label next = a.newLabel(); - x86::Mem address; - - mov_arg(ARG4, Ctx); - - address = emit_bs_get_integer_prologue(next, - resolve_beam_label(Fail), - flags, - 16); - - if (flags & BSF_LITTLE) { - if (flags & BSF_SIGNED) { - a.movsx(RET, address); - } else { - a.movzx(RET, address); - } - } else { - if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { - a.movbe(x86::ax, address); - } else { - a.mov(x86::ax, address); - a.xchg(x86::al, x86::ah); - } - - if (flags & BSF_SIGNED) { - a.movsx(RET, x86::ax); - } else { - a.movzx(RET, x86::ax); - } - } - - a.shl(RET, imm(_TAG_IMMED1_SIZE)); - a.or_(RET, imm(_TAG_IMMED1_SMALL)); - - a.bind(next); - mov_arg(Dst, RET); -} - -void BeamModuleAssembler::emit_i_bs_get_integer_32(const ArgRegister &Ctx, - const ArgWord &Flags, - const ArgLabel &Fail, - const ArgRegister &Dst) { - int flags = Flags.get(); - Label next = a.newLabel(); - x86::Mem address; - - mov_arg(ARG4, Ctx); - - address = emit_bs_get_integer_prologue(next, - resolve_beam_label(Fail), - flags, - 32); - - if (flags & BSF_LITTLE) { - if (flags & BSF_SIGNED) { - a.movsxd(RET, address); - } else { - /* Implicitly zero-extends to 64 bits */ - a.mov(RETd, address); - } +void BeamModuleAssembler::emit_bs_get_integer2(const ArgLabel &Fail, + const ArgRegister &Ctx, + const ArgWord &Live, + const ArgSource &Sz, + const ArgWord &Unit, + const ArgWord &Flags, + const ArgRegister &Dst) { + Uint size; + Uint flags = Flags.get(); + + if (flags & BSF_NATIVE) { + flags &= ~BSF_NATIVE; + flags |= BSF_LITTLE; + } + + if (Sz.isSmall() && + (size = Sz.as<ArgSmall>().getUnsigned()) < 8 * sizeof(Uint)) { + /* Segment of a fixed size supported by bs_match. */ + const ArgVal match[] = {ArgAtom(am_ensure_at_least), + ArgWord(size), + Unit, + ArgAtom(am_integer), + Live, + ArgWord(flags), + ArgWord(size), + Unit, + Dst}; + + const Span<ArgVal> args(match, sizeof(match) / sizeof(match[0])); + emit_i_bs_match(Fail, Ctx, args); } else { - if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { - a.movbe(RETd, address); - } else { - a.mov(RETd, address); - a.bswap(RETd); - } - - if (flags & BSF_SIGNED) { - a.movsxd(RET, RETd); - } - } - - a.shl(RET, imm(_TAG_IMMED1_SIZE)); - a.or_(RET, imm(_TAG_IMMED1_SMALL)); - - a.bind(next); - mov_arg(Dst, RET); -} + Label fail = resolve_beam_label(Fail); + int unit = Unit.get(); + + /* Clobbers RET + ARG3, returns a negative result if we always + * fail and further work is redundant. */ + if (emit_bs_get_field_size(Sz, unit, fail, ARG5) >= 0) { + /* This operation can be expensive if a bignum can be + * created because there can be a garbage collection. */ + auto max = std::get<1>(getClampedRange(Sz)); + bool potentially_expensive = + max >= SMALL_BITS || (max * Unit.get()) >= SMALL_BITS; + + mov_arg(ARG3, Ctx); + mov_imm(ARG4, flags); + if (potentially_expensive) { + mov_arg(ARG6, Live); + } else { +#ifdef DEBUG + /* Never actually used. */ + mov_imm(ARG6, 1023); +#endif + } -void BeamModuleAssembler::emit_i_bs_get_integer_64(const ArgRegister &Ctx, - const ArgWord &Flags, - const ArgLabel &Fail, - const ArgWord &Live, - const ArgRegister &Dst) { - int flags = Flags.get(); - Label next = a.newLabel(); - x86::Mem address; + if (potentially_expensive) { + emit_enter_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } else { + comment("simplified entering runtime because result is always " + "small"); + emit_enter_runtime(); + } - mov_arg(ARG4, Ctx); + a.mov(ARG1, c_p); + if (potentially_expensive) { + load_x_reg_array(ARG2); + } else { +#ifdef DEBUG + /* Never actually used. */ + mov_imm(ARG2, 0); +#endif + } + runtime_call<6>(beam_jit_bs_get_integer); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgWord(BIG_UINT_HEAP_SIZE), Live, ARG4); + if (potentially_expensive) { + emit_leave_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } else { + emit_leave_runtime(); + } - address = emit_bs_get_integer_prologue(next, - resolve_beam_label(Fail), - flags, - 64); + emit_test_the_non_value(RET); + a.je(fail); - if (flags & BSF_LITTLE) { - a.mov(RET, address); - } else { - if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { - a.movbe(RET, address); - } else { - a.mov(RET, address); - a.bswap(RET); + mov_arg(Dst, RET); } } - - a.mov(ARG1, RET); - a.mov(ARG2, RET); - - /* Speculatively make a small out of the result even though it might not - * be one, and jump to the next instruction if it is. */ - a.shl(RET, imm(_TAG_IMMED1_SIZE)); - a.or_(RET, imm(_TAG_IMMED1_SMALL)); - - if (flags & BSF_SIGNED) { - a.sar(ARG2, imm(SMALL_BITS - 1)); - a.add(ARG2, imm(1)); - a.cmp(ARG2, imm(1)); - a.jbe(next); - } else { - a.shr(ARG2, imm(SMALL_BITS - 1)); - a.jz(next); - } - - emit_enter_runtime(); - - a.mov(ARG2, HTOP); - if (flags & BSF_SIGNED) { - runtime_call<2>(small_to_big); - } else { - runtime_call<2>(uword_to_big); - } - a.add(HTOP, imm(sizeof(Eterm) * BIG_UINT_HEAP_SIZE)); - - emit_leave_runtime(); - - a.bind(next); - mov_arg(Dst, RET); -} - -void BeamModuleAssembler::emit_i_bs_get_integer(const ArgRegister &Ctx, - const ArgLabel &Fail, - const ArgWord &Live, - const ArgWord &FlagsAndUnit, - const ArgSource &Sz, - const ArgRegister &Dst) { - Label fail; - int unit; - - fail = resolve_beam_label(Fail); - unit = FlagsAndUnit.get() >> 3; - - /* Clobbers RET + ARG3, returns a negative result if we always fail and - * further work is redundant. */ - if (emit_bs_get_field_size(Sz, unit, fail, ARG5) >= 0) { - mov_arg(ARG3, Ctx); - mov_arg(ARG4, FlagsAndUnit); - mov_arg(ARG6, Live); - - emit_enter_runtime<Update::eReductions | Update::eStack | - Update::eHeap>(); - - a.mov(ARG1, c_p); - load_x_reg_array(ARG2); - runtime_call<6>(beam_jit_bs_get_integer); - - emit_leave_runtime<Update::eReductions | Update::eStack | - Update::eHeap>(); - - emit_test_the_non_value(RET); - a.je(fail); - - mov_arg(Dst, RET); - } } void BeamModuleAssembler::emit_bs_test_tail2(const ArgLabel &Fail, @@ -962,9 +805,7 @@ void BeamModuleAssembler::emit_i_bs_get_binary_all2(const ArgRegister &Ctx, mov_arg(ARG1, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, ARG1); + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, Ctx, ARG1); a.mov(RET, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.size))); a.sub(RET, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.offset))); @@ -1016,9 +857,7 @@ void BeamModuleAssembler::emit_bs_get_tail(const ArgRegister &Ctx, const ArgWord &Live) { mov_arg(ARG1, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, ARG1); + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, Ctx, ARG1); safe_fragment_call(ga->get_bs_get_tail_shared()); @@ -1044,7 +883,6 @@ void BeamModuleAssembler::emit_i_bs_skip_bits2(const ArgRegister &Ctx, Label fail; fail = resolve_beam_label(Fail); - if (emit_bs_get_field_size(Bits, Unit.get(), fail, RET) >= 0) { emit_bs_skip_bits(Fail, Ctx); } @@ -1076,9 +914,10 @@ void BeamModuleAssembler::emit_i_bs_get_binary2(const ArgRegister &Ctx, mov_arg(ARG4, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to - * stash it during GC. */ - emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), Live, ARG4); + emit_gc_test_preserve(ArgWord(EXTRACT_SUB_BIN_HEAP_NEED), + Live, + Ctx, + ARG4); emit_enter_runtime<Update::eHeap>(); @@ -1111,9 +950,7 @@ void BeamModuleAssembler::emit_i_bs_get_float2(const ArgRegister &Ctx, mov_arg(ARG4, Ctx); - /* Ctx is not guaranteed to be inside the live range, so we need to stash - * it during GC. */ - emit_gc_test_preserve(ArgWord(FLOAT_SIZE_OBJECT), Live, ARG4); + emit_gc_test_preserve(ArgWord(FLOAT_SIZE_OBJECT), Live, Ctx, ARG4); if (emit_bs_get_field_size(Sz, unit, fail, ARG2, 64) >= 0) { emit_enter_runtime<Update::eHeap>(); @@ -1286,8 +1123,8 @@ void BeamModuleAssembler::emit_validate_unicode(Label next, Label fail, x86::Gp value) { a.mov(ARG3d, value.r32()); - a.and_(ARG3d, imm(_TAG_IMMED1_MASK)); - a.cmp(ARG3d, imm(_TAG_IMMED1_SMALL)); + a.and_(ARG3d.r8(), imm(_TAG_IMMED1_MASK)); + a.cmp(ARG3d.r8(), imm(_TAG_IMMED1_SMALL)); a.jne(fail); a.cmp(value, imm(make_small(0xD800UL))); @@ -1587,10 +1424,52 @@ void BeamGlobalAssembler::emit_bs_create_bin_error_shared() { a.jmp(labels[raise_exception_shared]); } +/* + * ARG1 = tagged bignum term + * + * On return, Z is set if ARG1 is not a bignum. Otherwise, Z is clear and + * ARG1 is the 64 least significant bits of the bignum. + */ +void BeamGlobalAssembler::emit_get_sint64_shared() { + Label success = a.newLabel(); + Label fail = a.newLabel(); + + emit_is_boxed(fail, ARG1); + x86::Gp boxed_ptr = emit_ptr_val(ARG4, ARG1); + a.mov(ARG2, emit_boxed_val(boxed_ptr)); + a.mov(ARG3, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.and_(ARG2, imm(_TAG_HEADER_MASK)); + a.cmp(ARG2, imm(POS_BIG_SUBTAG)); + a.je(success); + + a.cmp(ARG2, imm(NEG_BIG_SUBTAG)); + a.jne(fail); + + a.neg(ARG3); + + a.bind(success); + { + a.mov(ARG1, ARG3); + /* Clear Z flag. + * + * ARG2 is known to be POS_BIG_SUBTAG or NEG_BIG_SUBTAG at this point. + */ + ERTS_CT_ASSERT(POS_BIG_SUBTAG != 0 && NEG_BIG_SUBTAG != 0); + a.test(ARG2, ARG2); + a.ret(); + } + + a.bind(fail); + { + a.xor_(ARG2, ARG2); /* Set Z flag */ + a.ret(); + } +} + struct BscSegment { BscSegment() : type(am_false), unit(1), flags(0), src(ArgNil()), size(ArgNil()), - error_info(0), effectiveSize(-1) { + error_info(0), effectiveSize(-1), action(action::DIRECT) { } Eterm type; @@ -1601,8 +1480,484 @@ struct BscSegment { Uint error_info; Sint effectiveSize; + + /* Here are sub actions for storing integer segments. + * + * We use the ACCUMULATE_FIRST and ACCUMULATE actions to shift the + * values of segments with known, small sizes (no more than 64 bits) + * into an accumulator register. + * + * When no more segments can be accumulated, the STORE action is + * used to store the value of the accumulator into the binary. + * + * The DIRECT action is used when it is not possible to use the + * accumulator (for unknown or too large sizes). + */ + enum class action { DIRECT, ACCUMULATE_FIRST, ACCUMULATE, STORE } action; }; +static std::vector<BscSegment> bs_combine_segments( + const std::vector<BscSegment> segments) { + std::vector<BscSegment> segs; + + for (auto seg : segments) { + switch (seg.type) { + case am_integer: { + if (!(0 < seg.effectiveSize && seg.effectiveSize <= 64)) { + /* Unknown or too large size. Handle using the default + * DIRECT action. */ + segs.push_back(seg); + continue; + } + + if (seg.flags & BSF_LITTLE || segs.size() == 0 || + segs.back().action == BscSegment::action::DIRECT) { + /* There are no previous compatible ACCUMULATE / STORE + * actions. Create the first ones. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + continue; + } + + auto prev = segs.back(); + if (prev.flags & BSF_LITTLE) { + /* Little-endian segments cannot be combined with other + * segments. Create new ACCUMULATE_FIRST / STORE actions. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + continue; + } + + /* The current segment is compatible with the previous + * segment. Try combining them. */ + if (prev.effectiveSize + seg.effectiveSize <= 64) { + /* The combined values of the segments fits in the + * accumulator. Insert an ACCUMULATE action for the + * current segment before the pre-existing STORE + * action. */ + segs.pop_back(); + prev.effectiveSize += seg.effectiveSize; + seg.action = BscSegment::action::ACCUMULATE; + segs.push_back(seg); + segs.push_back(prev); + } else { + /* The size exceeds 64 bits. Can't combine. */ + seg.action = BscSegment::action::ACCUMULATE_FIRST; + segs.push_back(seg); + seg.action = BscSegment::action::STORE; + segs.push_back(seg); + } + break; + } + default: + segs.push_back(seg); + break; + } + } + return segs; +} + +/* + * In: + * bin_offset = if valid, register to store the lower 32 bits + * of the bit offset into the binary + * bin_ptr = register to store pointer to current byte in + * bit_offset = current bit offset into binary, or -1 if unknown + * size = size of segment to be constructed + * (ignored if size_reg is valid register) + * size_reg = if a valid register, it contains the size of + * the segment to be constructed + * + * Out: + * bin_offset register = the lower 32 bits of the bit offset + * into the binary + * bin_ptr register = pointer to current byte + * + * Preserves all other registers except RET. + */ +void BeamModuleAssembler::update_bin_state(x86::Gp bin_offset, + x86::Gp current_byte, + Sint bit_offset, + Sint size, + x86::Gp size_reg) { + const int x_reg_offset = offsetof(ErtsSchedulerRegisters, x_reg_array.d); + const int cur_bin_base = + offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + + offsetof(struct erl_bits_state, erts_current_bin_); + const int cur_bin_offset = + offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + + offsetof(struct erl_bits_state, erts_bin_offset_); + + x86::Mem mem_bin_base = + x86::Mem(registers, cur_bin_base - x_reg_offset, sizeof(UWord)); + x86::Mem mem_bin_offset = + x86::Mem(registers, cur_bin_offset - x_reg_offset, sizeof(UWord)); + + if (bit_offset % 8 != 0 || !Support::isInt32(bit_offset + size)) { + /* The bit offset is unknown or not byte-aligned. Alternatively, + * the sum of bit_offset and size does not fit in an immediate. */ + a.mov(current_byte, mem_bin_offset); + a.mov(RET, mem_bin_base); + + if (bin_offset.isValid()) { + a.mov(bin_offset.r32(), current_byte.r32()); + } + if (size_reg.isValid()) { + a.add(mem_bin_offset, size_reg); + } else { + a.add(mem_bin_offset, imm(size)); + } + a.shr(current_byte, imm(3)); + a.add(current_byte, RET); + } else { + ASSERT(size >= 0 || size_reg.isValid()); + ASSERT(bit_offset % 8 == 0); + + comment("optimized updating of binary construction state"); + a.mov(current_byte, mem_bin_base); + if (bit_offset) { + a.add(current_byte, imm(bit_offset >> 3)); + } + if (size_reg.isValid()) { + a.add(mem_bin_offset, size_reg); + } else { + a.mov(mem_bin_offset, imm(bit_offset + size)); + } + } +} + +bool BeamModuleAssembler::need_mask(const ArgVal Val, Sint size) { + if (size == 64) { + return false; + } else { + auto [min, max] = getClampedRange(Val); + return !(0 <= min && max >> size == 0); + } +} + +/* + * The size of the segment is assumed to be in ARG3. + */ +void BeamModuleAssembler::set_zero(Sint effectiveSize) { + update_bin_state(ARG2, ARG1, -1, -1, ARG3); + + mov_imm(RET, 0); + + if (effectiveSize < 0 || effectiveSize > 128) { + /* Size is unknown or greater than 128. Modern CPUs have an + * enhanced "rep stosb" instruction that in most circumstances + * is the fastest way to clear blocks of more than 128 + * bytes. */ + Label done = a.newLabel(); + + if (effectiveSize < 0) { + a.test(ARG3, ARG3); + a.short_().jz(done); + } + + if (ARG1 != x86::rdi) { + a.mov(x86::rdi, ARG1); + } + a.mov(x86::rcx, ARG3); + a.add(x86::rcx, imm(7)); + a.shr(x86::rcx, imm(3)); + a.rep().stosb(); + + a.bind(done); + } else { + /* The size is known and it is at most 128 bits. */ + Uint offset = 0; + + ASSERT(0 <= effectiveSize && effectiveSize <= 128); + + if (effectiveSize == 128) { + a.mov(x86::Mem(ARG1, offset, 8), RET); + offset += 8; + } + + if (effectiveSize >= 64) { + a.mov(x86::Mem(ARG1, offset, 8), RET); + offset += 8; + } + + if ((effectiveSize & 63) >= 32) { + a.mov(x86::Mem(ARG1, offset, 4), RETd); + offset += 4; + } + + if ((effectiveSize & 31) >= 16) { + a.mov(x86::Mem(ARG1, offset, 2), RET.r16()); + offset += 2; + } + + if ((effectiveSize & 15) >= 8) { + a.mov(x86::Mem(ARG1, offset, 1), RET.r8()); + offset += 1; + } + + if ((effectiveSize & 7) > 0) { + a.mov(x86::Mem(ARG1, offset, 1), RET.r8()); + } + } +} + +/* + * In: + * + * ARG3 = valid unicode code point (=> 0x80) to encode + * + * Out: + * + * ARG3d = the code point encoded in UTF-8. + * ARG2 = number of bits of result (16, 24, or 32) + * + * Clobbers RET and the other ARG* registers. + */ +void BeamGlobalAssembler::emit_construct_utf8_shared() { + Label more_than_two_bytes = a.newLabel(); + Label four_bytes = a.newLabel(); + const x86::Gp tmp1 = ARG1; + const x86::Gp tmp2 = ARG2; + const x86::Gp value = ARG3; + const x86::Gp num_bits = ARG2; + + a.mov(RETd, value.r32()); + a.and_(RETd, imm(0x3f)); + + a.cmp(value.r32(), imm(0x800)); + a.jae(more_than_two_bytes); + + a.shl(RETd, imm(8)); + + a.shr(value, imm(6)); + + a.or_(value.r32(), RETd); + a.or_(value.r32(), imm(0x80c0)); + + mov_imm(num_bits, 16); + a.ret(); + + /* Test whether the value should be encoded in four bytes. */ + a.bind(more_than_two_bytes); + a.cmp(value.r32(), imm(0x10000)); + a.jae(four_bytes); + + /* Encode Unicode code point in three bytes. */ + a.shl(RETd, imm(16)); + + a.lea(tmp1.r32(), x86::Mem(0ULL, ARG3, 2, 0)); + a.and_(tmp1.r32(), imm(0x3f00)); + + a.shr(value.r32(), imm(12)); + a.or_(value.r32(), tmp1.r32()); + a.or_(value.r32(), RETd); + a.or_(value.r32(), imm(0x8080e0)); + + mov_imm(num_bits, 24); + a.ret(); + + /* Encode Unicode code point in four bytes. */ + a.bind(four_bytes); + a.shl(RETd, imm(24)); + + a.mov(tmp1.r32(), value.r32()); + a.shl(tmp1.r32(), imm(10)); + a.and_(tmp1.r32(), imm(0x3f0000)); + + a.mov(tmp2.r32(), value.r32()); + a.shr(tmp2.r32(), imm(4)); + a.and_(tmp2.r32(), imm(0x3f00)); + + a.shr(value.r32(), imm(18)); + + a.or_(value.r32(), RETd); + a.or_(value.r32(), tmp1.r32()); + a.or_(value.r32(), tmp2.r32()); + a.or_(value.r32(), imm(0xffffffff808080f0)); + + mov_imm(num_bits, 32); + a.ret(); +} + +void BeamModuleAssembler::emit_construct_utf8(const ArgVal &Src, + Sint bit_offset, + bool is_byte_aligned) { + Label prepare_store = a.newLabel(); + Label store = a.newLabel(); + Label next = a.newLabel(); + +#ifdef WIN32 + const x86::Gp bin_ptr = ARG4; + const x86::Gp bin_offset = is_byte_aligned ? x86::Gp() : ARG1; +#else + const x86::Gp bin_ptr = ARG1; + const x86::Gp bin_offset = is_byte_aligned ? x86::Gp() : ARG4; +#endif + ASSERT(!bin_offset.isValid() || bin_offset == x86::rcx); + + /* The following two registers must be the same as + * emit_construct_utf8_shared() expects. */ + const x86::Gp code_point = ARG3; + const x86::Gp size_reg = ARG2; + + comment("construct utf8 segment"); + + mov_arg(code_point, Src); + a.shr(code_point.r32(), imm(_TAG_IMMED1_SIZE)); + mov_imm(size_reg, 8); + a.cmp(code_point, imm(0x80)); + a.jb(prepare_store); + + safe_fragment_call(ga->get_construct_utf8_shared()); + + a.bind(prepare_store); + + update_bin_state(bin_offset, bin_ptr, bit_offset, -1, size_reg); + + if (!is_byte_aligned) { + /* Bit offset is unknown and is not known to be + * byte aligned. Must test alignment. */ + a.and_(bin_offset.r32(), imm(7)); + a.je(store); + + /* We must combine the last partial byte with the UTF-8 + * encoded code point. */ + + a.movzx(RETd, x86::byte_ptr(bin_ptr)); + + a.bswap(code_point); + a.shr(code_point, bin_offset.r8()); + a.bswap(code_point); + + a.shl(RETd, bin_offset.r8()); + a.and_(RETd, imm(~0xff)); + a.shr(RETd, bin_offset.r8()); + + a.or_(code_point, RET); + + a.add(size_reg.r32(), imm(8)); + } + + a.bind(store); + if (bit_offset % (4 * 8) == 0) { + /* This segment is aligned on a 4-byte boundary. This implies + * that a 4-byte write will be inside the allocated binary. */ + a.mov(x86::dword_ptr(bin_ptr), code_point.r32()); + } else { + Label do_store_1 = a.newLabel(); + Label do_store_2 = a.newLabel(); + + /* Unsuitable or unknown alignment. We must be careful not + * to write beyound the allocated end of the binary. */ + a.cmp(size_reg.r8(), imm(8)); + a.short_().jne(do_store_1); + + a.mov(x86::byte_ptr(bin_ptr), code_point.r8()); + a.short_().jmp(next); + + a.bind(do_store_1); + a.cmp(size_reg.r8(), imm(24)); + a.ja(do_store_2); + + a.mov(x86::word_ptr(bin_ptr), code_point.r16()); + a.cmp(size_reg.r8(), imm(16)); + a.short_().je(next); + + a.shr(code_point.r32(), imm(16)); + a.mov(x86::byte_ptr(bin_ptr, 2), code_point.r8()); + a.short_().jmp(next); + + a.bind(do_store_2); + a.mov(x86::dword_ptr(bin_ptr), code_point.r32()); + + if (!is_byte_aligned) { + a.cmp(size_reg.r8(), imm(32)); + a.je(next); + + a.shr(code_point, imm(32)); + a.mov(x86::byte_ptr(bin_ptr, 4), code_point.r8()); + } + } + + a.bind(next); +} +/* + * In: + * ARG1 = pointer to current byte + * ARG3 = bit offset + * ARG4 = number of bits to write + * ARG5 = data to write + */ +void BeamGlobalAssembler::emit_store_unaligned() { + Label loop = a.newLabel(); + Label done = a.newLabel(); + const x86::Gp bin_ptr = ARG1; + const x86::Gp left_bit_offset = ARG3; + const x86::Gp right_bit_offset = ARG2; + const x86::Gp num_bits = ARG4; + const x86::Gp bitdata = ARG5; + + a.movzx(RETd, x86::byte_ptr(bin_ptr)); + + a.xchg(left_bit_offset, x86::rcx); + + a.mov(right_bit_offset, bitdata); + a.and_(right_bit_offset, imm(0xff)); + a.shr(right_bit_offset, x86::cl); + + a.shl(RETd, x86::cl); + a.and_(RETd, imm(~0xff)); + a.shr(RETd, x86::cl); + + a.xchg(left_bit_offset, x86::rcx); + + a.or_(RETd, ARG2d); + a.mov(byte_ptr(ARG1), RETb); + a.add(ARG1, imm(1)); + + mov_imm(right_bit_offset, 8); + a.sub(right_bit_offset, left_bit_offset); + + a.xchg(right_bit_offset, x86::rcx); + a.bswap(bitdata); + a.shl(bitdata, x86::cl); + a.xchg(right_bit_offset, x86::rcx); + + a.sub(ARG4, right_bit_offset); + a.jle(done); + + a.bind(loop); + a.rol(bitdata, imm(8)); + a.mov(byte_ptr(ARG1), bitdata.r8()); + a.add(ARG1, imm(1)); + a.sub(num_bits, imm(8)); + a.jg(loop); + + a.bind(done); + a.ret(); +} + +bool BeamModuleAssembler::bs_maybe_enter_runtime(bool entered) { + if (!entered) { + comment("enter runtime"); + emit_enter_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } + return true; +} + +void BeamModuleAssembler::bs_maybe_leave_runtime(bool entered) { + if (entered) { + comment("leave runtime"); + emit_leave_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } +} + void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, const ArgWord &Alloc, const ArgWord &Live0, @@ -1611,10 +1966,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, Uint num_bits = 0; std::size_t n = args.size(); std::vector<BscSegment> segments; - Label error = a.newLabel(); - Label past_error = a.newLabel(); + Label error; /* Intentionally uninitialized */ ArgWord Live = Live0; x86::Gp sizeReg; + Sint allocated_size = -1; + bool need_error_handler = false; + bool runtime_entered = false; /* * Collect information about each segment and calculate sizes of @@ -1660,12 +2017,43 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, seg.error_info = beam_jit_set_bsc_segment_op(bsc_segment, bsc_op); /* + * Test whether we can omit the code for the error handler. + */ + switch (seg.type) { + case am_append: + if (std::gcd(seg.unit, getSizeUnit(seg.src)) != seg.unit) { + need_error_handler = true; + } + break; + case am_binary: + if (!(seg.size.isAtom() && seg.size.as<ArgAtom>().get() == am_all && + std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit)) { + need_error_handler = true; + } + break; + case am_integer: + if (!always_one_of(seg.src, BEAM_TYPE_INTEGER)) { + need_error_handler = true; + } + break; + case am_private_append: + case am_string: + break; + default: + need_error_handler = true; + break; + } + + /* * As soon as we have entered runtime mode, Y registers can no * longer be accessed in the usual way. Therefore, if the source - * and/or size are in Y register, copy them to X registers. + * and/or size are in Y registers, copy them to X registers. Be + * careful to preserve any associated type information. */ if (seg.src.isYRegister()) { - ArgVal reg = ArgXRegister(Live.get()); + auto reg = + seg.src.as<ArgYRegister>().copy<ArgXRegister>(Live.get()); + ASSERT(reg.typeIndex() == seg.src.as<ArgYRegister>().typeIndex()); mov_arg(reg, seg.src); Live = Live + 1; @@ -1673,7 +2061,9 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } if (seg.size.isYRegister()) { - ArgVal reg = ArgXRegister(Live.get()); + auto reg = + seg.size.as<ArgYRegister>().copy<ArgXRegister>(Live.get()); + ASSERT(reg.typeIndex() == seg.size.as<ArgYRegister>().typeIndex()); mov_arg(reg, seg.size); Live = Live + 1; @@ -1694,16 +2084,64 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, if (seg.effectiveSize < 0 && seg.type != am_append && seg.type != am_private_append) { sizeReg = FCALLS; + need_error_handler = true; } segments.insert(segments.end(), seg); } - emit_enter_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); + /* + * Test whether a heap binary of fixed size will result from the + * construction. If so, allocate and construct the binary now + * before entering the runtime mode. + */ + if (!sizeReg.isValid() && num_bits % 8 == 0 && + num_bits / 8 <= ERL_ONHEAP_BIN_LIMIT && segments[0].type != am_append && + segments[0].type != am_private_append) { + const int x_reg_offset = + offsetof(ErtsSchedulerRegisters, x_reg_array.d); + const int cur_bin_base = + offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + + offsetof(struct erl_bits_state, erts_current_bin_); + const int cur_bin_offset = + offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + + offsetof(struct erl_bits_state, erts_bin_offset_); + x86::Mem mem_bin_base = + x86::qword_ptr(registers, cur_bin_base - x_reg_offset); + x86::Mem mem_bin_offset = + x86::qword_ptr(registers, cur_bin_offset - x_reg_offset); + Uint num_bytes = num_bits / 8; + + comment("allocate heap binary"); + allocated_size = (num_bytes + 7) & (-8); + + /* Ensure that there is enough room on the heap. */ + Uint need = heap_bin_size(num_bytes) + Alloc.get(); + emit_gc_test(ArgWord(0), ArgWord(need), Live); + + /* Create the heap binary. */ + a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED)); + a.mov(TMP_MEM1q, RET); + a.mov(x86::qword_ptr(HTOP, 0), imm(header_heap_bin(num_bytes))); + a.mov(x86::qword_ptr(HTOP, sizeof(Eterm)), imm(num_bytes)); + + /* Initialize the erl_bin_state struct. */ + a.add(HTOP, imm(sizeof(Eterm[2]))); + a.mov(mem_bin_base, HTOP); + a.mov(mem_bin_offset, imm(0)); + + /* Update HTOP. */ + a.add(HTOP, imm(allocated_size)); + } + + if (!need_error_handler) { + comment("(cannot fail)"); + } else { + Label past_error = a.newLabel(); + + runtime_entered = bs_maybe_enter_runtime(false); + a.short_().jmp(past_error); - a.short_().jmp(past_error); - a.bind(error); - { /* * ARG1 = optional bad size value; valid if BSC_VALUE_ARG1 is set in * ARG4 @@ -1713,17 +2151,18 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, * * ARG4 = packed error information */ + error = a.newLabel(); + a.bind(error); + bs_maybe_leave_runtime(runtime_entered); comment("handle error"); - emit_leave_runtime<Update::eReductions | Update::eStack | - Update::eHeap>(); if (Fail.get() != 0) { a.jmp(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_bs_create_bin_error_shared()); } - } - a.bind(past_error); + a.bind(past_error); + } /* We count the total number of bits in an unsigned integer. To * avoid having to check for overflow when adding to the counter, @@ -1748,12 +2187,50 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, if (seg.size.isAtom() && seg.size.as<ArgAtom>().get() == am_all && seg.type == am_binary) { comment("size of an entire binary"); + runtime_entered = bs_maybe_enter_runtime(runtime_entered); mov_arg(ARG1, seg.src); - runtime_call<1>(beam_jit_bs_bit_size); + if (exact_type(seg.src, BEAM_TYPE_BITSTRING)) { - comment("skipped check for success since the source " - "is always a bit string"); + auto unit = getSizeUnit(seg.src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + + if (is_bitstring) { + comment("inlined size code because the value is always " + "a bitstring"); + } else { + comment("inlined size code because the value is always " + "a binary"); + } + + a.mov(ARG2, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + + if (is_bitstring) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + } + + a.lea(sizeReg, x86::Mem(sizeReg, ARG2, 3, 0, 1)); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const auto diff_mask = + _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); + a.test(RETb, imm(diff_mask)); + a.short_().jz(not_sub_bin); + + a.movzx(RETd, + emit_boxed_val(boxed_ptr, + offsetof(ErlSubBin, bitsize), + 1)); + a.add(sizeReg, RET); + + a.bind(not_sub_bin); + } } else { + runtime_call<1>(beam_jit_bs_bit_size); if (Fail.get() == 0) { mov_arg(ARG1, seg.src); mov_imm(ARG4, @@ -1764,17 +2241,15 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } a.test(RET, RET); a.js(error); + a.add(sizeReg, RET); } - a.add(sizeReg, RET); } else if (seg.unit != 0) { bool can_fail = true; comment("size binary/integer/float/string"); - if (always_small(seg.size)) { - auto min = std::get<0>(getIntRange(seg.size)); - if (min >= 0) { - can_fail = false; - } + if (std::get<0>(getClampedRange(seg.size)) >= 0) { + /* Can't fail if size is always positive. */ + can_fail = false; } if (can_fail && Fail.get() == 0) { @@ -1792,7 +2267,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } else if (always_one_of(seg.size, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { comment("simplified test for small size since it is a number"); - a.test(ARG1d, imm(TAG_PRIMARY_LIST)); + a.test(ARG1.r8(), imm(TAG_PRIMARY_LIST)); a.je(error); } else { a.mov(RETd, ARG1d); @@ -1827,23 +2302,60 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, comment("size utf8"); mov_arg(ARG1, seg.src); + if (Fail.get() == 0) { + mov_imm(ARG4, + beam_jit_update_bsc_reason_info(seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG1)); + } + + if (always_small(seg.src)) { + comment("skipped test for small value since it is always " + "small"); + } else if (always_one_of(seg.src, + BEAM_TYPE_INTEGER | + BEAM_TYPE_MASK_BOXED)) { + comment("simplified test for small operand since other " + "types are boxed"); + emit_is_not_boxed(error, ARG1); + } else { + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.jne(error); + } + mov_imm(RET, 0); - a.mov(RETb, imm(1 * 8)); + a.mov(RETb, imm(1)); a.cmp(ARG1, imm(make_small(0x80UL))); - a.short_().jl(next); + a.short_().jb(next); - a.mov(RETb, imm(2 * 8)); + a.mov(RETb, imm(2)); a.cmp(ARG1, imm(make_small(0x800UL))); - a.short_().jl(next); + a.short_().jb(next); - a.mov(RETb, imm(3 * 8)); - a.cmp(ARG1, imm(make_small(0x10000UL))); - a.short_().jl(next); + /* Ensure that the value is not in the invalid range + * 0xD800 through 0xDFFF. */ + a.mov(ARG2, ARG1); + a.sar(ARG2, imm(11 + _TAG_IMMED1_SIZE)); + a.cmp(ARG2, imm(0x1b)); + a.je(error); - a.mov(RETb, imm(4 * 8)); + a.cmp(ARG1, imm(make_small(0x10000UL))); + a.setae(RETb); + a.add(RETb, imm(3)); + + auto [min, max] = getClampedRange(seg.src); + if (0 <= min && max < 0x110000) { + comment("skipped range check for unicode code point"); + } else { + a.cmp(ARG1, imm(make_small(0x110000))); + a.jae(error); + } a.bind(next); - a.add(sizeReg, RET); + a.lea(sizeReg, x86::Mem(sizeReg, RET, 3, 0, 1)); break; } case am_utf16: { @@ -1891,9 +2403,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } } + segments = bs_combine_segments(segments); + /* Allocate the binary. */ if (segments[0].type == am_append) { BscSegment seg = segments[0]; + runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("append to binary"); mov_arg(ARG3, Live); if (sizeReg.isValid()) { @@ -1907,18 +2422,27 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, a.mov(ARG1, c_p); load_x_reg_array(ARG2); runtime_call<6>(erts_bs_append_checked); - if (Fail.get() == 0) { - mov_arg(ARG1, ArgXRegister(Live.get())); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_FVALUE, - BSC_VALUE_ARG1)); + + if (std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit) { + /* There is no way the call can fail with a system_limit + * exception on a 64-bit architecture. */ + comment("skipped test for success because units are compatible"); + } else { + if (Fail.get() == 0) { + mov_arg(ARG1, ArgXRegister(Live.get())); + mov_imm(ARG4, + beam_jit_update_bsc_reason_info(seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_FVALUE, + BSC_VALUE_ARG1)); + } + emit_test_the_non_value(RET); + a.je(error); } - emit_test_the_non_value(RET); - a.je(error); + a.mov(TMP_MEM1q, RET); } else if (segments[0].type == am_private_append) { BscSegment seg = segments[0]; + runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("private append to binary"); ASSERT(Alloc.get() == 0); mov_arg(ARG2, seg.src); @@ -1931,38 +2455,53 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, a.mov(ARG1, c_p); runtime_call<4>(erts_bs_private_append_checked); /* There is no way the call can fail on a 64-bit architecture. */ + a.mov(TMP_MEM1q, RET); + } else if (allocated_size >= 0) { + /* The binary has already been allocated. */ } else { comment("allocate binary"); + runtime_entered = bs_maybe_enter_runtime(runtime_entered); mov_arg(ARG5, Alloc); mov_arg(ARG6, Live); load_erl_bits_state(ARG3); load_x_reg_array(ARG2); a.mov(ARG1, c_p); if (sizeReg.isValid()) { - comment("(size in bits)"); a.mov(ARG4, sizeReg); runtime_call<6>(beam_jit_bs_init_bits); - } else if (num_bits % 8 == 0) { - comment("(size in bytes)"); - mov_imm(ARG4, num_bits / 8); - runtime_call<6>(beam_jit_bs_init); } else { + allocated_size = (num_bits + 7) / 8; + if (allocated_size <= ERL_ONHEAP_BIN_LIMIT) { + allocated_size = (allocated_size + 7) & (-8); + } mov_imm(ARG4, num_bits); runtime_call<6>(beam_jit_bs_init_bits); } + a.mov(TMP_MEM1q, RET); } - a.mov(TMP_MEM1q, RET); + + /* Keep track of the bit offset from the being of the binary. + * Set to -1 if offset is not known (when a segment of unknown + * size has been seen). */ + Sint bit_offset = 0; + + /* Keep track of whether the current segment is byte-aligned. (A + * segment can be known to be byte-aligned even if the bit offset + * is unknown.) */ + bool is_byte_aligned = true; /* Build each segment of the binary. */ for (auto seg : segments) { switch (seg.type) { case am_append: case am_private_append: + bit_offset = -1; break; case am_binary: { Uint error_info; bool can_fail = true; + runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("construct a binary segment"); if (seg.effectiveSize >= 0) { /* The segment has a literal size. */ @@ -1986,8 +2525,9 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, BSC_REASON_BADARG, BSC_INFO_UNIT, BSC_VALUE_FVALUE); - if (seg.unit == 1) { - comment("skipped test for success because unit =:= 1"); + if (std::gcd(seg.unit, getSizeUnit(seg.src)) == seg.unit) { + comment("skipped test for success because units are " + "compatible"); can_fail = false; } } else { @@ -2021,6 +2561,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, break; } case am_float: + runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("construct float segment"); if (seg.effectiveSize >= 0) { mov_imm(ARG3, seg.effectiveSize); @@ -2049,42 +2590,293 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, a.jne(error); break; case am_integer: - comment("construct integer segment"); - if (seg.effectiveSize >= 0) { - mov_imm(ARG3, seg.effectiveSize); - } else { - mov_arg(ARG3, seg.size); - a.sar(ARG3, imm(_TAG_IMMED1_SIZE)); - if (seg.unit != 1) { - mov_imm(RET, seg.unit); - a.mul(ARG3); /* CLOBBERS RDX = ARG3! */ - a.mov(ARG3, RET); + switch (seg.action) { + case BscSegment::action::ACCUMULATE_FIRST: + case BscSegment::action::ACCUMULATE: { + /* Shift an integer of known size (no more than 64 bits) + * into a word-size accumulator. */ + Label accumulate = a.newLabel(); + Label value_is_small = a.newLabel(); + x86::Gp tmp = ARG4; + x86::Gp bin_data = ARG5; + + comment("accumulate value for integer segment"); + if (seg.action == BscSegment::action::ACCUMULATE_FIRST) { + mov_imm(bin_data, 0); + } else if (seg.effectiveSize < 64) { + a.shl(bin_data, imm(seg.effectiveSize)); } + mov_arg(ARG1, seg.src); + + if (!always_small(seg.src)) { + if (always_one_of(seg.src, + BEAM_TYPE_INTEGER | + BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types " + "are boxed"); + emit_is_boxed(value_is_small, seg.src, ARG1); + } else { + a.mov(ARG2d, ARG1d); + a.and_(ARG2d, imm(_TAG_IMMED1_MASK)); + a.cmp(ARG2d, imm(_TAG_IMMED1_SMALL)); + a.short_().je(value_is_small); + } + + /* The value is boxed. If it is a bignum, extract the + * least significant 64 bits. */ + safe_fragment_call(ga->get_get_sint64_shared()); + if (always_one_of(seg.src, BEAM_TYPE_INTEGER)) { + a.short_().jmp(accumulate); + } else { + a.short_().jne(accumulate); + + /* Not a bignum. Signal error. */ + if (Fail.get() == 0) { + mov_imm(ARG4, + beam_jit_update_bsc_reason_info( + seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG1)); + } + a.jmp(error); + } + } + + a.bind(value_is_small); + a.sar(ARG1, imm(_TAG_IMMED1_SIZE)); + + /* Mask (if needed) and accumulate. */ + a.bind(accumulate); + if (seg.effectiveSize == 64) { + a.mov(bin_data, ARG1); + } else if (!need_mask(seg.src, seg.effectiveSize)) { + comment("skipped masking because the value always fits"); + a.or_(bin_data, ARG1); + } else if (seg.effectiveSize == 32) { + a.mov(ARG1d, ARG1d); + a.or_(bin_data, ARG1); + } else if (seg.effectiveSize < 32) { + a.and_(ARG1, (1ULL << seg.effectiveSize) - 1); + a.or_(bin_data, ARG1); + } else { + mov_imm(tmp, (1ULL << seg.effectiveSize) - 1); + a.and_(ARG1, tmp); + a.or_(bin_data, ARG1); + } + break; } - mov_arg(ARG2, seg.src); - mov_imm(ARG4, seg.flags); - load_erl_bits_state(ARG1); - runtime_call<4>(erts_new_bs_put_integer); - if (exact_type(seg.src, BEAM_TYPE_INTEGER)) { - comment("skipped test for success because construction can't " - "fail"); - } else { - if (Fail.get() == 0) { - mov_arg(ARG1, seg.src); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_TYPE, - BSC_VALUE_ARG1)); + case BscSegment::action::STORE: { + /* The accumulator is now full or the next segment is + * not possible to accumulate, so it's time to store + * the accumulator to the current position in the + * binary. */ + Label store = a.newLabel(); + Label done = a.newLabel(); + x86::Gp bin_ptr = ARG1; + x86::Gp bin_offset = ARG3; + x86::Gp tmp = ARG4; + x86::Gp bin_data = ARG5; + + comment("construct integer segment from accumulator"); + + /* First we'll need to ensure that the value in the + * accumulator is in little endian format. */ + if (seg.effectiveSize % 8 != 0) { + Uint complete_bytes = 8 * (seg.effectiveSize / 8); + Uint num_partial = seg.effectiveSize % 8; + if ((seg.flags & BSF_LITTLE) == 0) { + a.shl(bin_data, imm(64 - seg.effectiveSize)); + a.bswap(bin_data); + } else { + Sint mask = (1ll << complete_bytes) - 1; + a.mov(RET, bin_data); + a.shr(RET, imm(complete_bytes)); + a.and_(RETd, imm((1ull << num_partial) - 1)); + a.shl(RET, imm(complete_bytes + 8 - num_partial)); + if (Support::isInt32(mask)) { + a.and_(bin_data, imm(mask)); + } else { + mov_imm(tmp, mask); + a.and_(bin_data, tmp); + } + a.or_(bin_data, RET); + } + } else if ((seg.flags & BSF_LITTLE) == 0) { + switch (seg.effectiveSize) { + case 8: + break; + case 32: + a.bswap(bin_data.r32()); + break; + case 64: + a.bswap(bin_data); + break; + default: + a.bswap(bin_data); + a.shr(bin_data, imm(64 - seg.effectiveSize)); + break; + } } - a.test(RETd, RETd); - a.je(error); + + update_bin_state(bin_offset, + bin_ptr, + bit_offset, + seg.effectiveSize, + x86::Gp()); + + if (!is_byte_aligned) { + if (bit_offset < 0) { + /* Bit offset is unknown. Must test alignment. */ + a.and_(bin_offset, imm(7)); + a.short_().je(store); + } else if (bit_offset >= 0) { + /* Alignment is known to be unaligned. */ + mov_imm(bin_offset, bit_offset & 7); + } + + /* Bit offset is tested or known to be unaligned. */ + mov_imm(ARG4, seg.effectiveSize); + safe_fragment_call(ga->get_store_unaligned()); + + if (bit_offset < 0) { + /* The bit offset is unknown, which implies + * that there exists store code that we will + * need to branch past. */ + a.short_().jmp(done); + } + } + + a.bind(store); + + if (bit_offset <= 0 || is_byte_aligned) { + /* Bit offset is tested or known to be + * byte-aligned. Emit inline code to store the + * value of the accumulator into the binary. */ + int num_bytes = (seg.effectiveSize + 7) / 8; + + /* If more than one instruction is required for + * doing the store, test whether it would be safe + * to do a single 32 or 64 bit store. */ + switch (num_bytes) { + case 3: + if (bit_offset >= 0 && + allocated_size * 8 - bit_offset >= 32) { + comment("simplified complicated store"); + num_bytes = 4; + } + break; + case 5: + case 6: + case 7: + if (bit_offset >= 0 && + allocated_size * 8 - bit_offset >= 64) { + comment("simplified complicated store"); + num_bytes = 8; + } + break; + } + + do { + switch (num_bytes) { + case 1: + a.mov(x86::Mem(bin_ptr, 0, 1), bin_data.r8()); + break; + case 2: + a.mov(x86::Mem(bin_ptr, 0, 2), bin_data.r16()); + break; + case 3: + a.mov(x86::Mem(bin_ptr, 0, 2), bin_data.r16()); + a.shr(bin_data, imm(16)); + a.mov(x86::Mem(bin_ptr, 2, 1), bin_data.r8()); + break; + case 4: + a.mov(x86::Mem(bin_ptr, 0, 4), bin_data.r32()); + break; + case 5: + case 6: + case 7: + a.mov(x86::Mem(bin_ptr, 0, 4), bin_data.r32()); + a.add(bin_ptr, imm(4)); + a.shr(bin_data, imm(32)); + break; + case 8: + a.mov(x86::Mem(bin_ptr, 0, 8), bin_data); + num_bytes = 0; + break; + default: + ASSERT(0); + } + num_bytes -= 4; + } while (num_bytes > 0); + } + + a.bind(done); + break; + } + case BscSegment::action::DIRECT: + /* This segment either has a size exceeding the maximum + * accumulator size of 64 bits or has a variable size. + * + * First load the effective size (size * unit) into ARG3. + */ + comment("construct integer segment"); + if (seg.effectiveSize >= 0) { + mov_imm(ARG3, seg.effectiveSize); + } else { + mov_arg(ARG3, seg.size); + a.sar(ARG3, imm(_TAG_IMMED1_SIZE)); + if (Support::isPowerOf2(seg.unit)) { + Uint trailing_bits = Support::ctz<Eterm>(seg.unit); + if (trailing_bits) { + a.shl(ARG3, imm(trailing_bits)); + } + } else { + mov_imm(RET, seg.unit); + a.mul(ARG3); /* CLOBBERS RDX = ARG3! */ + a.mov(ARG3, RET); + } + } + + if (is_byte_aligned && seg.src.isSmall() && + seg.src.as<ArgSmall>().getSigned() == 0) { + /* Optimize the special case of setting a known + * byte-aligned segment to zero. */ + comment("optimized setting segment to 0"); + set_zero(seg.effectiveSize); + } else { + /* Call the helper function to fetch and store the + * integer into the binary. */ + runtime_entered = bs_maybe_enter_runtime(runtime_entered); + mov_arg(ARG2, seg.src); + mov_imm(ARG4, seg.flags); + load_erl_bits_state(ARG1); + runtime_call<4>(erts_new_bs_put_integer); + if (exact_type(seg.src, BEAM_TYPE_INTEGER)) { + comment("skipped test for success because construction " + "can't fail"); + } else { + if (Fail.get() == 0) { + mov_arg(ARG1, seg.src); + mov_imm(ARG4, + beam_jit_update_bsc_reason_info( + seg.error_info, + BSC_REASON_BADARG, + BSC_INFO_TYPE, + BSC_VALUE_ARG1)); + } + a.test(RETd, RETd); + a.je(error); + } + } + break; } break; case am_string: { ArgBytePtr string_ptr( ArgVal(ArgVal::BytePtr, seg.src.as<ArgWord>().get())); + runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("insert string"); ASSERT(seg.effectiveSize >= 0); mov_imm(ARG3, seg.effectiveSize / 8); @@ -2092,22 +2884,13 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, load_erl_bits_state(ARG1); runtime_call<3>(erts_new_bs_put_string); } break; - case am_utf8: - mov_arg(ARG2, seg.src); - load_erl_bits_state(ARG1); - runtime_call<2>(erts_bs_put_utf8); - if (Fail.get() == 0) { - mov_arg(ARG1, seg.src); - mov_imm(ARG4, - beam_jit_update_bsc_reason_info(seg.error_info, - BSC_REASON_BADARG, - BSC_INFO_TYPE, - BSC_VALUE_ARG1)); - } - a.test(RETd, RETd); - a.je(error); + case am_utf8: { + runtime_entered = bs_maybe_enter_runtime(runtime_entered); + emit_construct_utf8(seg.src, bit_offset, is_byte_aligned); break; + } case am_utf16: + runtime_entered = bs_maybe_enter_runtime(runtime_entered); mov_arg(ARG2, seg.src); a.mov(ARG3, seg.flags); load_erl_bits_state(ARG1); @@ -2124,6 +2907,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, a.je(error); break; case am_utf32: + runtime_entered = bs_maybe_enter_runtime(runtime_entered); mov_arg(ARG2, seg.src); mov_imm(ARG3, 4 * 8); a.mov(ARG4, seg.flags); @@ -2144,10 +2928,1084 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, ASSERT(0); break; } + + /* Try to keep track of the bit offset. */ + if (bit_offset >= 0 && (seg.action == BscSegment::action::DIRECT || + seg.action == BscSegment::action::STORE)) { + if (seg.effectiveSize >= 0) { + bit_offset += seg.effectiveSize; + } else { + bit_offset = -1; + } + } + + /* Try to keep track whether the next segment is byte + * aligned. */ + if (seg.type == am_append || seg.type == am_private_append) { + if (std::gcd(getSizeUnit(seg.src), 8) != 8) { + is_byte_aligned = false; + } + } else if (bit_offset % 8 == 0) { + is_byte_aligned = true; + } else if (seg.effectiveSize >= 0) { + if (seg.effectiveSize % 8 != 0) { + is_byte_aligned = false; + } + } else if (std::gcd(seg.unit, 8) != 8) { + is_byte_aligned = false; + } } + bs_maybe_leave_runtime(runtime_entered); comment("done"); - emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); a.mov(RET, TMP_MEM1q); mov_arg(Dst, RET); } + +/* + * Here follows the bs_match instruction and friends. + */ + +struct BsmSegment { + BsmSegment() + : action(action::TEST_HEAP), live(ArgNil()), size(0), unit(1), + flags(0), dst(ArgXRegister(0)){}; + + enum class action { + TEST_HEAP, + ENSURE_AT_LEAST, + ENSURE_EXACTLY, + READ, + EXTRACT_BINARY, + EXTRACT_INTEGER, + READ_INTEGER, + GET_INTEGER, + GET_BINARY, + SKIP, + DROP, + GET_TAIL, + EQ + } action; + ArgVal live; + Uint size; + Uint unit; + Uint flags; + ArgRegister dst; +}; + +void BeamModuleAssembler::emit_read_bits(Uint bits, + const x86::Gp bin_base, + const x86::Gp bin_offset, + const x86::Gp bitdata) { + Label read_done = a.newLabel(); + auto num_partial = bits % 8; + auto num_bytes_to_read = (bits + 7) / 8; + + ASSERT(bin_offset == x86::rcx); + + a.mov(RET, bin_offset); + a.shr(RET, imm(3)); + if (num_bytes_to_read != 1) { + a.add(bin_base, RET); + } + a.and_(bin_offset.r32(), imm(7)); + + /* + * Special-case handling of reading 8 or 9 bytes. + */ + if (num_bytes_to_read == 8) { + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(bitdata, x86::qword_ptr(bin_base, num_bytes_to_read - 8)); + } else { + a.mov(bitdata, x86::qword_ptr(bin_base, num_bytes_to_read - 8)); + a.bswap(bitdata); + } + + a.shl(bitdata, bin_offset.r8()); + + a.test(x86::cl, imm(7)); + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, this + * segment always needs an additional byte. */ + a.jz(read_done); + } else if (num_partial > 1) { + /* Non-byte-sized segment. Test whether we will need an + * additional byte. */ + a.cmp(bin_offset.r32(), imm(8 - num_partial)); + a.jle(read_done); + } + + if (num_partial != 1) { + /* Read an extra byte. */ + a.movzx(RETd, x86::byte_ptr(bin_base, 8)); + a.shl(RETd, bin_offset.r8()); + a.shr(RETd, imm(8)); + a.or_(bitdata, RET); + } + + a.bind(read_done); + + return; + } + + /* + * Handle reading of up to 7 bytes. + */ + Label handle_partial = a.newLabel(); + Label swap = a.newLabel(); + Label shift = a.newLabel(); + + if (num_partial == 0) { + /* Byte-sized segment. If bit_offset is not byte-aligned, this + * segment always needs an additional byte. */ + a.jnz(handle_partial); + } else if (num_partial > 1) { + /* Non-byte-sized segment. Test whether we will need an + * additional byte. */ + a.cmp(bin_offset.r32(), imm(8 - num_partial)); + a.jg(handle_partial); + } + + /* We don't need an extra byte. */ + if (num_bytes_to_read == 1) { + a.movzx(bitdata.r32(), x86::byte_ptr(bin_base, RET)); + if (num_partial == 0) { + a.bswap(bitdata); + a.short_().jmp(read_done); + } else if (num_partial > 1) { + a.short_().jmp(swap); + } + } else if (num_bytes_to_read <= 4) { + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(bitdata.r32(), + x86::dword_ptr(bin_base, num_bytes_to_read - 4)); + } else { + a.mov(bitdata.r32(), + x86::dword_ptr(bin_base, num_bytes_to_read - 4)); + a.bswap(bitdata.r32()); + } + a.add(bin_offset.r32(), imm(64 - 8 * num_bytes_to_read)); + a.short_().jmp(shift); + } else { + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(bitdata, x86::qword_ptr(bin_base, num_bytes_to_read - 8)); + } else { + a.mov(bitdata, x86::qword_ptr(bin_base, num_bytes_to_read - 8)); + a.bswap(bitdata); + } + ASSERT(num_bytes_to_read < 8); + a.add(bin_offset.r32(), imm(64 - 8 * num_bytes_to_read)); + a.short_().jmp(shift); + } + + /* We'll need an extra byte and we will need to shift. */ + a.bind(handle_partial); + if (num_partial != 1) { + if (num_bytes_to_read == 1) { + a.mov(bitdata.r16(), x86::word_ptr(bin_base, RET)); + } else { + ASSERT(num_bytes_to_read < 8); + a.mov(bitdata, x86::qword_ptr(bin_base, num_bytes_to_read - 7)); + a.shr(bitdata, imm(64 - 8 * (num_bytes_to_read + 1))); + } + } + + a.bind(swap); + a.bswap(bitdata); + + /* Shift the read data into the most significant bits of the + * word. */ + a.bind(shift); + a.shl(bitdata, bin_offset.r8()); + + a.bind(read_done); +} + +/* + * Read an integer and store as a term. This function only handles + * integers of certain common sizes. This is a special optimization + * when only one integer is to be extracted from a binary. + * + * Input: bin_base, bin_offset + * + * Clobbers: bin_base, bin_offset, tmp, RET + */ +void BeamModuleAssembler::emit_read_integer(const x86::Gp bin_base, + const x86::Gp bin_offset, + const x86::Gp tmp, + Uint flags, + Uint bits, + const ArgRegister &Dst) { + Label handle_unaligned = a.newLabel(); + Label store = a.newLabel(); + x86::Mem address; + + a.mov(tmp, bin_offset); + a.shr(tmp, imm(3)); + a.and_(bin_offset.r32(), imm(7)); + + switch (bits) { + case 8: + address = x86::Mem(bin_base, tmp, 0, 0, 1); + if ((flags & BSF_SIGNED) == 0) { + a.movzx(RETd, address); + } else { + a.movsx(RET, address); + } + + a.short_().jz(store); + + a.bind(handle_unaligned); + address = x86::Mem(bin_base, tmp, 0, 0, 2); + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(RET.r16(), address); + } else { + a.mov(RET.r16(), address); + a.xchg(x86::al, x86::ah); + } + ASSERT(bin_offset == x86::rcx); + a.shl(RETd, bin_offset.r8()); + a.mov(x86::al, x86::ah); + if ((flags & BSF_SIGNED) == 0) { + a.movzx(RETd, RETb); + } else { + a.movsx(RET, RETb); + } + break; + case 16: + address = x86::Mem(bin_base, tmp, 0, 0, 2); + if ((flags & BSF_LITTLE) != 0) { + if ((flags & BSF_SIGNED) == 0) { + a.movzx(RETd, address); + } else { + a.movsx(RET, address); + } + } else { + /* Big-endian segment. */ + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(RET.r16(), address); + } else { + a.mov(RET.r16(), address); + a.xchg(x86::al, x86::ah); + } + + if ((flags & BSF_SIGNED) != 0) { + a.movsx(RET, RET.r16()); + } else { + a.movzx(RET, RET.r16()); + } + } + + a.short_().jz(store); + + a.bind(handle_unaligned); + a.add(bin_base, tmp); + address = x86::Mem(bin_base, -1, 4); + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(RETd, address); + } else { + a.mov(RETd, address); + a.bswap(RETd); + } + ASSERT(bin_offset == x86::rcx); + a.shl(RETd, bin_offset.r8()); + a.shr(RETd, imm(8)); + + if ((flags & BSF_LITTLE) != 0) { + a.xchg(x86::al, x86::ah); + } + + if ((flags & BSF_SIGNED) == 0) { + a.movzx(RETd, RET.r16()); + } else { + a.movsx(RET, RET.r16()); + } + break; + case 32: + address = x86::Mem(bin_base, tmp, 0, 0, 4); + if ((flags & BSF_LITTLE) != 0) { + /* Little-endian segment. */ + if ((flags & BSF_SIGNED) == 0) { + a.mov(RETd, address); + } else { + a.movsxd(RET, address); + } + } else { + /* Big-endian segment. */ + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(RETd, address); + } else { + a.mov(RETd, address); + a.bswap(RETd); + } + + if ((flags & BSF_SIGNED) != 0) { + a.movsxd(RET, RETd); + } + } + + a.short_().jz(store); + + a.bind(handle_unaligned); + a.add(bin_base, tmp); + address = x86::Mem(bin_base, -3, 8); + if (hasCpuFeature(CpuFeatures::X86::kMOVBE)) { + a.movbe(RET, address); + } else { + a.mov(RET, address); + a.bswap(RET); + } + ASSERT(bin_offset == x86::rcx); + a.shl(RET, bin_offset.r8()); + a.shr(RET, imm(8)); + + if ((flags & BSF_LITTLE) != 0) { + a.bswap(RETd); + } + + if ((flags & BSF_SIGNED) == 0) { + a.mov(RETd, RETd); + } else { + a.movsxd(RET, RETd); + } + break; + default: + ASSERT(0); + break; + } + + a.bind(store); + a.shl(RET, imm(_TAG_IMMED1_SIZE)); + a.or_(RET, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, RET); +} + +void BeamModuleAssembler::emit_extract_integer(const x86::Gp bitdata, + const x86::Gp tmp, + Uint flags, + Uint bits, + const ArgRegister &Dst) { + if (bits == 0) { + /* Necessary for correctness when matching a zero-size + * signed segment. + */ + mov_arg(Dst, make_small(0)); + return; + } + + Label big = a.newLabel(); + Label done = a.newLabel(); + Uint num_partial = bits % 8; + Uint num_complete = 8 * (bits / 8); + + if (bits <= 8) { + /* Endian does not matter for values that fit in a byte. */ + flags &= ~BSF_LITTLE; + } + + if ((flags & BSF_LITTLE) == 0) { + /* Big-endian segment. */ + a.mov(RET, bitdata); + } else if ((flags & BSF_LITTLE) != 0) { + /* Reverse endianness for this little-endian segment. */ + if (num_partial == 0) { + a.mov(RET, bitdata); + a.bswap(RET); + if (bits < 64) { + a.shl(RET, imm(64 - num_complete)); + } + } else { + Uint shifted_mask = ((1 << num_partial) - 1) << (8 - num_partial); + a.mov(tmp, bitdata); + a.shr(tmp, imm(64 - num_complete)); + a.bswap(tmp); + a.shr(tmp, imm(num_partial)); + + a.mov(RET, bitdata); + a.rol(RET, imm(num_complete + 8)); + a.and_(RETd, imm(shifted_mask)); + a.ror(RET, imm(8)); + a.or_(RET, tmp); + } + } + + /* Now the extracted data is in RET. */ + if (bits >= SMALL_BITS) { + /* Handle segments whose values might not fit in a small + * integer. */ + Label small = a.newLabel(); + comment("test whether this integer is a small"); + if (bits < 64) { + if ((flags & BSF_SIGNED) == 0) { + /* Unsigned segment. */ + a.shr(RET, imm(64 - bits)); + } else { + /* Signed segment. */ + a.sar(RET, imm(64 - bits)); + } + } + a.mov(tmp, RET); + a.shr(tmp, imm(SMALL_BITS - 1)); + if ((flags & BSF_SIGNED) == 0) { + /* Unsigned segment. */ + a.jnz(big); + } else { + /* Signed segment. */ + a.jz(small); + a.cmp(tmp.r32(), imm(_TAG_IMMED1_MASK << 1 | 1)); + a.jnz(big); + } + + comment("store extracted integer as a small"); + a.bind(small); + a.shl(RET, imm(_TAG_IMMED1_SIZE)); + a.or_(RET, imm(_TAG_IMMED1_SMALL)); + a.short_().jmp(done); + } else { + /* This segment always fits in a small. */ + comment("store extracted integer as a small"); + if ((flags & BSF_SIGNED) == 0) { + /* Unsigned segment. */ + a.shr(RET, imm(64 - bits - _TAG_IMMED1_SIZE)); + } else { + /* Signed segment. */ + a.sar(RET, imm(64 - bits - _TAG_IMMED1_SIZE)); + } + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == (1 << _TAG_IMMED1_SIZE) - 1); + a.or_(RET, imm(_TAG_IMMED1_SMALL)); + } + + a.bind(big); + if (bits >= SMALL_BITS) { + comment("store extracted integer as a bignum"); + if ((flags & BSF_SIGNED) == 0) { + /* Unsigned segment. */ + a.mov(x86::qword_ptr(HTOP), make_pos_bignum_header(1)); + a.mov(x86::qword_ptr(HTOP, sizeof(Eterm)), RET); + } else { + Label negative = a.newLabel(); + Label sign_done = a.newLabel(); + + /* Signed segment. */ + a.test(RET, RET); + a.short_().jl(negative); + + a.mov(x86::qword_ptr(HTOP), make_pos_bignum_header(1)); + a.short_().jmp(sign_done); + + a.bind(negative); + a.mov(x86::qword_ptr(HTOP), make_neg_bignum_header(1)); + a.neg(RET); + + a.bind(sign_done); + a.mov(x86::qword_ptr(HTOP, sizeof(Eterm)), RET); + } + a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED)); + a.add(HTOP, imm(sizeof(Eterm[2]))); + } + + a.bind(done); + mov_arg(Dst, RET); +} + +/* + * Clobbers: RET + */ +void BeamModuleAssembler::emit_extract_binary(const x86::Gp bitdata, + Uint bits, + const ArgRegister &Dst) { + Uint num_bytes = bits / 8; + + a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED)); + mov_arg(Dst, RET); + a.mov(x86::qword_ptr(HTOP), header_heap_bin(num_bytes)); + a.mov(x86::qword_ptr(HTOP, sizeof(Eterm)), imm(num_bytes)); + a.mov(RET, bitdata); + a.bswap(RET); + a.mov(x86::qword_ptr(HTOP, 2 * sizeof(Eterm)), RET); + a.add(HTOP, imm(sizeof(Eterm[3]))); +} + +static std::vector<BsmSegment> opt_bsm_segments( + const std::vector<BsmSegment> segments, + const ArgWord &Need, + const ArgWord &Live) { + std::vector<BsmSegment> segs; + + Uint heap_need = Need.get(); + + /* + * First calculate the total number of heap words needed for + * bignums and binaries. + */ + for (auto seg : segments) { + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + if (seg.size >= SMALL_BITS) { + heap_need += BIG_NEED_FOR_BITS(seg.size); + } + break; + case BsmSegment::action::GET_BINARY: + heap_need += heap_bin_size((seg.size + 7) / 8); + break; + case BsmSegment::action::GET_TAIL: + heap_need += EXTRACT_SUB_BIN_HEAP_NEED; + break; + default: + break; + } + } + + int read_action_pos = -1; + int seg_index = 0; + int count = segments.size(); + + for (int i = 0; i < count; i++) { + auto seg = segments[i]; + if (heap_need != 0 && seg.live.isWord()) { + BsmSegment s = seg; + + read_action_pos = -1; + s.action = BsmSegment::action::TEST_HEAP; + s.size = heap_need; + segs.push_back(s); + heap_need = 0; + seg_index++; + } + + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + case BsmSegment::action::GET_BINARY: { + bool is_common_size; + switch (seg.size) { + case 8: + case 16: + case 32: + is_common_size = true; + break; + default: + is_common_size = false; + break; + } + + if (seg.size > 64) { + read_action_pos = -1; + } else if (seg.action == BsmSegment::action::GET_BINARY && + seg.size % 8 != 0) { + read_action_pos = -1; + } else if ((seg.flags & BSF_LITTLE) != 0 && is_common_size) { + seg.action = BsmSegment::action::READ_INTEGER; + read_action_pos = -1; + } else if (read_action_pos < 0 && + seg.action == BsmSegment::action::GET_INTEGER && + is_common_size && i + 1 == count) { + seg.action = BsmSegment::action::READ_INTEGER; + read_action_pos = -1; + } else { + if ((seg.flags & BSF_LITTLE) != 0 || read_action_pos < 0 || + seg.size + segs.at(read_action_pos).size > 64) { + BsmSegment s; + + /* Create a new READ action. */ + read_action_pos = seg_index; + s.action = BsmSegment::action::READ; + s.size = seg.size; + segs.push_back(s); + seg_index++; + } else { + /* Reuse previous READ action. */ + segs.at(read_action_pos).size += seg.size; + } + switch (seg.action) { + case BsmSegment::action::GET_INTEGER: + seg.action = BsmSegment::action::EXTRACT_INTEGER; + break; + case BsmSegment::action::GET_BINARY: + seg.action = BsmSegment::action::EXTRACT_BINARY; + break; + default: + break; + } + } + segs.push_back(seg); + break; + } + case BsmSegment::action::EQ: { + if (read_action_pos < 0 || + seg.size + segs.at(read_action_pos).size > 64) { + BsmSegment s; + + /* Create a new READ action. */ + read_action_pos = seg_index; + s.action = BsmSegment::action::READ; + s.size = seg.size; + segs.push_back(s); + seg_index++; + } else { + /* Reuse previous READ action. */ + segs.at(read_action_pos).size += seg.size; + } + auto &prev = segs.back(); + if (prev.action == BsmSegment::action::EQ && + prev.size + seg.size <= 64) { + /* Coalesce with the previous EQ instruction. */ + prev.size += seg.size; + prev.unit = prev.unit << seg.size | seg.unit; + seg_index--; + } else { + segs.push_back(seg); + } + break; + } + case BsmSegment::action::SKIP: + if (read_action_pos >= 0 && + seg.size + segs.at(read_action_pos).size <= 64) { + segs.at(read_action_pos).size += seg.size; + seg.action = BsmSegment::action::DROP; + } else { + read_action_pos = -1; + } + segs.push_back(seg); + break; + default: + read_action_pos = -1; + segs.push_back(seg); + break; + } + seg_index++; + } + + /* Handle a trailing test_heap instruction (for the + * i_bs_match_test_heap instruction). */ + if (heap_need) { + BsmSegment seg; + + seg.action = BsmSegment::action::TEST_HEAP; + seg.size = heap_need; + seg.live = Live; + segs.push_back(seg); + } + return segs; +} + +UWord BeamModuleAssembler::bs_get_flags(const ArgVal &val) { + if (val.isNil()) { + return 0; + } else if (val.isLiteral()) { + Eterm term = beamfile_get_literal(beam, val.as<ArgLiteral>().get()); + UWord flags = 0; + + while (is_list(term)) { + Eterm *consp = list_val(term); + Eterm elem = CAR(consp); + switch (elem) { + case am_little: + case am_native: + flags |= BSF_LITTLE; + break; + case am_signed: + flags |= BSF_SIGNED; + break; + } + term = CDR(consp); + } + ASSERT(is_nil(term)); + return flags; + } else if (val.isWord()) { + /* Originates from bs_get_integer2 instruction. */ + return val.as<ArgWord>().get(); + } else { + ASSERT(0); /* Should not happen. */ + return 0; + } +} + +void BeamModuleAssembler::emit_i_bs_match(ArgLabel const &Fail, + ArgRegister const &Ctx, + Span<ArgVal> const &List) { + emit_i_bs_match_test_heap(Fail, Ctx, ArgWord(0), ArgWord(0), List); +} + +void BeamModuleAssembler::emit_i_bs_match_test_heap(ArgLabel const &Fail, + ArgRegister const &Ctx, + ArgWord const &Need, + ArgWord const &Live, + Span<ArgVal> const &List) { + const int orig_offset = offsetof(ErlBinMatchState, mb.orig); + const int base_offset = offsetof(ErlBinMatchState, mb.base); + const int position_offset = offsetof(ErlBinMatchState, mb.offset); + const int size_offset = offsetof(ErlBinMatchState, mb.size); + + std::vector<BsmSegment> segments; + + auto current = List.begin(); + auto end = List.begin() + List.size(); + + while (current < end) { + auto cmd = current++->as<ArgImmed>().get(); + BsmSegment seg; + + switch (cmd) { + case am_ensure_at_least: { + seg.action = BsmSegment::action::ENSURE_AT_LEAST; + seg.size = current[0].as<ArgWord>().get(); + seg.unit = current[1].as<ArgWord>().get(); + current += 2; + break; + } + case am_ensure_exactly: { + seg.action = BsmSegment::action::ENSURE_EXACTLY; + seg.size = current[0].as<ArgWord>().get(); + current += 1; + break; + } + case am_binary: + case am_integer: { + auto size = current[2].as<ArgWord>().get(); + auto unit = current[3].as<ArgWord>().get(); + + switch (cmd) { + case am_integer: + seg.action = BsmSegment::action::GET_INTEGER; + break; + case am_binary: + seg.action = BsmSegment::action::GET_BINARY; + break; + } + + seg.live = current[0]; + seg.size = size * unit; + seg.unit = unit; + seg.flags = bs_get_flags(current[1]); + seg.dst = current[4].as<ArgRegister>(); + current += 5; + break; + } + case am_get_tail: { + seg.action = BsmSegment::action::GET_TAIL; + seg.live = current[0].as<ArgWord>(); + seg.dst = current[2].as<ArgRegister>(); + current += 3; + break; + } + case am_skip: { + seg.action = BsmSegment::action::SKIP; + seg.size = current[0].as<ArgWord>().get(); + seg.flags = 0; + current += 1; + break; + } + case am_Eq: { + seg.action = BsmSegment::action::EQ; + seg.live = current[0]; + seg.size = current[1].as<ArgWord>().get(); + seg.unit = current[2].as<ArgWord>().get(); + current += 3; + break; + } + default: + abort(); + break; + } + segments.push_back(seg); + } + + segments = opt_bsm_segments(segments, Need, Live); + + /* Constraints: + * + * bin_position must be RCX because only CL can be used for + * a variable shift without using the SHLX instruction from BMI2. + */ +#ifdef WIN32 + const x86::Gp bin_position = ARG1; + const x86::Gp bitdata = ARG2; + const x86::Gp bin_base = ARG3; + const x86::Gp ctx = ARG4; +#else + const x86::Gp bin_position = ARG4; + const x86::Gp bitdata = ARG3; + const x86::Gp bin_base = ARG1; + const x86::Gp ctx = ARG2; +#endif + ASSERT(bin_position == x86::rcx); + const x86::Gp tmp = ARG5; + + bool is_ctx_valid = false; + bool is_position_valid = false; + bool is_last = false; + int count = segments.size(); + + for (int i = 0; i < count; i++) { + auto seg = segments[i]; + is_last = i == count - 1; + switch (seg.action) { + case BsmSegment::action::ENSURE_AT_LEAST: { + auto size = seg.size; + auto unit = seg.unit; + comment("ensure_at_least %ld %ld", size, seg.unit); + mov_arg(ctx, Ctx); + if (unit == 1) { + a.mov(bin_position, emit_boxed_val(ctx, position_offset)); + a.lea(RET, qword_ptr(bin_position, size)); + a.cmp(RET, emit_boxed_val(ctx, size_offset)); + a.ja(resolve_beam_label(Fail)); + } else { + a.mov(RET, emit_boxed_val(ctx, size_offset)); + a.mov(bin_position, emit_boxed_val(ctx, position_offset)); + a.sub(RET, bin_position); + if (size != 0) { + cmp(RET, size, tmp); + } + a.jl(resolve_beam_label(Fail)); + } + + is_ctx_valid = is_position_valid = true; + + if (unit != 1) { + if (size % unit != 0) { + sub(RET, size, tmp); + } + + if ((unit & (unit - 1))) { + /* Clobbers ARG3 */ + a.cqo(); + mov_imm(tmp, unit); + a.div(tmp); + a.test(x86::rdx, x86::rdx); + is_ctx_valid = is_position_valid = false; + } else { + a.test(RETb, imm(unit - 1)); + } + a.jnz(resolve_beam_label(Fail)); + } + break; + } + case BsmSegment::action::ENSURE_EXACTLY: { + auto size = seg.size; + comment("ensure_exactly %ld", size); + + mov_arg(ctx, Ctx); + a.mov(RET, emit_boxed_val(ctx, size_offset)); + if (is_last) { + a.sub(RET, emit_boxed_val(ctx, position_offset)); + is_ctx_valid = is_position_valid = false; + } else { + a.mov(bin_position, emit_boxed_val(ctx, position_offset)); + a.sub(RET, bin_position); + is_ctx_valid = is_position_valid = true; + } + if (size != 0) { + cmp(RET, size, tmp); + } + a.jne(resolve_beam_label(Fail)); + break; + } + case BsmSegment::action::EQ: { + comment("=:= %ld %ld", seg.size, seg.unit); + auto bits = seg.size; + x86::Gp extract_reg; + + if (is_last) { + extract_reg = bitdata; + } else { + extract_reg = RET; + a.mov(extract_reg, bitdata); + } + if (bits != 0 && bits != 64) { + a.shr(extract_reg, imm(64 - bits)); + } + + if (seg.size <= 32) { + cmp(extract_reg.r32(), seg.unit, tmp); + } else { + cmp(extract_reg, seg.unit, tmp); + } + + a.jne(resolve_beam_label(Fail)); + + if (!is_last && bits != 0 && bits != 64) { + a.shl(bitdata, imm(bits)); + } + + /* bin_position is clobbered. */ + is_position_valid = false; + break; + } + case BsmSegment::action::TEST_HEAP: { + comment("test_heap %ld", seg.size); + emit_gc_test(ArgWord(0), ArgWord(seg.size), seg.live); + is_ctx_valid = is_position_valid = false; + break; + } + case BsmSegment::action::READ: { + comment("read %ld", seg.size); + if (seg.size == 0) { + comment("(nothing to do)"); + } else { + if (!is_ctx_valid) { + mov_arg(ctx, Ctx); + is_ctx_valid = true; + } + if (!is_position_valid) { + a.mov(bin_position, emit_boxed_val(ctx, position_offset)); + is_position_valid = true; + } + a.mov(bin_base, emit_boxed_val(ctx, base_offset)); + a.add(emit_boxed_val(ctx, position_offset), imm(seg.size)); + + emit_read_bits(seg.size, bin_base, bin_position, bitdata); + } + + is_position_valid = false; + break; + } + case BsmSegment::action::EXTRACT_BINARY: { + auto bits = seg.size; + auto Dst = seg.dst; + + comment("extract binary %ld", bits); + emit_extract_binary(bitdata, bits, Dst); + if (!is_last && bits != 0 && bits != 64) { + a.shl(bitdata, imm(bits)); + } + break; + } + case BsmSegment::action::EXTRACT_INTEGER: { + auto bits = seg.size; + auto flags = seg.flags; + auto Dst = seg.dst; + + comment("extract integer %ld", bits); + if (is_last && flags == 0 && bits < SMALL_BITS) { + a.shr(bitdata, imm(64 - bits - _TAG_IMMED1_SIZE)); + a.or_(bitdata, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, bitdata); + } else { + emit_extract_integer(bitdata, tmp, flags, bits, Dst); + if (!is_last && bits != 0 && bits != 64) { + a.shl(bitdata, imm(bits)); + } + } + + /* bin_position is clobbered. */ + is_position_valid = false; + break; + } + case BsmSegment::action::READ_INTEGER: { + auto bits = seg.size; + auto flags = seg.flags; + auto Dst = seg.dst; + + comment("read integer %ld", bits); + if (!is_ctx_valid) { + mov_arg(ctx, Ctx); + is_ctx_valid = true; + } + if (!is_position_valid) { + a.mov(bin_position, emit_boxed_val(ctx, position_offset)); + is_position_valid = true; + } + + a.mov(bin_base, emit_boxed_val(ctx, base_offset)); + a.add(emit_boxed_val(ctx, position_offset), imm(seg.size)); + emit_read_integer(bin_base, bin_position, tmp, flags, bits, Dst); + + is_position_valid = false; + break; + } + case BsmSegment::action::GET_INTEGER: { + Uint flags = seg.flags; + auto bits = seg.size; + auto Dst = seg.dst; + + comment("get integer %ld", bits); + if (!is_ctx_valid) { + mov_arg(ctx, Ctx); + } + + a.lea(ARG4, emit_boxed_val(ctx, offsetof(ErlBinMatchState, mb))); + + if (bits >= SMALL_BITS) { + emit_enter_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } else { + emit_enter_runtime(); + } + + a.mov(ARG1, c_p); + a.mov(ARG2, bits); + a.mov(ARG3, flags); + /* ARG4 set above */ + runtime_call<4>(erts_bs_get_integer_2); + + if (bits >= SMALL_BITS) { + emit_leave_runtime<Update::eReductions | Update::eStack | + Update::eHeap>(); + } else { + emit_leave_runtime(); + } + + mov_arg(Dst, RET); + + is_ctx_valid = is_position_valid = false; + break; + } + case BsmSegment::action::GET_BINARY: { + comment("get binary %ld", seg.size); + if (is_ctx_valid) { + a.mov(RET, ctx); + } else { + mov_arg(RET, Ctx); + } + emit_enter_runtime<Update::eHeap>(); + a.lea(ARG1, x86::qword_ptr(c_p, offsetof(Process, htop))); + a.mov(ARG2, emit_boxed_val(RET, orig_offset)); + a.mov(ARG3, emit_boxed_val(RET, base_offset)); + a.mov(ARG4, emit_boxed_val(RET, position_offset)); + mov_imm(ARG5, seg.size); + a.add(emit_boxed_val(RET, position_offset), ARG5); + + runtime_call<5>(erts_extract_sub_binary); + + emit_leave_runtime<Update::eHeap>(); + mov_arg(seg.dst, RET); + + is_ctx_valid = is_position_valid = false; + break; + } + case BsmSegment::action::GET_TAIL: { + comment("get_tail"); + if (is_ctx_valid) { + a.mov(ARG1, ctx); + } else { + mov_arg(ARG1, Ctx); + } + safe_fragment_call(ga->get_bs_get_tail_shared()); + mov_arg(seg.dst, RET); + is_ctx_valid = is_position_valid = false; + break; + } + case BsmSegment::action::SKIP: { + comment("skip %ld", seg.size); + if (!is_ctx_valid) { + mov_arg(ctx, Ctx); + is_ctx_valid = true; + } + /* The compiler limits the size of any segment in a bs_match + * instruction to 24 bits. */ + ASSERT((seg.size >> 24) == 0); + a.add(emit_boxed_val(ctx, position_offset), imm(seg.size)); + is_position_valid = false; + break; + } + case BsmSegment::action::DROP: + auto bits = seg.size; + comment("drop %ld", bits); + if (bits != 0 && bits != 64) { + a.shl(bitdata, imm(bits)); + } + break; + } + } +} diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp index 9157ed6e9c..248afc9e14 100644 --- a/erts/emulator/beam/jit/x86/instr_common.cpp +++ b/erts/emulator/beam/jit/x86/instr_common.cpp @@ -81,6 +81,7 @@ */ #include <algorithm> +#include <numeric> #include "beam_asm.hpp" extern "C" @@ -90,6 +91,7 @@ extern "C" #include "beam_catches.h" #include "beam_common.h" #include "code_ix.h" +#include "erl_binary.h" } using namespace asmjit; @@ -103,20 +105,44 @@ void BeamModuleAssembler::emit_error(int reason) { void BeamModuleAssembler::emit_gc_test_preserve(const ArgWord &Need, const ArgWord &Live, - x86::Gp term) { + const ArgSource &Preserve, + x86::Gp preserve_reg) { const int32_t bytes_needed = (Need.get() + S_RESERVED) * sizeof(Eterm); Label after_gc_check = a.newLabel(); - ASSERT(term != ARG3); + ASSERT(preserve_reg != ARG3); + +#ifdef DEBUG + comment("(debug: fill dead X registers with garbage)"); + const x86::Gp garbage_reg = ARG3; + mov_imm(garbage_reg, ERTS_HOLE_MARKER); + if (!(Preserve.isXRegister() && + Preserve.as<ArgXRegister>().get() >= Live.get())) { + mov_arg(ArgXRegister(Live.get()), garbage_reg); + mov_arg(ArgXRegister(Live.get() + 1), garbage_reg); + } else { + mov_arg(ArgXRegister(Live.get() + 1), garbage_reg); + mov_arg(ArgXRegister(Live.get() + 2), garbage_reg); + } +#endif a.lea(ARG3, x86::qword_ptr(HTOP, bytes_needed)); a.cmp(ARG3, E); a.short_().jbe(after_gc_check); - a.mov(getXRef(Live.get()), term); - mov_imm(ARG4, Live.get() + 1); - fragment_call(ga->get_garbage_collect()); - a.mov(term, getXRef(Live.get())); + /* We don't need to stash the preserved term if it's currently live, making + * the code slightly shorter. */ + if (!(Preserve.isXRegister() && + Preserve.as<ArgXRegister>().get() >= Live.get())) { + mov_imm(ARG4, Live.get()); + fragment_call(ga->get_garbage_collect()); + mov_arg(preserve_reg, Preserve); + } else { + a.mov(getXRef(Live.get()), preserve_reg); + mov_imm(ARG4, Live.get() + 1); + fragment_call(ga->get_garbage_collect()); + a.mov(preserve_reg, getXRef(Live.get())); + } a.bind(after_gc_check); } @@ -128,6 +154,13 @@ void BeamModuleAssembler::emit_gc_test(const ArgWord &Ns, (Ns.get() + Nh.get() + S_RESERVED) * sizeof(Eterm); Label after_gc_check = a.newLabel(); +#ifdef DEBUG + comment("(debug: fill dead X registers with garbage)"); + mov_imm(ARG4, ERTS_HOLE_MARKER); + mov_arg(ArgXRegister(Live.get()), ARG4); + mov_arg(ArgXRegister(Live.get() + 1), ARG4); +#endif + a.lea(ARG3, x86::qword_ptr(HTOP, bytes_needed)); a.cmp(ARG3, E); a.short_().jbe(after_gc_check); @@ -760,6 +793,70 @@ void BeamModuleAssembler::emit_self(const ArgRegister &Dst) { mov_arg(Dst, ARG1); } +void BeamModuleAssembler::emit_update_record(const ArgAtom &Hint, + const ArgWord &TupleSize, + const ArgSource &Src, + const ArgRegister &Dst, + const ArgWord &UpdateCount, + const Span<ArgVal> &updates) { + size_t copy_index = 0, size_on_heap = TupleSize.get() + 1; + Label next = a.newLabel(); + + x86::Gp ptr_val; + + ASSERT(UpdateCount.get() == updates.size()); + ASSERT((UpdateCount.get() % 2) == 0); + + ASSERT(size_on_heap > 2); + + mov_arg(RET, Src); + + /* Setting a field to the same value is pretty common, so we'll check for + * that since it's vastly cheaper than copying if we're right, and doesn't + * cost much if we're wrong. */ + if (Hint.get() == am_reuse && updates.size() == 2) { + const auto next_index = updates[0].as<ArgWord>().get(); + const auto &next_value = updates[1].as<ArgSource>(); + + a.mov(ARG1, RET); + ptr_val = emit_ptr_val(ARG1, ARG1); + cmp_arg(emit_boxed_val(ptr_val, next_index * sizeof(Eterm)), + next_value, + ARG2); + a.je(next); + } + + ptr_val = emit_ptr_val(RET, RET); + + for (size_t i = 0; i < updates.size(); i += 2) { + const auto next_index = updates[i].as<ArgWord>().get(); + const auto &next_value = updates[i + 1].as<ArgSource>(); + + ASSERT(next_index > 0 && next_index >= copy_index); + + emit_copy_words(emit_boxed_val(ptr_val, copy_index * sizeof(Eterm)), + x86::qword_ptr(HTOP, copy_index * sizeof(Eterm)), + next_index - copy_index, + ARG1); + + mov_arg(x86::qword_ptr(HTOP, next_index * sizeof(Eterm)), + next_value, + ARG1); + copy_index = next_index + 1; + } + + emit_copy_words(emit_boxed_val(ptr_val, copy_index * sizeof(Eterm)), + x86::qword_ptr(HTOP, copy_index * sizeof(Eterm)), + size_on_heap - copy_index, + ARG1); + + a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED)); + a.add(HTOP, imm(size_on_heap * sizeof(Eterm))); + + a.bind(next); + mov_arg(Dst, RET); +} + void BeamModuleAssembler::emit_set_tuple_element(const ArgSource &Element, const ArgRegister &Tuple, const ArgWord &Offset) { @@ -811,50 +908,44 @@ void BeamModuleAssembler::emit_is_boolean(const ArgLabel &Fail, a.jne(resolve_beam_label(Fail)); } -x86::Gp BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src, - Label next, - Label subbin) { +void BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, + const ArgSource &Src) { + Label is_binary = a.newLabel(), next = a.newLabel(); + mov_arg(ARG1, Src); emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK)); - a.cmp(RETb, imm(_TAG_HEADER_SUB_BIN)); - a.short_().je(subbin); - if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_BITSTRING) { + const auto diff_mask = _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); comment("simplified binary test since source is always a bitstring " "when boxed"); + a.test(emit_boxed_val(boxed_ptr, 0, 1), diff_mask); + a.short_().je(next); } else { - ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); - a.and_(RETb, imm(~4)); - a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); - a.jne(resolve_beam_label(Fail)); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_SUB_BIN)); + a.short_().jne(is_binary); } - a.short_().jmp(next); - - return boxed_ptr; -} - -void BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, - const ArgSource &Src) { - Label next = a.newLabel(), subbin = a.newLabel(); - x86::Gp boxed_ptr; - - boxed_ptr = emit_is_binary(Fail, Src, next, subbin); + /* This is a sub binary. */ + a.cmp(emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize), sizeof(byte)), + imm(0)); + a.jne(resolve_beam_label(Fail)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) != BEAM_TYPE_BITSTRING) { + a.short_().jmp(next); + } - a.bind(subbin); - { - /* emit_is_binary has already removed the literal tag from Src, if - * applicable. */ - a.cmp(emit_boxed_val(boxed_ptr, - offsetof(ErlSubBin, bitsize), - sizeof(byte)), - imm(0)); + a.bind(is_binary); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) != BEAM_TYPE_BITSTRING) { + ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); + a.and_(RETb, imm(~4)); + a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); a.jne(resolve_beam_label(Fail)); } @@ -863,11 +954,19 @@ void BeamModuleAssembler::emit_is_binary(const ArgLabel &Fail, void BeamModuleAssembler::emit_is_bitstring(const ArgLabel &Fail, const ArgSource &Src) { - Label next = a.newLabel(); + mov_arg(ARG1, Src); - emit_is_binary(Fail, Src, next, next); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); - a.bind(next); + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + + const auto mask = _HEADER_SUBTAG_MASK - _BINARY_XXX_MASK; + ERTS_CT_ASSERT(TAG_PRIMARY_HEADER == 0); + ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN == (_TAG_HEADER_REFC_BIN & mask)); + a.and_(RETb, imm(mask)); + a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_float(const ArgLabel &Fail, @@ -1024,7 +1123,7 @@ void BeamModuleAssembler::emit_is_map(const ArgLabel &Fail, void BeamModuleAssembler::emit_is_nil(const ArgLabel &Fail, const ArgRegister &Src) { - a.cmp(getArgRef(Src), imm(NIL)); + a.cmp(getArgRef(Src, 1), imm(NIL)); a.jne(resolve_beam_label(Fail)); } @@ -1249,6 +1348,28 @@ void BeamModuleAssembler::emit_i_test_arity(const ArgLabel &Fail, void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, const ArgSource &X, const ArgSource &Y) { + bool is_empty_binary = false; + if (exact_type(X, BEAM_TYPE_BITSTRING) && Y.isLiteral()) { + auto unit = getSizeUnit(X); + if (unit != 0 && std::gcd(unit, 8) == 8) { + Eterm literal = + beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + is_empty_binary = is_binary(literal) && binary_size(literal) == 0; + } + } + + if (is_empty_binary) { + mov_arg(RET, X); + + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + + comment("simplified equality test with empty binary"); + a.cmp(emit_boxed_val(boxed_ptr, sizeof(Eterm)), 0); + a.jne(resolve_beam_label(Fail)); + + return; + } + /* If either argument is known to be an immediate, we can fail immediately * if they're not equal. */ if (always_immediate(X) || always_immediate(Y)) { @@ -1275,14 +1396,24 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, #endif if (always_same_types(X, Y)) { - comment("skipped test of tags since they are always equal"); + comment("skipped tag test since they are always equal"); + } else if (Y.isLiteral()) { + /* Fail immediately unless X is the same type of pointer as + * the literal Y. + */ + Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK); + a.test(ARG1.r8(), imm(tag_test)); + a.jne(resolve_beam_label(Fail)); } else { - /* The terms could still be equal if both operands are pointers - * having the same tag. */ + /* Fail immediately if the pointer tags are not equal. */ emit_is_unequal_based_on_tags(ARG1, ARG2); a.je(resolve_beam_label(Fail)); } + /* Both operands are pointers having the same tag. Must do a + * deeper comparison. */ + emit_enter_runtime(); runtime_call<2>(eq); @@ -1295,31 +1426,31 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail, a.bind(next); } -void BeamModuleAssembler::emit_i_is_eq_exact_literal(const ArgLabel &Fail, - const ArgSource &Src, - const ArgConstant &Literal, - const ArgWord &tag_test) { - mov_arg(ARG2, Literal); /* May clobber ARG1 */ - mov_arg(ARG1, Src); - - /* Fail immediately unless Src is the same type of pointer as the literal. - */ - a.test(ARG1.r8(), imm(tag_test.get())); - a.jne(resolve_beam_label(Fail)); +void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, + const ArgSource &X, + const ArgSource &Y) { + bool is_empty_binary = false; + if (exact_type(X, BEAM_TYPE_BITSTRING) && Y.isLiteral()) { + auto unit = getSizeUnit(X); + if (unit != 0 && std::gcd(unit, 8) == 8) { + Eterm literal = + beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + is_empty_binary = is_binary(literal) && binary_size(literal) == 0; + } + } - emit_enter_runtime(); + if (is_empty_binary) { + mov_arg(RET, X); - runtime_call<2>(eq); + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - emit_leave_runtime(); + comment("simplified non-equality test with empty binary"); + a.cmp(emit_boxed_val(boxed_ptr, sizeof(Eterm)), 0); + a.je(resolve_beam_label(Fail)); - a.test(RETd, RETd); - a.jz(resolve_beam_label(Fail)); -} + return; + } -void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, - const ArgSource &X, - const ArgSource &Y) { /* If either argument is known to be an immediate, we can fail immediately * if they're equal. */ if (always_immediate(X) || always_immediate(Y)) { @@ -1343,6 +1474,18 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, if (always_same_types(X, Y)) { comment("skipped tag test since they are always equal"); + } else if (Y.isLiteral()) { + /* Succeed immediately if X is not the same type of pointer as + * the literal Y. + */ + Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get()); + Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK); + a.test(ARG1.r8(), imm(tag_test)); +#ifdef JIT_HARD_DEBUG + a.jne(next); +#else + a.short_().jne(next); +#endif } else { /* Test whether the terms are definitely unequal based on the tags * alone. */ @@ -1367,32 +1510,6 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail, a.bind(next); } -void BeamModuleAssembler::emit_i_is_ne_exact_literal( - const ArgLabel &Fail, - const ArgSource &Src, - const ArgConstant &Literal) { - Label next = a.newLabel(); - - mov_arg(ARG2, Literal); /* May clobber ARG1 */ - mov_arg(ARG1, Src); - - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); - a.short_().je(next); - - emit_enter_runtime(); - - runtime_call<2>(eq); - - emit_leave_runtime(); - - a.test(RETd, RETd); - a.jnz(resolve_beam_label(Fail)); - - a.bind(next); -} - void BeamGlobalAssembler::emit_arith_eq_shared() { Label generic_compare = a.newLabel(); @@ -1578,14 +1695,36 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS) { - Label generic = a.newLabel(), next = a.newLabel(); + Label generic = a.newLabel(), do_jge = a.newLabel(), next = a.newLabel(); bool both_small = always_small(LHS) && always_small(RHS); + bool need_generic = !both_small; mov_arg(ARG2, RHS); /* May clobber ARG1 */ mov_arg(ARG1, LHS); if (both_small) { comment("skipped test for small operands since they are always small"); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasLowerBound(RHS)) { + comment("simplified test because it always succeeds when RHS is a " + "bignum"); + need_generic = false; + emit_is_not_boxed(next, ARG2, dShort); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasUpperBound(RHS)) { + comment("simplified test because it always fails when RHS is a bignum"); + need_generic = false; + emit_is_not_boxed(resolve_beam_label(Fail), ARG2); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasLowerBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always fails when LHS is a bignum"); + need_generic = false; + emit_is_not_boxed(resolve_beam_label(Fail), ARG1); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasUpperBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always succeeds when LHS is a " + "bignum"); + emit_is_not_boxed(next, ARG1, dShort); } else if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { /* The only possible kind of immediate is a small and all other @@ -1604,6 +1743,10 @@ void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, } else if (LHS.isSmall()) { a.mov(RETd, ARG2d); } else { + /* Avoid the expensive generic comparison for equal terms. */ + a.cmp(ARG1, ARG2); + a.short_().je(do_jge); + a.mov(RETd, ARG1d); a.and_(RETd, ARG2d); } @@ -1615,32 +1758,72 @@ void BeamModuleAssembler::emit_is_lt(const ArgLabel &Fail, /* Both arguments are smalls. */ a.cmp(ARG1, ARG2); - if (!both_small) { - a.short_().jmp(next); + if (need_generic) { + a.short_().jmp(do_jge); } a.bind(generic); { - if (!both_small) { + if (need_generic) { safe_fragment_call(ga->get_arith_compare_shared()); } } - a.bind(next); + a.bind(do_jge); a.jge(resolve_beam_label(Fail)); + + a.bind(next); } void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS) { - Label generic = a.newLabel(), next = a.newLabel(); bool both_small = always_small(LHS) && always_small(RHS); + if (both_small && LHS.isRegister() && RHS.isImmed() && + Support::isInt32(RHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(LHS.as<ArgRegister>()), imm(RHS.as<ArgImmed>().get())); + a.jl(resolve_beam_label(Fail)); + return; + } else if (both_small && RHS.isRegister() && LHS.isImmed() && + Support::isInt32(LHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(RHS.as<ArgRegister>()), imm(LHS.as<ArgImmed>().get())); + a.jg(resolve_beam_label(Fail)); + return; + } + + Label generic = a.newLabel(), do_jl = a.newLabel(), next = a.newLabel(); + bool need_generic = !both_small; + mov_arg(ARG2, RHS); /* May clobber ARG1 */ mov_arg(ARG1, LHS); if (both_small) { comment("skipped test for small operands since they are always small"); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasLowerBound(RHS)) { + comment("simplified test because it always fails when RHS is a bignum"); + need_generic = false; + emit_is_not_boxed(resolve_beam_label(Fail), ARG2); + } else if (always_small(LHS) && exact_type(RHS, BEAM_TYPE_INTEGER) && + hasUpperBound(RHS)) { + comment("simplified test because it always succeeds when RHS is a " + "bignum"); + need_generic = false; + emit_is_not_boxed(next, ARG2, dShort); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasUpperBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always fails when LHS is a bignum"); + need_generic = false; + emit_is_not_boxed(resolve_beam_label(Fail), ARG1); + } else if (exact_type(LHS, BEAM_TYPE_INTEGER) && hasLowerBound(LHS) && + always_small(RHS)) { + comment("simplified test because it always succeeds when LHS is a " + "bignum"); + need_generic = false; + emit_is_not_boxed(next, ARG1, dShort); } else if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { /* The only possible kind of immediate is a small and all other @@ -1659,6 +1842,10 @@ void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, } else if (LHS.isSmall()) { a.mov(RETd, ARG2d); } else { + /* Avoid the expensive generic comparison for equal terms. */ + a.cmp(ARG1, ARG2); + a.short_().je(do_jl); + a.mov(RETd, ARG1d); a.and_(RETd, ARG2d); } @@ -1670,72 +1857,381 @@ void BeamModuleAssembler::emit_is_ge(const ArgLabel &Fail, /* Both arguments are smalls. */ a.cmp(ARG1, ARG2); - if (!both_small) { - a.short_().jmp(next); + if (need_generic) { + a.short_().jmp(do_jl); } a.bind(generic); { - if (!both_small) { + if (need_generic) { safe_fragment_call(ga->get_arith_compare_shared()); } } - a.bind(next); + a.bind(do_jl); a.jl(resolve_beam_label(Fail)); + + a.bind(next); } -void BeamModuleAssembler::emit_bif_is_eq_ne_exact(const ArgSource &LHS, - const ArgSource &RHS, - const ArgRegister &Dst, - Eterm fail_value, - Eterm succ_value) { - /* `mov_imm` may clobber the flags if either value is zero. */ - ASSERT(fail_value && succ_value); +/* + * ARG1 = Src + * ARG2 = Min + * ARG3 = Max + * + * Result is returned in the flags. + */ +void BeamGlobalAssembler::emit_is_in_range_shared() { + Label immediate = a.newLabel(); + Label generic_compare = a.newLabel(); + Label done = a.newLabel(); - mov_imm(RET, succ_value); - cmp_arg(getArgRef(LHS), RHS); + /* Is the source a float? */ + emit_is_boxed(immediate, ARG1); - if (always_immediate(LHS) || always_immediate(RHS)) { - if (!LHS.isImmed() && !RHS.isImmed()) { - comment("simplified check since one argument is an immediate"); - } - mov_imm(ARG1, fail_value); - a.cmovne(RET, ARG1); - } else { - Label next = a.newLabel(); + x86::Gp boxed_ptr = emit_ptr_val(ARG4, ARG1); + a.cmp(emit_boxed_val(boxed_ptr), imm(HEADER_FLONUM)); + a.short_().jne(generic_compare); - a.je(next); + /* Compare the float to the limits. */ + a.movsd(x86::xmm0, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.sar(ARG2, imm(_TAG_IMMED1_SIZE)); + a.sar(ARG3, imm(_TAG_IMMED1_SIZE)); + a.cvtsi2sd(x86::xmm1, ARG2); + a.cvtsi2sd(x86::xmm2, ARG3); + a.xor_(x86::ecx, x86::ecx); + a.ucomisd(x86::xmm0, x86::xmm2); + a.seta(x86::cl); + mov_imm(RET, -1); + a.ucomisd(x86::xmm1, x86::xmm0); + a.cmovbe(RET, x86::rcx); + + a.cmp(RET, imm(0)); + + a.ret(); - mov_arg(ARG1, LHS); - mov_arg(ARG2, RHS); + a.bind(immediate); + { + /* + * Src is an immediate (such as ATOM) but not SMALL. + * That means that Src must be greater than the upper + * limit. + */ + mov_imm(RET, 1); + a.cmp(RET, imm(0)); + a.ret(); + } + a.bind(generic_compare); + { emit_enter_runtime(); - runtime_call<2>(eq); - emit_leave_runtime(); + a.mov(TMP_MEM1q, ARG1); + a.mov(TMP_MEM2q, ARG3); + + comment("erts_cmp_compound(X, Y, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); a.test(RET, RET); + a.js(done); - mov_imm(RET, succ_value); - mov_imm(ARG1, fail_value); - a.cmove(RET, ARG1); + a.mov(ARG1, TMP_MEM1q); + a.mov(ARG2, TMP_MEM2q); - a.bind(next); + comment("erts_cmp_compound(X, Y, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.test(RET, RET); + + a.bind(done); + emit_leave_runtime(); + + a.ret(); } +} - mov_arg(Dst, RET); +void BeamModuleAssembler::emit_is_in_range(ArgLabel const &Small, + ArgLabel const &Large, + ArgRegister const &Src, + ArgConstant const &Min, + ArgConstant const &Max) { + Label next = a.newLabel(), generic = a.newLabel(); + bool need_generic = true; + + mov_arg(ARG1, Src); + + if (always_small(Src)) { + need_generic = false; + comment("skipped test for small operand since it always small"); + } else if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all + * other values are boxed, so we can test for smalls by + * testing boxed. */ + comment("simplified small test since all other types are boxed"); + ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); + if (Small == Large && + always_one_of(Src, BEAM_TYPE_MASK_BOXED - BEAM_TYPE_FLOAT)) { + /* Src is never a float and the failure labels are + * equal. Therefore, since a bignum will never be within + * the range, we can fail immediately if Src is not a + * small. */ + need_generic = false; + a.test(ARG1.r8(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.je(resolve_beam_label(Small)); + } else { + /* Src can be a float or the failures labels are distinct. + * We need to call the generic routine if Src is not a small. */ + a.test(ARG1.r8(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.short_().je(generic); + } + } else if (Small == Large) { + /* We can save one instruction if we incorporate the test for + * small into the range check. */ + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + comment("simplified small & range tests since failure labels are " + "equal"); + a.mov(RET, ARG1); + sub(RET, Min.as<ArgImmed>().get(), ARG4); + + /* Since we have subtracted the (tagged) lower bound, the + * tag bits of the difference is 0 if and only if Src is + * a small. Testing for a tag of 0 can be done in two + * instructions. */ + a.test(RETb, imm(_TAG_IMMED1_MASK)); + a.jne(generic); + + /* Now do the range check. */ + cmp(RET, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get(), ARG4); + a.ja(resolve_beam_label(Small)); + + /* Bypass the test code. */ + goto test_done; + } else { + /* We have no applicable type information and the failure + * labels are distinct. Emit the standard test for small + * and call the generic routine if Src is not a small. */ + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + + /* We have now established that the operand is small. */ + if (Small == Large) { + comment("simplified range test since failure labels are equal"); + sub(ARG1, Min.as<ArgImmed>().get(), RET); + cmp(ARG1, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get(), RET); + a.ja(resolve_beam_label(Small)); + } else { + cmp(ARG1, Min.as<ArgImmed>().get(), RET); + a.jl(resolve_beam_label(Small)); + cmp(ARG1, Max.as<ArgImmed>().get(), RET); + a.jg(resolve_beam_label(Large)); + } + +test_done: + if (need_generic) { + a.short_().jmp(next); + } + + a.bind(generic); + if (!need_generic) { + comment("skipped generic comparison because it is not needed"); + } else { + mov_arg(ARG2, Min); + mov_arg(ARG3, Max); + safe_fragment_call(ga->get_is_in_range_shared()); + if (Small == Large) { + a.jne(resolve_beam_label(Small)); + } else { + a.jl(resolve_beam_label(Small)); + a.jg(resolve_beam_label(Large)); + } + } + + a.bind(next); } -void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgRegister &LHS, - const ArgSource &RHS, - const ArgRegister &Dst) { - emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_false, am_true); +/* + * ARG1 = Src + * ARG2 = A + * ARG3 = B + * + * Result is returned in the flags. + */ +void BeamGlobalAssembler::emit_is_ge_lt_shared() { + Label done = a.newLabel(); + + emit_enter_runtime(); + + a.mov(TMP_MEM1q, ARG1); + a.mov(TMP_MEM2q, ARG3); + + comment("erts_cmp_compound(Src, A, 0, 0);"); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + a.test(RET, RET); + a.short_().js(done); + + comment("erts_cmp_compound(B, Src, 0, 0);"); + a.mov(ARG1, TMP_MEM2q); + a.mov(ARG2, TMP_MEM1q); + mov_imm(ARG3, 0); + mov_imm(ARG4, 0); + runtime_call<4>(erts_cmp_compound); + + /* The following instructions implements the signum function. */ + mov_imm(ARG1, -1); + mov_imm(ARG4, 1); + a.test(RET, RET); + a.cmovs(RET, ARG1); + a.cmovg(RET, ARG4); + + /* RET is now -1, 0, or 1. */ + a.add(RET, imm(1)); + + /* We now have: + * RET == 0 if B < SRC + * RET > 0 if B => SRC + * and flags set accordingly. */ + + a.bind(done); + emit_leave_runtime(); + + a.ret(); +} + +/* + * is_ge + is_lt is 20 instructions. + * + * is_ge_lt is 15 instructions. + */ +void BeamModuleAssembler::emit_is_ge_lt(ArgLabel const &Fail1, + ArgLabel const &Fail2, + ArgRegister const &Src, + ArgConstant const &A, + ArgConstant const &B) { + Label generic = a.newLabel(), next = a.newLabel(); + + mov_arg(ARG2, A); + mov_arg(ARG3, B); + mov_arg(ARG1, Src); + + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + + a.cmp(ARG1, ARG2); + a.jl(resolve_beam_label(Fail1)); + a.cmp(ARG3, ARG1); + a.jge(resolve_beam_label(Fail2)); + a.short_().jmp(next); + + a.bind(generic); + safe_fragment_call(ga->get_is_ge_lt_shared()); + a.jl(resolve_beam_label(Fail1)); + a.jg(resolve_beam_label(Fail2)); + + a.bind(next); } -void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgRegister &LHS, - const ArgSource &RHS, - const ArgRegister &Dst) { - emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_true, am_false); +/* + * The optimized instruction sequence is not always shorter, + * but it ensures that Src is only read from memory once. + */ +void BeamModuleAssembler::emit_is_ge_ge(ArgLabel const &Fail1, + ArgLabel const &Fail2, + ArgRegister const &Src, + ArgConstant const &A, + ArgConstant const &B) { + if (!always_small(Src)) { + /* In practice, it is uncommon that Src is not a known small + * integer, so we will not bother optimizing that case. */ + emit_is_ge(Fail1, Src, A); + emit_is_ge(Fail2, Src, B); + return; + } + + mov_arg(RET, Src); + sub(RET, A.as<ArgImmed>().get(), ARG1); + a.jl(resolve_beam_label(Fail1)); + cmp(RET, B.as<ArgImmed>().get() - A.as<ArgImmed>().get(), ARG1); + a.jb(resolve_beam_label(Fail2)); +} + +/* + * Combine is_integer with range check. + * + * is_integer + is_ge + is_ge is 31 instructions. + * + * is_int_in_range is 6 instructions. + */ +void BeamModuleAssembler::emit_is_int_in_range(ArgLabel const &Fail, + ArgRegister const &Src, + ArgConstant const &Min, + ArgConstant const &Max) { + mov_arg(RET, Src); + + sub(RET, Min.as<ArgImmed>().get(), ARG1); + + /* Since we have subtracted the (tagged) lower bound, the + * tag bits of the difference is 0 if and only if Src is + * a small. */ + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.test(RETb, imm(_TAG_IMMED1_MASK)); + a.jne(resolve_beam_label(Fail)); + cmp(RET, Max.as<ArgImmed>().get() - Min.as<ArgImmed>().get(), ARG1); + a.ja(resolve_beam_label(Fail)); +} + +/* + * is_integer + is_ge is 21 instructions. + * + * is_int_ge is 14 instructions. + */ +void BeamModuleAssembler::emit_is_int_ge(ArgLabel const &Fail, + ArgRegister const &Src, + ArgConstant const &Min) { + Label small = a.newLabel(); + Label fail = a.newLabel(); + Label next = a.newLabel(); + /* On Unix, using rcx instead of ARG1 makes the `test` instruction + * in the boxed test one byte shorter. */ + const x86::Gp src_reg = x86::rcx; + + mov_arg(src_reg, Src); + + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types are boxed"); + emit_is_boxed(small, Src, src_reg); + } else { + a.mov(RETd, src_reg.r32()); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().je(small); + + emit_is_boxed(resolve_beam_label(Fail), Src, src_reg); + } + + /* Src is boxed. Jump to failure unless Src is a positive bignum. */ + x86::Gp boxed_ptr = emit_ptr_val(src_reg, src_reg); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_POS_BIG)); + a.short_().je(next); + + a.bind(fail); + a.jmp(resolve_beam_label(Fail)); + + a.bind(small); + cmp(src_reg, Min.as<ArgImmed>().get(), RET); + a.short_().jl(fail); + + a.bind(next); } void BeamModuleAssembler::emit_badmatch(const ArgSource &Src) { diff --git a/erts/emulator/beam/jit/x86/instr_fun.cpp b/erts/emulator/beam/jit/x86/instr_fun.cpp index 1ae19aaaba..f51c81f4c9 100644 --- a/erts/emulator/beam/jit/x86/instr_fun.cpp +++ b/erts/emulator/beam/jit/x86/instr_fun.cpp @@ -242,7 +242,7 @@ void BeamGlobalAssembler::emit_apply_fun_shared() { a.cmp(ARG1d, imm(NIL)); a.short_().je(finished); - a.test(ARG1d, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); + a.test(ARG1.r8(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); a.short_().jne(malformed_list); emit_ptr_val(ARG1, ARG1); diff --git a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp index 4dfe39cb98..e4691b5943 100644 --- a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp @@ -19,6 +19,7 @@ */ #include <algorithm> +#include <numeric> #include "beam_asm.hpp" extern "C" @@ -28,58 +29,306 @@ extern "C" #include "beam_catches.h" #include "beam_common.h" #include "code_ix.h" +#include "erl_map.h" } using namespace asmjit; -/* - * We considered specializing tuple_size/1, but ultimately didn't - * consider it worth doing. - * - * At the time of writing, there were 294 uses of tuple_size/1 - * in the OTP source code. (11 of them were in dialyzer.) - * - * The code size for the specialization was 34 bytes, - * while the code size for the bif1 instruction was 24 bytes. +/* ================================================================ + * '=:='/2 + * '=/='/2 + * '>='/2 + * '<'/2 + * ================================================================ */ -void BeamGlobalAssembler::emit_handle_hd_error() { - static ErtsCodeMFA mfa = {am_erlang, am_hd, 1}; +void BeamModuleAssembler::emit_bif_is_eq_ne_exact(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst, + Eterm fail_value, + Eterm succ_value) { + /* `mov_imm` may clobber the flags if either value is zero. */ + ASSERT(fail_value && succ_value); - a.mov(getXRef(0), RET); - a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADARG)); - a.mov(ARG4, imm(&mfa)); - a.jmp(labels[raise_exception]); + cmp_arg(getArgRef(LHS), RHS); + mov_imm(RET, succ_value); + + if (always_immediate(LHS) || always_immediate(RHS)) { + if (!LHS.isImmed() && !RHS.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + mov_imm(ARG1, fail_value); + a.cmovne(RET, ARG1); + } else { + Label next = a.newLabel(); + + a.je(next); + + mov_arg(ARG1, LHS); + mov_arg(ARG2, RHS); + + emit_enter_runtime(); + runtime_call<2>(eq); + emit_leave_runtime(); + + a.test(RET, RET); + + mov_imm(RET, succ_value); + mov_imm(ARG1, fail_value); + a.cmove(RET, ARG1); + + a.bind(next); + } + + mov_arg(Dst, RET); } -/* - * At the time of implementation, there were 3285 uses of hd/1 in - * the OTP source code. Most of them were in code generated by - * yecc. - * - * The code size for this specialization of hd/1 is 21 bytes, - * while the code size for the bif1 instruction is 24 bytes. +void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgRegister &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_false, am_true); +} + +void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgRegister &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_true, am_false); +} + +void BeamModuleAssembler::emit_cond_to_bool(uint32_t instId, + const ArgRegister &Dst) { + mov_imm(RET, am_true); + mov_imm(ARG1, am_false); + a.emit(instId, RET, ARG1); + mov_arg(Dst, RET); +} + +void BeamModuleAssembler::emit_bif_is_ge(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + bool both_small = always_small(LHS) && always_small(RHS); + + if (both_small && LHS.isRegister() && RHS.isImmed() && + Support::isInt32(RHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(LHS.as<ArgRegister>()), imm(RHS.as<ArgImmed>().get())); + emit_cond_to_bool(x86::Inst::kIdCmovl, Dst); + + return; + } else if (both_small && RHS.isRegister() && LHS.isImmed() && + Support::isInt32(LHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(RHS.as<ArgRegister>()), imm(LHS.as<ArgImmed>().get())); + emit_cond_to_bool(x86::Inst::kIdCmovg, Dst); + + return; + } + + Label generic = a.newLabel(), do_jl = a.newLabel(); + + mov_arg(ARG2, RHS); /* May clobber ARG1 */ + mov_arg(ARG1, LHS); + + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all other + * values are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.short_().je(generic); + } else { + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (RHS.isSmall()) { + a.mov(RETd, ARG1d); + } else if (LHS.isSmall()) { + a.mov(RETd, ARG2d); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } + + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + + /* Both arguments are smalls. */ + a.cmp(ARG1, ARG2); + a.short_().jmp(do_jl); + + a.bind(generic); + { + a.cmp(ARG1, ARG2); + a.short_().je(do_jl); + safe_fragment_call(ga->get_arith_compare_shared()); + } + + a.bind(do_jl); + emit_cond_to_bool(x86::Inst::kIdCmovl, Dst); +} + +void BeamModuleAssembler::emit_bif_is_lt(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + Label generic = a.newLabel(), do_jge = a.newLabel(); + + mov_arg(ARG2, RHS); /* May clobber ARG1 */ + mov_arg(ARG1, LHS); + + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (RHS.isSmall()) { + a.mov(RETd, ARG1d); + } else if (LHS.isSmall()) { + a.mov(RETd, ARG2d); + } else { + /* Avoid the expensive generic comparison for equal terms. */ + a.cmp(ARG1, ARG2); + a.short_().je(do_jge); + + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } + + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + + /* Both arguments are smalls. */ + a.cmp(ARG1, ARG2); + a.short_().jmp(do_jge); + + a.bind(generic); + safe_fragment_call(ga->get_arith_compare_shared()); + + a.bind(do_jge); + emit_cond_to_bool(x86::Inst::kIdCmovge, Dst); +} + +/* ================================================================ + * bit_size/1 + * ================================================================ */ -void BeamModuleAssembler::emit_bif_hd(const ArgLabel &Fail, - const ArgSource &Src, - const ArgRegister &Hd) { - mov_arg(RET, Src); - a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - if (Fail.get() != 0) { - a.jne(resolve_beam_label(Fail)); +void BeamModuleAssembler::emit_bif_bit_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type(Src, BEAM_TYPE_BITSTRING)) { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + return; + } + + mov_arg(ARG2, Src); + + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); + + if (is_bitstring) { + comment("inlined bit_size/1 because " + "its argument is a bitstring"); } else { - Label next = a.newLabel(); - a.short_().je(next); - safe_fragment_call(ga->get_handle_hd_error()); - a.bind(next); + comment("inlined and simplified bit_size/1 because " + "its argument is a binary"); } - x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - a.mov(ARG2, getCARRef(boxed_ptr)); - mov_arg(Hd, ARG2); + if (is_bitstring) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + } + + a.mov(ARG1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.shl(ARG1, imm(3 + _TAG_IMMED1_SIZE)); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const auto diff_mask = _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); + a.test(RETb, imm(diff_mask)); + a.short_().jz(not_sub_bin); + + a.mov(RETb, emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize), 1)); + a.shl(RETb, imm(_TAG_IMMED1_SIZE)); + a.add(ARG1.r8(), RETb); + + a.bind(not_sub_bin); + } + + a.or_(ARG1, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, ARG1); } +/* ================================================================ + * byte_size/1 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_byte_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type(Src, BEAM_TYPE_BITSTRING)) { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + return; + } + + mov_arg(ARG2, Src); + + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); + + if (is_bitstring) { + comment("inlined byte_size/1 because " + "its argument is a bitstring"); + } else { + comment("inlined and simplified byte_size/1 because " + "its argument is a binary"); + } + + if (is_bitstring) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + } + + a.mov(ARG1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const auto diff_mask = _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); + a.test(RETb, imm(diff_mask)); + a.short_().jz(not_sub_bin); + + a.mov(RETb, emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize), 1)); + a.test(RETb, RETb); + a.setne(RETb); + a.movzx(RETd, RETb); + a.add(ARG1, RET); + + a.bind(not_sub_bin); + } + + a.shl(ARG1, imm(_TAG_IMMED1_SIZE)); + a.or_(ARG1, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, ARG1); +} + +/* ================================================================ + * element/2 + * ================================================================ + */ + void BeamGlobalAssembler::emit_handle_element_error() { static ErtsCodeMFA mfa = {am_erlang, am_element, 2}; @@ -169,9 +418,8 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, if (is_tuple(tuple)) { Label error = a.newLabel(), next = a.newLabel(); Sint size = Sint(arityval(*tuple_val(tuple))); - auto [min, max] = getIntRange(Pos); - bool is_bounded = min <= max; - bool can_fail = !is_bounded || min < 1 || size < max; + auto [min, max] = getClampedRange(Pos); + bool can_fail = min < 1 || size < max; comment("skipped tuple test since source is always a literal " "tuple"); @@ -191,13 +439,13 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, a.mov(RET, ARG1); a.sar(RET, imm(_TAG_IMMED1_SIZE)); - if (is_bounded && min >= 1) { + if (min >= 1) { comment("skipped check for position =:= 0 since it is always " ">= 1"); } else { a.short_().jz(error); } - if (is_bounded && min >= 0 && size >= max) { + if (min >= 0 && size >= max) { comment("skipped check for negative position and position " "beyond tuple"); } else { @@ -319,3 +567,268 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, mov_arg(Dst, RET); } + +/* ================================================================ + * hd/1 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_hd_error() { + static ErtsCodeMFA mfa = {am_erlang, am_hd, 1}; + + a.mov(getXRef(0), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADARG)); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +/* + * At the time of implementation, there were 3285 uses of hd/1 in + * the OTP source code. Most of them were in code generated by + * yecc. + * + * The code size for this specialization of hd/1 is 21 bytes, + * while the code size for the bif1 instruction is 24 bytes. + */ +void BeamModuleAssembler::emit_bif_hd(const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Hd) { + mov_arg(RET, Src); + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); + + if (Fail.get() != 0) { + a.jne(resolve_beam_label(Fail)); + } else { + Label next = a.newLabel(); + a.short_().je(next); + safe_fragment_call(ga->get_handle_hd_error()); + a.bind(next); + } + + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(ARG2, getCARRef(boxed_ptr)); + mov_arg(Hd, ARG2); +} + +/* ================================================================ + * is_map_key/2 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_is_map_key(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type(Src, BEAM_TYPE_MAP)) { + emit_i_bif2(Key, Src, Fail, Bif, Dst); + return; + } + + comment("inlined BIF is_map_key/2"); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (masked_types(Key, BEAM_TYPE_MASK_IMMEDIATE) != BEAM_TYPE_NONE && + hasCpuFeature(CpuFeatures::X86::kBMI2)) { + safe_fragment_call(ga->get_i_get_map_element_shared()); + emit_cond_to_bool(x86::Inst::kIdCmovne, Dst); + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + emit_test_the_non_value(RET); + emit_cond_to_bool(x86::Inst::kIdCmove, Dst); + } +} + +/* ================================================================ + * map_get/2 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_map_get_badmap() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + a.mov(getXRef(0), ARG2); + a.mov(getXRef(1), ARG1); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADMAP)); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG1); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamGlobalAssembler::emit_handle_map_get_badkey() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + a.mov(getXRef(0), ARG2); + a.mov(getXRef(1), ARG1); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADKEY)); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG2); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_map_get(const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + Label good_key = a.newLabel(); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (exact_type(Src, BEAM_TYPE_MAP)) { + comment("skipped test for map for known map argument"); + } else { + Label bad_map = a.newLabel(); + Label good_map = a.newLabel(); + + if (Fail.get() == 0) { + emit_is_boxed(bad_map, Src, ARG1); + } else { + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); + } + + /* As an optimization for the `error | #{}` case, skip checking the + * header word when we know that the only possible boxed type + * is a map. */ + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_MAP) { + comment("skipped header test since we know it's a map when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(RET, ARG1); + a.mov(RET, emit_boxed_val(boxed_ptr)); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_MAP)); + if (Fail.get() == 0) { + a.je(good_map); + } else { + a.jne(resolve_beam_label(Fail)); + } + } + + a.bind(bad_map); + if (Fail.get() == 0) { + fragment_call(ga->get_handle_map_get_badmap()); + } + + a.bind(good_map); + } + + if (masked_types(Key, BEAM_TYPE_MASK_IMMEDIATE) != BEAM_TYPE_NONE && + hasCpuFeature(CpuFeatures::X86::kBMI2)) { + safe_fragment_call(ga->get_i_get_map_element_shared()); + if (Fail.get() == 0) { + a.je(good_key); + } else { + a.jne(resolve_beam_label(Fail)); + } + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + emit_test_the_non_value(RET); + if (Fail.get() == 0) { + a.short_().jne(good_key); + } else { + a.je(resolve_beam_label(Fail)); + } + } + + if (Fail.get() == 0) { + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + fragment_call(ga->get_handle_map_get_badkey()); + } + + a.bind(good_key); + mov_arg(Dst, RET); +} + +/* ================================================================ + * map_size/1 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_map_size_error() { + static ErtsCodeMFA mfa = {am_erlang, am_map_size, 1}; + + a.mov(getXRef(0), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADMAP)); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_map_size(const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + Label error = a.newLabel(), good_map = a.newLabel(); + + mov_arg(RET, Src); + + if (Fail.get() == 0) { + emit_is_boxed(error, Src, RET); + } else { + emit_is_boxed(resolve_beam_label(Fail), Src, RET); + } + + x86::Gp boxed_ptr = emit_ptr_val(x86::rdx, RET); + + if (exact_type(Src, BEAM_TYPE_MAP)) { + comment("skipped type check because the argument is always a map"); + a.bind(error); /* Never referenced. */ + } else { + a.mov(x86::ecx, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(x86::cl, imm(_TAG_HEADER_MASK)); + a.cmp(x86::cl, imm(_TAG_HEADER_MAP)); + if (Fail.get() == 0) { + a.short_().je(good_map); + + a.bind(error); + safe_fragment_call(ga->get_handle_map_size_error()); + } else { + a.jne(resolve_beam_label(Fail)); + a.bind(error); /* Never referenced. */ + } + } + + a.bind(good_map); + { + ERTS_CT_ASSERT(offsetof(flatmap_t, size) == sizeof(Eterm)); + a.mov(RET, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.shl(RET, imm(4)); + a.or_(RETb, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, RET); + } +} + +/* ================================================================ + * tuple_size/1 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_tuple_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgRegister &Src, + const ArgRegister &Dst) { + if (exact_type(Src, BEAM_TYPE_TUPLE)) { + comment("inlined tuple_size/1 because the argument is always a tuple"); + mov_arg(RET, Src); + + /* Instructions operating on dwords are shorter. */ + ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + + ERTS_CT_ASSERT(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE > 0); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.shr(RETd, imm(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE)); + a.or_(RETb, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, RET); + } else { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + } +} diff --git a/erts/emulator/beam/jit/x86/instr_map.cpp b/erts/emulator/beam/jit/x86/instr_map.cpp index cf0063e967..9a94e37118 100644 --- a/erts/emulator/beam/jit/x86/instr_map.cpp +++ b/erts/emulator/beam/jit/x86/instr_map.cpp @@ -25,6 +25,7 @@ using namespace asmjit; extern "C" { #include "erl_map.h" +#include "erl_term_hashing.h" #include "beam_common.h" } @@ -46,11 +47,17 @@ static const Uint32 HCONST = 0x9E3779B9; * * Result is returned in ARG3. */ void BeamGlobalAssembler::emit_internal_hash_helper() { - x86::Gp hash = ARG3d, lower = ARG4d, upper = ARG5d; + x86::Gp hash = ARG3d, lower = ARG4d, upper = ARG5d, constant = ARG6d; - a.add(lower, ARG6d); - a.add(upper, ARG6d); + a.add(lower, constant); + a.add(upper, constant); +#if defined(ERL_INTERNAL_HASH_CRC32C) + a.mov(constant, hash); + a.crc32(hash, lower); + a.add(hash, constant); + a.crc32(hash, upper); +#else using rounds = std::initializer_list<std::tuple<x86::Gp, x86::Gp, x86::Gp, int>>; for (const auto &round : rounds{{lower, upper, hash, 13}, @@ -79,6 +86,7 @@ void BeamGlobalAssembler::emit_internal_hash_helper() { a.xor_(r_a, ARG6d); } +#endif a.ret(); } @@ -140,7 +148,7 @@ void BeamGlobalAssembler::emit_hashmap_get_element() { emit_ptr_val(node, node); /* Have we found our leaf? */ - a.test(node.r32(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); + a.test(node.r8(), imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); a.short_().je(leaf_node); /* Nope, we have to search another node. */ @@ -252,37 +260,80 @@ void BeamModuleAssembler::emit_new_map(const ArgRegister &Dst, mov_arg(Dst, RET); } -void BeamGlobalAssembler::emit_i_new_small_map_lit_shared() { - emit_enter_frame(); - emit_enter_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); - - a.mov(ARG1, c_p); - load_x_reg_array(ARG2); - runtime_call<5>(erts_gc_new_small_map_lit); - - emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); - emit_leave_frame(); - - a.ret(); -} - void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgRegister &Dst, const ArgWord &Live, const ArgLiteral &Keys, const ArgWord &Size, const Span<ArgVal> &args) { - Label data = embed_vararg_rodata(args, CP_SIZE); - ASSERT(Size.get() == args.size()); - ASSERT(Keys.isLiteral()); - mov_arg(ARG3, Keys); - mov_imm(ARG4, Live.get()); - a.lea(ARG5, x86::qword_ptr(data)); + emit_gc_test(ArgWord(0), + ArgWord(args.size() + MAP_HEADER_FLATMAP_SZ + 1), + Live); - fragment_call(ga->get_i_new_small_map_lit_shared()); + std::vector<ArgVal> data; + data.reserve(args.size() + MAP_HEADER_FLATMAP_SZ + 1); + data.push_back(ArgWord(MAP_HEADER_FLATMAP)); + data.push_back(Size); + data.push_back(Keys); - mov_arg(Dst, RET); + for (auto arg : args) { + data.push_back(arg); + } + + size_t size = data.size(); + unsigned i; + + mov_arg(x86::qword_ptr(HTOP), data[0]); + + /* Starting from 1 instead of 0 gives more opportunities for + * applying the MMX optimizations. */ + for (i = 1; i < size - 1; i += 2) { + x86::Mem dst_ptr0 = x86::qword_ptr(HTOP, i * sizeof(Eterm)); + x86::Mem dst_ptr1 = x86::qword_ptr(HTOP, (i + 1) * sizeof(Eterm)); + auto first = data[i]; + auto second = data[i + 1]; + + switch (ArgVal::memory_relation(first, second)) { + case ArgVal::consecutive: { + x86::Mem src_ptr = getArgRef(first, 16); + + comment("(initializing two elements at once)"); + dst_ptr0.setSize(16); + a.movups(x86::xmm0, src_ptr); + a.movups(dst_ptr0, x86::xmm0); + break; + } + case ArgVal::reverse_consecutive: { + if (!hasCpuFeature(CpuFeatures::X86::kAVX)) { + mov_arg(dst_ptr0, first); + mov_arg(dst_ptr1, second); + } else { + x86::Mem src_ptr = getArgRef(second, 16); + + comment("(initializing with two swapped elements at once)"); + dst_ptr0.setSize(16); + a.vpermilpd(x86::xmm0, src_ptr, 1); /* Load and swap */ + a.vmovups(dst_ptr0, x86::xmm0); + } + break; + } + case ArgVal::none: + mov_arg(dst_ptr0, first); + mov_arg(dst_ptr1, second); + break; + } + } + + if (i < size) { + x86::Mem dst_ptr = x86::qword_ptr(HTOP, i * sizeof(Eterm)); + mov_arg(dst_ptr, data[i]); + } + + a.lea(ARG1, x86::byte_ptr(HTOP, TAG_PRIMARY_BOXED)); + a.add(HTOP, imm(size * sizeof(Eterm))); + + mov_arg(Dst, ARG1); } /* ARG1 = map, ARG2 = key diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab index 1ce0d6f9c2..788d09f1eb 100644 --- a/erts/emulator/beam/jit/x86/ops.tab +++ b/erts/emulator/beam/jit/x86/ops.tab @@ -427,21 +427,48 @@ is_eq_exact Lbl LHS RHS | equal(LHS, RHS) => _ is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C is_eq_exact Lbl R=xy n => is_nil Lbl R -is_eq_exact Lbl R=xy C=q => is_eq_exact_literal(Lbl, R, C) is_ne_exact Lbl LHS RHS | equal(LHS, RHS) => jump Lbl is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C -is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C +is_eq_exact f s s -i_is_eq_exact_literal/4 -i_is_eq_exact_literal f s c I +is_ne_exact f s s -i_is_ne_exact_literal f s c +is_integer NotInt N0 | is_ge Small N1=xy Min=i | is_ge Large Max=i N2=xy | + equal(N0, N1) | equal(N1, N2) | + equal(NotInt, Small) | equal(Small, Large) => + is_int_in_range NotInt N0 Min Max -is_eq_exact f s s +is_integer NotInt N0 | is_ge Large Max=i N2=xy | is_ge Small N1=xy Min=i | + equal(N0, N1) | equal(N1, N2) | + equal(NotInt, Small) | equal(Small, Large) => + is_int_in_range NotInt N0 Min Max -is_ne_exact f s s +is_integer NotInt N0 | is_ge Fail N1=xy Min=i | + equal(N0, N1) | equal(NotInt, Fail) => + is_int_ge NotInt N0 Min + +is_int_in_range f S c c +is_int_ge f S c + +is_ge Small N1=xy Min=i | is_ge Large Max=i N2=xy | equal(N1, N2) => + is_in_range Small Large N1 Min Max + +is_ge Large Max=i N2=xy | is_ge Small N1=xy Min=i | equal(N1, N2) => + is_in_range Small Large N2 Min Max + +is_in_range f f S c c + +is_ge Small N1=xy A=i | is_lt Large B=i N2=xy | equal(N1, N2) => + is_ge_lt Small Large N1 A B + +is_ge_lt f f S c c + +is_ge Fail1 N1=xy A=i | is_ge Fail2 N2=xy B=i | equal(N1, N2) => + is_ge_ge Fail1 Fail2 N1 A B + +is_ge_ge f f S c c is_lt f s s is_ge f s s @@ -679,6 +706,26 @@ bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => get(Src, Dst) bif2 Fail u$bif:erlang:element/2 S1=ixy S2 Dst => bif_element Fail S1 S2 Dst bif_element j s s d +gc_bif1 Fail Live Bif=u$bif:erlang:bit_size/1 Src Dst=d => + bif_bit_size Bif Fail Src Dst +bif_bit_size b j s d + +gc_bif1 Fail Live Bif=u$bif:erlang:byte_size/1 Src Dst=d => + bif_byte_size Bif Fail Src Dst +bif_byte_size b j s d + +bif1 Fail Bif=u$bif:erlang:tuple_size/1 Src=d Dst=d => + bif_tuple_size Bif Fail Src Dst +bif_tuple_size b j S d + +bif2 Fail Bif=u$bif:erlang:map_get/2 Src1 Src2=xy Dst=d => + bif_map_get Fail Src1 Src2 Dst +bif_map_get j s s d + +bif2 Fail Bif=u$bif:erlang:is_map_key/2 Key Map=xy Dst=d => + bif_is_map_key Bif Fail Key Map Dst +bif_is_map_key b j s s d + bif1 Fail Bif S1 Dst | never_fails(Bif) => nofail_bif1 S1 Bif Dst bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst @@ -687,6 +734,8 @@ bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst nofail_bif2 S1=d S2 Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact S1 S2 Dst nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst +nofail_bif2 S1 S2 Bif Dst | is_ge_bif(Bif) => bif_is_ge S1 S2 Dst +nofail_bif2 S1 S2 Bif Dst | is_lt_bif(Bif) => bif_is_lt S1 S2 Dst i_get_hash c I d i_get s d @@ -704,6 +753,8 @@ i_bif3 s s s j b d bif_is_eq_exact S s d bif_is_ne_exact S s d +bif_is_ge s s d +bif_is_lt s s d # # Internal calls. @@ -814,26 +865,30 @@ i_lambda_trampoline F f W W i_breakpoint_trampoline # ================================================================ -# New bit syntax matching (R11B). +# New bit syntax matching for fixed sizes (from OTP 26). +# ================================================================ + +bs_match Fail Ctx Size Rest=* => bs_match(Fail, Ctx, Size, Rest) + +i_bs_match Fail Ctx Rest=* | test_heap Need Live => + i_bs_match_test_heap Fail Ctx Need Live Rest + +i_bs_match f S * +i_bs_match_test_heap f S I t * + +# ================================================================ +# Bit syntax matching (from R11B). # ================================================================ %warm -# Matching integers +# Matching integers. bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val i_bs_match_string S f W M # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => - get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) - -i_bs_get_integer S f t t s d - -i_bs_get_integer_8 S t f d -i_bs_get_integer_16 S t f d -i_bs_get_integer_32 S t f d -i_bs_get_integer_64 S t f t d +bs_get_integer2 f S t s t t d # Fetching binaries from binaries. bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => @@ -1323,6 +1378,13 @@ i_length_setup j t s i_length j t d # +# Specialized guard BIFs. +# + +gc_bif1 Fail Live Bif=u$bif:erlang:map_size/1 Src Dst=d => bif_map_size Fail Src Dst +bif_map_size j s d + +# # Guard BIFs. # gc_bif1 Fail Live Bif Src Dst => i_bif1 Src Fail Bif Dst @@ -1379,3 +1441,9 @@ recv_marker_reserve S recv_marker_bind S S recv_marker_clear S recv_marker_use S + +# +# OTP 26 +# + +update_record a I s d I * diff --git a/erts/emulator/beam/jit/x86/predicates.tab b/erts/emulator/beam/jit/x86/predicates.tab index cd1230cd1a..278dbb24c3 100644 --- a/erts/emulator/beam/jit/x86/predicates.tab +++ b/erts/emulator/beam/jit/x86/predicates.tab @@ -35,7 +35,7 @@ pred.is_mfa_bif(M, F, A) { pred.never_fails(Bif) { static Eterm nofail_bifs[] = {am_Neqeq, - am_Le, + am_Lt, am_Neq, am_Eq, am_Le, @@ -107,3 +107,25 @@ pred.consecutive_words(S1, D1, S2, D2) { return S1.type == S2.type && S1.val + 1 == S2.val && D1.type == D2.type && D1.val + 1 == D2.val; } + +pred.is_ge_bif(Bif) { + Uint index = Bif.val; + + if (Bif.type == TAG_u && index < S->beam.imports.count) { + BeamFile_ImportEntry *entry = &S->beam.imports.entries[index]; + + return entry->module == am_erlang && entry->function == am_Ge && entry->arity == 2; + } + return 0; +} + +pred.is_lt_bif(Bif) { + Uint index = Bif.val; + + if (Bif.type == TAG_u && index < S->beam.imports.count) { + BeamFile_ImportEntry *entry = &S->beam.imports.entries[index]; + + return entry->module == am_erlang && entry->function == am_Lt && entry->arity == 2; + } + return 0; +} diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 20b0571e43..b57cfd6952 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -608,7 +608,7 @@ extern erts_tsd_key_t erts_is_crash_dumping_key; static unsigned long zero_value = 0, one_value = 1; # define SET_BLOCKING(fd) { if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); } # define SET_NONBLOCKING(fd) ioctlsocket((fd), FIONBIO, &one_value) - +# define ERRNO_BLOCK EAGAIN /* We use the posix way for windows */ # else # ifdef NB_FIONBIO /* Old BSD */ # include <sys/ioctl.h> diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 756f103287..064109331f 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -764,1699 +764,6 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, return res; } -/* *\ - * * -\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* make a hash index from an erlang term */ - -/* -** There are two hash functions. -** -** make_hash: A hash function that will give the same values for the same -** terms regardless of the internal representation. Small integers are -** hashed using the same algorithm as bignums and bignums are hashed -** independent of the CPU endianess. -** Make_hash also hashes pids, ports and references like 32 bit numbers -** (but with different constants). -** make_hash() is called from the bif erlang:phash/2 -** -** The idea behind the hash algorithm is to produce values suitable for -** linear dynamic hashing. We cannot choose the range at all while hashing -** (it's not even supplied to the hashing functions). The good old algorithm -** [H = H*C+X mod M, where H is the hash value, C is a "random" constant(or M), -** M is the range, preferably a prime, and X is each byte value] is therefore -** modified to: -** H = H*C+X mod 2^32, where C is a large prime. This gives acceptable -** "spreading" of the hashes, so that later modulo calculations also will give -** acceptable "spreading" in the range. -** We really need to hash on bytes, otherwise the -** upper bytes of a word will be less significant than the lower ones. That's -** not acceptable at all. For internal use one could maybe optimize by using -** another hash function, that is less strict but faster. That is, however, not -** implemented. -** -** Short semi-formal description of make_hash: -** -** In make_hash, the number N is treated like this: -** Abs(N) is hashed bytewise with the least significant byte, B(0), first. -** The number of bytes (J) to calculate hash on in N is -** (the number of _32_ bit words needed to store the unsigned -** value of abs(N)) * 4. -** X = FUNNY_NUMBER2 -** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3. -** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like -** h(0) = <initial hash> -** h(i) = h(i-1)*X + B(i-1) -** The above should hold regardless of internal representation. -** Pids are hashed like small numbers but with different constants, as are -** ports. -** References are hashed like ports but only on the least significant byte. -** Binaries are hashed on all bytes (not on the 15 first as in -** make_broken_hash()). -** Bytes in lists (possibly text strings) use a simpler multiplication inlined -** in the handling of lists, that is an optimization. -** Everything else is like in the old hash (make_broken_hash()). -** -** make_hash2() is faster than make_hash, in particular for bignums -** and binaries, and produces better hash values. -*/ - -/* some prime numbers just above 2 ^ 28 */ - -#define FUNNY_NUMBER1 268440163 -#define FUNNY_NUMBER2 268439161 -#define FUNNY_NUMBER3 268435459 -#define FUNNY_NUMBER4 268436141 -#define FUNNY_NUMBER5 268438633 -#define FUNNY_NUMBER6 268437017 -#define FUNNY_NUMBER7 268438039 -#define FUNNY_NUMBER8 268437511 -#define FUNNY_NUMBER9 268439627 -#define FUNNY_NUMBER10 268440479 -#define FUNNY_NUMBER11 268440577 -#define FUNNY_NUMBER12 268440581 -#define FUNNY_NUMBER13 268440593 -#define FUNNY_NUMBER14 268440611 - -static Uint32 -hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) -{ - byte* ptr; - Uint bitoffs; - Uint bitsize; - - ERTS_GET_BINARY_BYTES(bin, ptr, bitoffs, bitsize); - if (bitoffs == 0) { - while (sz--) { - hash = hash*FUNNY_NUMBER1 + *ptr++; - } - if (bitsize > 0) { - byte b = *ptr; - - b >>= 8 - bitsize; - hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; - } - } else { - Uint previous = *ptr++; - Uint b; - Uint lshift = bitoffs; - Uint rshift = 8 - lshift; - - while (sz--) { - b = (previous << lshift) & 0xFF; - previous = *ptr++; - b |= previous >> rshift; - hash = hash*FUNNY_NUMBER1 + b; - } - if (bitsize > 0) { - b = (previous << lshift) & 0xFF; - previous = *ptr++; - b |= previous >> rshift; - - b >>= 8 - bitsize; - hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; - } - } - return hash; -} - -Uint32 make_hash(Eterm term_arg) -{ - DECLARE_WSTACK(stack); - Eterm term = term_arg; - Eterm hash = 0; - unsigned op; - -#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF) -#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1) -#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2) -#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3) - - /* - ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit - ** integer. - ** If the endianess is known, we could be smarter here, - ** but that gives no significant speedup (on a sparc at least) - */ -#define UINT32_HASH_STEP(Expr, Prime1) \ - do { \ - Uint32 x = (Uint32) (Expr); \ - hash = \ - (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ - ((x >> 8) & 0xFF)) * (Prime1) + \ - ((x >> 16) & 0xFF)) * (Prime1) + \ - (x >> 24)); \ - } while(0) - -#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ - UINT32_HASH_STEP(Expr, Prime1); \ - hash = hash * (Prime2); \ - break - - - /* - * Significant additions needed for real 64 bit port with larger fixnums. - */ - - /* - * Note, for the simple 64bit port, not utilizing the - * larger word size this function will work without modification. - */ -tail_recur: - op = tag_val_def(term); - - for (;;) { - switch (op) { - case NIL_DEF: - hash = hash*FUNNY_NUMBER3 + 1; - break; - case ATOM_DEF: - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(term))->slot.bucket.hvalue); - break; - case SMALL_DEF: - { - Sint y1 = signed_val(term); - Uint y2 = y1 < 0 ? -(Uint)y1 : y1; - - UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#if defined(ARCH_64) - if (y2 >> 32) - UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); -#endif - hash *= (y1 < 0 ? FUNNY_NUMBER4 : FUNNY_NUMBER3); - break; - } - case BINARY_DEF: - { - Uint sz = binary_size(term); - - hash = hash_binary_bytes(term, sz, hash); - hash = hash*FUNNY_NUMBER4 + sz; - break; - } - case FUN_DEF: - { - ErlFunThing* funp = (ErlFunThing *) fun_val(term); - - if (is_local_fun(funp)) { - - ErlFunEntry* fe = funp->entry.fun; - Uint num_free = funp->num_free; - - hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(fe->module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER2 + fe->index; - hash = hash*FUNNY_NUMBER2 + fe->old_uniq; - - if (num_free > 0) { - if (num_free > 1) { - WSTACK_PUSH3(stack, (UWord) &funp->env[1], - (num_free-1), MAKE_HASH_TERM_ARRAY_OP); - } - - term = funp->env[0]; - goto tail_recur; - } - } else { - const ErtsCodeMFA *mfa = &funp->entry.exp->info.mfa; - - ASSERT(is_external_fun(funp) && funp->next == NULL); - - hash = hash * FUNNY_NUMBER11 + mfa->arity; - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(mfa->module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(mfa->function))->slot.bucket.hvalue); - } - break; - } - case PID_DEF: - /* only 15 bits... */ - UINT32_HASH_RET(internal_pid_number(term),FUNNY_NUMBER5,FUNNY_NUMBER6); - case EXTERNAL_PID_DEF: - /* only 15 bits... */ - UINT32_HASH_RET(external_pid_number(term),FUNNY_NUMBER5,FUNNY_NUMBER6); - case PORT_DEF: - case EXTERNAL_PORT_DEF: { - Uint64 number = port_number(term); - Uint32 low = (Uint32) (number & 0xffffffff); - Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); - if (high) - UINT32_HASH_STEP(high, FUNNY_NUMBER11); - UINT32_HASH_RET(low,FUNNY_NUMBER9,FUNNY_NUMBER10); - } - case REF_DEF: - UINT32_HASH_RET(internal_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); - case EXTERNAL_REF_DEF: - UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); - case FLOAT_DEF: - { - FloatDef ff; - GET_DOUBLE(term, ff); - if (ff.fd == 0.0f) { - /* ensure positive 0.0 */ - ff.fd = erts_get_positive_zero_float(); - } - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); - break; - } - case MAKE_HASH_CDR_PRE_OP: - term = (Eterm) WSTACK_POP(stack); - if (is_not_list(term)) { - WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); - goto tail_recur; - } - /* fall through */ - case LIST_DEF: - { - Eterm* list = list_val(term); - while(is_byte(*list)) { - /* Optimization for strings. - ** Note that this hash is different from a 'small' hash, - ** as multiplications on a Sparc is so slow. - */ - hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); - - if (is_not_list(CDR(list))) { - WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); - term = CDR(list); - goto tail_recur; - } - list = list_val(CDR(list)); - } - WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); - term = CAR(list); - goto tail_recur; - } - case MAKE_HASH_CDR_POST_OP: - hash *= FUNNY_NUMBER8; - break; - - case BIG_DEF: - /* Note that this is the exact same thing as the hashing of smalls.*/ - { - Eterm* ptr = big_val(term); - Uint n = BIG_SIZE(ptr); - Uint k = n-1; - ErtsDigit d; - int is_neg = BIG_SIGN(ptr); - Uint i; - int j; - - for (i = 0; i < k; i++) { - d = BIG_DIGIT(ptr, i); - for(j = 0; j < sizeof(ErtsDigit); ++j) { - hash = (hash*FUNNY_NUMBER2) + (d & 0xff); - d >>= 8; - } - } - d = BIG_DIGIT(ptr, k); - k = sizeof(ErtsDigit); -#if defined(ARCH_64) - if (!(d >> 32)) - k /= 2; -#endif - for(j = 0; j < (int)k; ++j) { - hash = (hash*FUNNY_NUMBER2) + (d & 0xff); - d >>= 8; - } - hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3; - break; - } - case MAP_DEF: - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); - break; - case TUPLE_DEF: - { - Eterm* ptr = tuple_val(term); - Uint arity = arityval(*ptr); - - WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); - op = MAKE_HASH_TUPLE_OP; - }/*fall through*/ - case MAKE_HASH_TUPLE_OP: - case MAKE_HASH_TERM_ARRAY_OP: - { - Uint i = (Uint) WSTACK_POP(stack); - Eterm* ptr = (Eterm*) WSTACK_POP(stack); - if (i != 0) { - term = *ptr; - WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); - goto tail_recur; - } - if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = (Uint32) WSTACK_POP(stack); - hash = hash*FUNNY_NUMBER9 + arity; - } - break; - } - - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); - return 0; - } - if (WSTACK_ISEMPTY(stack)) break; - op = WSTACK_POP(stack); - } - DESTROY_WSTACK(stack); - return hash; - -#undef MAKE_HASH_TUPLE_OP -#undef MAKE_HASH_TERM_ARRAY_OP -#undef MAKE_HASH_CDR_PRE_OP -#undef MAKE_HASH_CDR_POST_OP -#undef UINT32_HASH_STEP -#undef UINT32_HASH_RET -} - - - -/* Hash function suggested by Bob Jenkins. */ - -#define MIX(a,b,c) \ -do { \ - a -= b; a -= c; a ^= (c>>13); \ - b -= c; b -= a; b ^= (a<<8); \ - c -= a; c -= b; c ^= (b>>13); \ - a -= b; a -= c; a ^= (c>>12); \ - b -= c; b -= a; b ^= (a<<16); \ - c -= a; c -= b; c ^= (b>>5); \ - a -= b; a -= c; a ^= (c>>3); \ - b -= c; b -= a; b ^= (a<<10); \ - c -= a; c -= b; c ^= (b>>15); \ -} while(0) - -#define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */ - -typedef struct { - Uint32 a,b,c; -} ErtsBlockHashHelperCtx; - -#define BLOCK_HASH_BYTES_PER_ITER 12 - -/* The three functions below are separated into different functions even - though they are always used together to make trapping and handling - of unaligned binaries easier. Examples of how they are used can be - found in block_hash and make_hash2_helper.*/ -static ERTS_INLINE -void block_hash_setup(Uint32 initval, - ErtsBlockHashHelperCtx* ctx /* out parameter */) -{ - ctx->a = ctx->b = HCONST; - ctx->c = initval; /* the previous hash value */ -} - -static ERTS_INLINE -void block_hash_buffer(byte *buf, - Uint buf_length, - ErtsBlockHashHelperCtx* ctx /* out parameter */) -{ - Uint len = buf_length; - byte *k = buf; - ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0); - while (len >= BLOCK_HASH_BYTES_PER_ITER) { - ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); - ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); - ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); - MIX(ctx->a,ctx->b,ctx->c); - k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER; - } -} - -static ERTS_INLINE -Uint32 block_hash_final_bytes(byte *buf, - Uint buf_length, - Uint full_length, - ErtsBlockHashHelperCtx* ctx) -{ - Uint len = buf_length; - byte *k = buf; - ctx->c += full_length; - switch(len) - { /* all the case statements fall through */ - case 11: ctx->c+=((Uint32)k[10]<<24); - case 10: ctx->c+=((Uint32)k[9]<<16); - case 9 : ctx->c+=((Uint32)k[8]<<8); - /* the first byte of c is reserved for the length */ - case 8 : ctx->b+=((Uint32)k[7]<<24); - case 7 : ctx->b+=((Uint32)k[6]<<16); - case 6 : ctx->b+=((Uint32)k[5]<<8); - case 5 : ctx->b+=k[4]; - case 4 : ctx->a+=((Uint32)k[3]<<24); - case 3 : ctx->a+=((Uint32)k[2]<<16); - case 2 : ctx->a+=((Uint32)k[1]<<8); - case 1 : ctx->a+=k[0]; - /* case 0: nothing left to add */ - } - MIX(ctx->a,ctx->b,ctx->c); - return ctx->c; -} - -static -Uint32 -block_hash(byte *block, Uint block_length, Uint32 initval) -{ - ErtsBlockHashHelperCtx ctx; - Uint no_bytes_not_in_loop = - (block_length % BLOCK_HASH_BYTES_PER_ITER); - Uint no_bytes_to_process_in_loop = - block_length - no_bytes_not_in_loop; - byte *final_bytes = block + no_bytes_to_process_in_loop; - block_hash_setup(initval, &ctx); - block_hash_buffer(block, - no_bytes_to_process_in_loop, - &ctx); - return block_hash_final_bytes(final_bytes, - no_bytes_not_in_loop, - block_length, - &ctx); -} - -typedef enum { - tag_primary_list, - arityval_subtag, - hamt_subtag_head_flatmap, - map_subtag, - fun_subtag, - neg_big_subtag, - sub_binary_subtag_1, - sub_binary_subtag_2, - hash2_common_1, - hash2_common_2, - hash2_common_3, -} ErtsMakeHash2TrapLocation; - -typedef struct { - int c; - Uint32 sh; - Eterm* ptr; -} ErtsMakeHash2Context_TAG_PRIMARY_LIST; - -typedef struct { - int i; - int arity; - Eterm* elem; -} ErtsMakeHash2Context_ARITYVAL_SUBTAG; - -typedef struct { - Eterm *ks; - Eterm *vs; - int i; - Uint size; -} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP; - -typedef struct { - Eterm* ptr; - int i; -} ErtsMakeHash2Context_MAP_SUBTAG; - -typedef struct { - Uint num_free; - Eterm* bptr; -} ErtsMakeHash2Context_FUN_SUBTAG; - -typedef struct { - Eterm* ptr; - Uint i; - Uint n; - Uint32 con; -} ErtsMakeHash2Context_NEG_BIG_SUBTAG; - -typedef struct { - byte* bptr; - Uint sz; - Uint bitsize; - Uint bitoffs; - Uint no_bytes_processed; - ErtsBlockHashHelperCtx block_hash_ctx; - /* The following fields are only used when bitoffs != 0 */ - byte* buf; - int done; - -} ErtsMakeHash2Context_SUB_BINARY_SUBTAG; - -typedef struct { - int dummy__; /* Empty structs are not supported on all platforms */ -} ErtsMakeHash2Context_EMPTY; - -typedef struct { - ErtsMakeHash2TrapLocation trap_location; - /* specific to the trap location: */ - union { - ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list; - ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag; - ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap; - ErtsMakeHash2Context_MAP_SUBTAG map_subtag; - ErtsMakeHash2Context_FUN_SUBTAG fun_subtag; - ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag; - ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1; - ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2; - ErtsMakeHash2Context_EMPTY hash2_common_1; - ErtsMakeHash2Context_EMPTY hash2_common_2; - ErtsMakeHash2Context_EMPTY hash2_common_3; - } trap_location_state; - /* same for all trap locations: */ - Eterm term; - Uint32 hash; - Uint32 hash_xor_pairs; - ErtsEStack stack; -} ErtsMakeHash2Context; - -static int make_hash2_ctx_bin_dtor(Binary *context_bin) { - ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin); - DESTROY_SAVED_ESTACK(&context->stack); - if (context->trap_location == sub_binary_subtag_2 && - context->trap_location_state.sub_binary_subtag_2.buf != NULL) { - erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf); - } - return 1; -} - -/* hash2_save_trap_state is called seldom so we want to avoid inlining */ -static ERTS_NOINLINE -Eterm hash2_save_trap_state(Eterm state_mref, - Uint32 hash_xor_pairs, - Uint32 hash, - Process* p, - Eterm term, - Eterm* ESTK_DEF_STACK(s), - ErtsEStack s, - ErtsMakeHash2TrapLocation trap_location, - void* trap_location_state_ptr, - size_t trap_location_state_size) { - Binary* state_bin; - ErtsMakeHash2Context* context; - if (state_mref == THE_NON_VALUE) { - Eterm* hp; - state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context), - make_hash2_ctx_bin_dtor); - hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin); - } else { - state_bin = erts_magic_ref2bin(state_mref); - } - context = ERTS_MAGIC_BIN_DATA(state_bin); - context->term = term; - context->hash = hash; - context->hash_xor_pairs = hash_xor_pairs; - ESTACK_SAVE(s, &context->stack); - context->trap_location = trap_location; - sys_memcpy(&context->trap_location_state, - trap_location_state_ptr, - trap_location_state_size); - erts_set_gc_state(p, 0); - BUMP_ALL_REDS(p); - return state_mref; -} -#undef NOINLINE_HASH2_SAVE_TRAP_STATE - -/* Writes back a magic reference to *state_mref_write_back when the - function traps */ -static ERTS_INLINE Uint32 -make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p) -{ - static const Uint ITERATIONS_PER_RED = 64; - Uint32 hash; - Uint32 hash_xor_pairs; - Eterm term = term_param; - ERTS_UNDEF(hash_xor_pairs, 0); - -/* (HCONST * {2, ..., 22}) mod 2^32 */ -#define HCONST_2 0x3c6ef372UL -#define HCONST_3 0xdaa66d2bUL -#define HCONST_4 0x78dde6e4UL -#define HCONST_5 0x1715609dUL -#define HCONST_6 0xb54cda56UL -#define HCONST_7 0x5384540fUL -#define HCONST_8 0xf1bbcdc8UL -#define HCONST_9 0x8ff34781UL -#define HCONST_10 0x2e2ac13aUL -#define HCONST_11 0xcc623af3UL -#define HCONST_12 0x6a99b4acUL -#define HCONST_13 0x08d12e65UL -#define HCONST_14 0xa708a81eUL -#define HCONST_15 0x454021d7UL -#define HCONST_16 0xe3779b90UL -#define HCONST_17 0x81af1549UL -#define HCONST_18 0x1fe68f02UL -#define HCONST_19 0xbe1e08bbUL -#define HCONST_20 0x5c558274UL -#define HCONST_21 0xfa8cfc2dUL -#define HCONST_22 0x98c475e6UL - -#define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) -#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) -#define HASH_CDR (_make_header(3,_TAG_HEADER_REF)) - -#define UINT32_HASH_2(Expr1, Expr2, AConst) \ - do { \ - Uint32 a,b; \ - a = AConst + (Uint32) (Expr1); \ - b = AConst + (Uint32) (Expr2); \ - MIX(a,b,hash); \ - } while(0) - -#define UINT32_HASH(Expr, AConst) UINT32_HASH_2(Expr, 0, AConst) - -#define SINT32_HASH(Expr, AConst) \ - do { \ - Sint32 y = (Sint32) (Expr); \ - if (y < 0) { \ - UINT32_HASH(-y, AConst); \ - /* Negative numbers are unnecessarily mixed twice. */ \ - } \ - UINT32_HASH(y, AConst); \ - } while(0) - -#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) - -#define NOT_SSMALL28_HASH(SMALL) \ - do { \ - Uint64 t; \ - Uint32 x, y; \ - Uint32 con; \ - if (SMALL < 0) { \ - con = HCONST_10; \ - t = (Uint64)(SMALL * (-1)); \ - } else { \ - con = HCONST_11; \ - t = SMALL; \ - } \ - x = t & 0xffffffff; \ - y = t >> 32; \ - UINT32_HASH_2(x, y, con); \ - } while(0) - -#ifdef ARCH_64 -# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) -#else -# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) -#endif - -#define TRAP_LOCATION_NO_RED(location_name) \ - do { \ - if(can_trap && iterations_until_trap <= 0) { \ - *state_mref_write_back = \ - hash2_save_trap_state(state_mref, \ - hash_xor_pairs, \ - hash, \ - p, \ - term, \ - ESTK_DEF_STACK(s), \ - s, \ - location_name, \ - &ctx, \ - sizeof(ctx)); \ - return 0; \ - L_##location_name: \ - ctx = context->trap_location_state. location_name; \ - } \ - } while(0) - -#define TRAP_LOCATION(location_name) \ - do { \ - if (can_trap) { \ - iterations_until_trap--; \ - TRAP_LOCATION_NO_RED(location_name); \ - } \ - } while(0) - -#define TRAP_LOCATION_NO_CTX(location_name) \ - do { \ - ErtsMakeHash2Context_EMPTY ctx; \ - TRAP_LOCATION(location_name); \ - } while(0) - - /* Optimization. Simple cases before declaration of estack. */ - if (primary_tag(term) == TAG_PRIMARY_IMMED1) { - switch (term & _TAG_IMMED1_MASK) { - case _TAG_IMMED1_IMMED2: - switch (term & _TAG_IMMED2_MASK) { - case _TAG_IMMED2_ATOM: - /* Fast, but the poor hash value should be mixed. */ - return atom_tab(atom_val(term))->slot.bucket.hvalue; - } - break; - case _TAG_IMMED1_SMALL: - { - Sint small = signed_val(term); - if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { - hash = 0; - NOT_SSMALL28_HASH(small); - return hash; - } - hash = 0; - SINT32_HASH(small, HCONST); - return hash; - } - } - }; - { - Eterm tmp; - long max_iterations = 0; - long iterations_until_trap = 0; - Eterm state_mref = THE_NON_VALUE; - ErtsMakeHash2Context* context = NULL; - DECLARE_ESTACK(s); - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); - if(can_trap){ -#ifdef DEBUG - (void)ITERATIONS_PER_RED; - iterations_until_trap = max_iterations = - (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227; -#else - iterations_until_trap = max_iterations = - ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); -#endif - } - if (can_trap && is_internal_magic_ref(term)) { - Binary* state_bin; - state_mref = term; - state_bin = erts_magic_ref2bin(state_mref); - if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) { - /* Restore state after a trap */ - context = ERTS_MAGIC_BIN_DATA(state_bin); - term = context->term; - hash = context->hash; - hash_xor_pairs = context->hash_xor_pairs; - ESTACK_RESTORE(s, &context->stack); - ASSERT(p->flags & F_DISABLE_GC); - erts_set_gc_state(p, 1); - switch (context->trap_location) { - case hash2_common_3: goto L_hash2_common_3; - case tag_primary_list: goto L_tag_primary_list; - case arityval_subtag: goto L_arityval_subtag; - case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap; - case map_subtag: goto L_map_subtag; - case fun_subtag: goto L_fun_subtag; - case neg_big_subtag: goto L_neg_big_subtag; - case sub_binary_subtag_1: goto L_sub_binary_subtag_1; - case sub_binary_subtag_2: goto L_sub_binary_subtag_2; - case hash2_common_1: goto L_hash2_common_1; - case hash2_common_2: goto L_hash2_common_2; - } - } - } - hash = 0; - for (;;) { - switch (primary_tag(term)) { - case TAG_PRIMARY_LIST: - { - ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = { - .c = 0, - .sh = 0, - .ptr = list_val(term)}; - while (is_byte(*ctx.ptr)) { - /* Optimization for strings. */ - ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr); - if (ctx.c == 3) { - UINT32_HASH(ctx.sh, HCONST_4); - ctx.c = ctx.sh = 0; - } else { - ctx.c++; - } - term = CDR(ctx.ptr); - if (is_not_list(term)) - break; - ctx.ptr = list_val(term); - TRAP_LOCATION(tag_primary_list); - } - if (ctx.c > 0) - UINT32_HASH(ctx.sh, HCONST_4); - if (is_list(term)) { - tmp = CDR(ctx.ptr); - ESTACK_PUSH(s, tmp); - term = CAR(ctx.ptr); - } - } - break; - case TAG_PRIMARY_BOXED: - { - Eterm hdr = *boxed_val(term); - ASSERT(is_header(hdr)); - switch (hdr & _TAG_HEADER_MASK) { - case ARITYVAL_SUBTAG: - { - ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = { - .i = 0, - .arity = header_arity(hdr), - .elem = tuple_val(term)}; - UINT32_HASH(ctx.arity, HCONST_9); - if (ctx.arity == 0) /* Empty tuple */ - goto hash2_common; - for (ctx.i = ctx.arity; ; ctx.i--) { - term = ctx.elem[ctx.i]; - if (ctx.i == 1) - break; - ESTACK_PUSH(s, term); - TRAP_LOCATION(arityval_subtag); - } - } - break; - case MAP_SUBTAG: - { - Uint size; - ErtsMakeHash2Context_MAP_SUBTAG ctx = { - .ptr = boxed_val(term) + 1, - .i = 0}; - switch (hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_FLATMAP: - { - flatmap_t *mp = (flatmap_t *)flatmap_val(term); - ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = { - .ks = flatmap_get_keys(mp), - .vs = flatmap_get_values(mp), - .i = 0, - .size = flatmap_get_size(mp)}; - UINT32_HASH(ctx.size, HCONST_16); - if (ctx.size == 0) - goto hash2_common; - - /* We want a portable hash function that is *independent* of - * the order in which keys and values are encountered. - * We therefore calculate context independent hashes for all . - * key-value pairs and then xor them together. - */ - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; - hash_xor_pairs = 0; - for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) { - ESTACK_PUSH(s, HASH_MAP_PAIR); - ESTACK_PUSH(s, ctx.vs[ctx.i]); - ESTACK_PUSH(s, ctx.ks[ctx.i]); - TRAP_LOCATION(hamt_subtag_head_flatmap); - } - goto hash2_common; - } - - case HAMT_SUBTAG_HEAD_ARRAY: - case HAMT_SUBTAG_HEAD_BITMAP: - size = *ctx.ptr++; - UINT32_HASH(size, HCONST_16); - if (size == 0) - goto hash2_common; - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; - hash_xor_pairs = 0; - } - switch (hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - ctx.i = 16; - break; - case HAMT_SUBTAG_HEAD_BITMAP: - case HAMT_SUBTAG_NODE_BITMAP: - ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); - break; - default: - erts_exit(ERTS_ERROR_EXIT, "bad header"); - } - while (ctx.i) { - if (is_list(*ctx.ptr)) { - Eterm* cons = list_val(*ctx.ptr); - ESTACK_PUSH(s, HASH_MAP_PAIR); - ESTACK_PUSH(s, CDR(cons)); - ESTACK_PUSH(s, CAR(cons)); - } - else { - ASSERT(is_boxed(*ctx.ptr)); - ESTACK_PUSH(s, *ctx.ptr); - } - ctx.i--; ctx.ptr++; - TRAP_LOCATION(map_subtag); - } - goto hash2_common; - } - break; - - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) fun_val(term); - - if (is_local_fun(funp)) { - ErlFunEntry* fe = funp->entry.fun; - ErtsMakeHash2Context_FUN_SUBTAG ctx = { - .num_free = funp->num_free, - .bptr = NULL}; - - UINT32_HASH_2 - (ctx.num_free, - atom_tab(atom_val(fe->module))->slot.bucket.hvalue, - HCONST); - UINT32_HASH_2 - (fe->index, fe->old_uniq, HCONST); - if (ctx.num_free == 0) { - goto hash2_common; - } else { - ctx.bptr = funp->env + ctx.num_free - 1; - while (ctx.num_free-- > 1) { - term = *ctx.bptr--; - ESTACK_PUSH(s, term); - TRAP_LOCATION(fun_subtag); - } - term = *ctx.bptr; - } - } else { - Export *ep = funp->entry.exp; - - ASSERT(is_external_fun(funp) && funp->next == NULL); - - UINT32_HASH_2 - (ep->info.mfa.arity, - atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, - HCONST); - UINT32_HASH - (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, - HCONST_14); - - goto hash2_common; - } - } - break; - case REFC_BINARY_SUBTAG: - case HEAP_BINARY_SUBTAG: - case SUB_BINARY_SUBTAG: - { -#define BYTE_BITS 8 - ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = { - .bptr = 0, - /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! - * - * The size is truncated to 32 bits on the line - * below so that the code is compatible with old - * versions of the code. This means that hash - * values for binaries with a size greater than - * 4GB do not take all bytes in consideration. - * - * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! - */ - .sz = (0xFFFFFFFF & binary_size(term)), - .bitsize = 0, - .bitoffs = 0, - .no_bytes_processed = 0 - }; - Uint32 con = HCONST_13 + hash; - Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER); - ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize); - if (ctx.sz == 0 && ctx.bitsize == 0) { - hash = con; - } else if (ctx.bitoffs == 0 && - (!can_trap || - (iterations_until_trap - iters_for_bin) > 0)) { - /* No need to trap while hashing binary */ - if (can_trap) iterations_until_trap -= iters_for_bin; - hash = block_hash(ctx.bptr, ctx.sz, con); - if (ctx.bitsize > 0) { - UINT32_HASH_2(ctx.bitsize, - (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), - HCONST_15); - } - } else if (ctx.bitoffs == 0) { - /* Need to trap while hashing binary */ - ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; - block_hash_setup(con, block_hash_ctx); - do { - Uint max_bytes_to_process = - iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER : - iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER; - Uint bytes_left = ctx.sz - ctx.no_bytes_processed; - Uint even_bytes_left = - bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); - Uint bytes_to_process = - MIN(max_bytes_to_process, even_bytes_left); - block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed], - bytes_to_process, - block_hash_ctx); - ctx.no_bytes_processed += bytes_to_process; - iterations_until_trap -= - MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); - TRAP_LOCATION_NO_RED(sub_binary_subtag_1); - block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ - } while ((ctx.sz - ctx.no_bytes_processed) >= - BLOCK_HASH_BYTES_PER_ITER); - hash = block_hash_final_bytes(ctx.bptr + - ctx.no_bytes_processed, - ctx.sz - ctx.no_bytes_processed, - ctx.sz, - block_hash_ctx); - if (ctx.bitsize > 0) { - UINT32_HASH_2(ctx.bitsize, - (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), - HCONST_15); - } - } else if (/* ctx.bitoffs != 0 && */ - (!can_trap || - (iterations_until_trap - iters_for_bin) > 0)) { - /* No need to trap while hashing binary */ - Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); - byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes); - Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize; - if (can_trap) iterations_until_trap -= iters_for_bin; - erts_copy_bits(ctx.bptr, - ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy); - hash = block_hash(buf, ctx.sz, con); - if (ctx.bitsize > 0) { - UINT32_HASH_2(ctx.bitsize, - (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), - HCONST_15); - } - erts_free(ERTS_ALC_T_TMP, buf); - } else /* ctx.bitoffs != 0 && */ { -#ifdef DEBUG -#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3) -#else -#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256) -#endif -#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS) - /* Need to trap while hashing binary */ - ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; - Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); - ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0); - ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP, - MIN(nr_of_bytes, BINARY_BUF_SIZE)); - block_hash_setup(con, block_hash_ctx); - do { - Uint bytes_left = - ctx.sz - ctx.no_bytes_processed; - Uint even_bytes_left = - bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); - Uint bytes_to_process = - MIN(BINARY_BUF_SIZE, even_bytes_left); - Uint nr_of_bits_left = - (ctx.sz*BYTE_BITS+ctx.bitsize) - - ctx.no_bytes_processed*BYTE_BITS; - Uint nr_of_bits_to_copy = - MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS); - ctx.done = nr_of_bits_left == nr_of_bits_to_copy; - erts_copy_bits(ctx.bptr + ctx.no_bytes_processed, - ctx.bitoffs, 1, ctx.buf, 0, 1, - nr_of_bits_to_copy); - block_hash_buffer(ctx.buf, - bytes_to_process, - block_hash_ctx); - ctx.no_bytes_processed += bytes_to_process; - iterations_until_trap -= - MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); - TRAP_LOCATION_NO_RED(sub_binary_subtag_2); - block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ - } while (!ctx.done); - nr_of_bytes = ctx.sz + (ctx.bitsize != 0); - hash = block_hash_final_bytes(ctx.buf + - (ctx.no_bytes_processed - - ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE), - ctx.sz - ctx.no_bytes_processed, - ctx.sz, - block_hash_ctx); - if (ctx.bitsize > 0) { - Uint last_byte_index = - nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1; - UINT32_HASH_2(ctx.bitsize, - (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)), - HCONST_15); - } - erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf); - context->trap_location_state.sub_binary_subtag_2.buf = NULL; - } - goto hash2_common; -#undef BYTE_BITS -#undef BINARY_BUF_SIZE -#undef BINARY_BUF_SIZE_BITS - } - break; - case POS_BIG_SUBTAG: - case NEG_BIG_SUBTAG: - { - Eterm* big_val_ptr = big_val(term); - ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = { - .ptr = big_val_ptr, - .i = 0, - .n = BIG_SIZE(big_val_ptr), - .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11}; -#if D_EXP == 16 - do { - Uint32 x, y; - x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; - x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; - y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; - y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; - UINT32_HASH_2(x, y, ctx.con); - TRAP_LOCATION(neg_big_subtag); - } while (ctx.i < ctx.n); -#elif D_EXP == 32 - do { - Uint32 x, y; - x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; - y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; - UINT32_HASH_2(x, y, ctx.con); - TRAP_LOCATION(neg_big_subtag); - } while (ctx.i < ctx.n); -#elif D_EXP == 64 - do { - Uint t; - Uint32 x, y; - ASSERT(ctx.i < ctx.n); - t = BIG_DIGIT(ctx.ptr, ctx.i++); - x = t & 0xffffffff; - y = t >> 32; - UINT32_HASH_2(x, y, ctx.con); - TRAP_LOCATION(neg_big_subtag); - } while (ctx.i < ctx.n); -#else -#error "unsupported D_EXP size" -#endif - goto hash2_common; - } - break; - case REF_SUBTAG: - /* All parts of the ref should be hashed. */ - UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); - goto hash2_common; - break; - case EXTERNAL_REF_SUBTAG: - /* All parts of the ref should be hashed. */ - UINT32_HASH(external_ref_numbers(term)[0], HCONST_7); - goto hash2_common; - break; - case EXTERNAL_PID_SUBTAG: - /* Only 15 bits are hashed. */ - UINT32_HASH(external_pid_number(term), HCONST_5); - goto hash2_common; - case EXTERNAL_PORT_SUBTAG: { - Uint64 number = external_port_number(term); - Uint32 low = (Uint32) (number & 0xffffffff); - Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); - UINT32_HASH_2(low, high, HCONST_6); - goto hash2_common; - } - case FLOAT_SUBTAG: - { - FloatDef ff; - GET_DOUBLE(term, ff); - if (ff.fd == 0.0f) { - /* ensure positive 0.0 */ - ff.fd = erts_get_positive_zero_float(); - } -#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) - UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); -#else - UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); -#endif - goto hash2_common; - } - break; - - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); - } - } - break; - case TAG_PRIMARY_IMMED1: - switch (term & _TAG_IMMED1_MASK) { - case _TAG_IMMED1_PID: - /* Only 15 bits are hashed. */ - UINT32_HASH(internal_pid_number(term), HCONST_5); - goto hash2_common; - case _TAG_IMMED1_PORT: { - Uint64 number = internal_port_number(term); - Uint32 low = (Uint32) (number & 0xffffffff); - Uint32 high = (Uint32) ((number >> 32) & 0xffffffff); - UINT32_HASH_2(low, high, HCONST_6); - goto hash2_common; - } - case _TAG_IMMED1_IMMED2: - switch (term & _TAG_IMMED2_MASK) { - case _TAG_IMMED2_ATOM: - if (hash == 0) - /* Fast, but the poor hash value should be mixed. */ - hash = atom_tab(atom_val(term))->slot.bucket.hvalue; - else - UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue, - HCONST_3); - goto hash2_common; - case _TAG_IMMED2_NIL: - if (hash == 0) - hash = 3468870702UL; - else - UINT32_HASH(NIL_DEF, HCONST_2); - goto hash2_common; - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); - } - case _TAG_IMMED1_SMALL: - { - Sint small = signed_val(term); - if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { - NOT_SSMALL28_HASH(small); - } else { - SINT32_HASH(small, HCONST); - } - - goto hash2_common; - } - } - break; - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); - hash2_common: - - /* Uint32 hash always has the hash value of the previous term, - * compounded or otherwise. - */ - - if (ESTACK_ISEMPTY(s)) { - DESTROY_ESTACK(s); - if (can_trap) { - BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); - ASSERT(!(p->flags & F_DISABLE_GC)); - } - return hash; - } - - term = ESTACK_POP(s); - - switch (term) { - case HASH_MAP_TAIL: { - hash = (Uint32) ESTACK_POP(s); - UINT32_HASH(hash_xor_pairs, HCONST_19); - hash_xor_pairs = (Uint32) ESTACK_POP(s); - TRAP_LOCATION_NO_CTX(hash2_common_1); - goto hash2_common; - } - case HASH_MAP_PAIR: - hash_xor_pairs ^= hash; - hash = 0; - TRAP_LOCATION_NO_CTX(hash2_common_2); - goto hash2_common; - default: - break; - } - - } - TRAP_LOCATION_NO_CTX(hash2_common_3); - } - } -#undef TRAP_LOCATION_NO_RED -#undef TRAP_LOCATION -#undef TRAP_LOCATION_NO_CTX -} - -Uint32 -make_hash2(Eterm term) -{ - return make_hash2_helper(term, 0, NULL, NULL); -} - -Uint32 -trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p) -{ - return make_hash2_helper(term, 1, state_mref_write_back, p); -} - -/* Term hash function for maps, with a separate depth parameter */ -Uint32 make_map_hash(Eterm key, int depth) { - Uint32 hash = 0; - - if (depth > 0) { - UINT32_HASH_2(depth, 1, HCONST_22); - } - - return make_internal_hash(key, hash); -} - -/* Term hash function for internal use. - * - * Limitation #1: Is not "portable" in any way between different VM instances. - * - * Limitation #2: The hash value is only valid as long as the term exists - * somewhere in the VM. Why? Because external pids, ports and refs are hashed - * by mixing the node *pointer* value. If a node disappears and later reappears - * with a new ErlNode struct, externals from that node will hash different than - * before. - * - * One IMPORTANT property must hold (for hamt). - * EVERY BIT of the term that is significant for equality (see EQ) - * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a - * chance of hashing different when salted. - * - * This is why we cannot use cached hash values for atoms for example. - * - */ - -#define CONST_HASH(AConst) \ -do { /* Lightweight mixing of constant (type info) */ \ - hash ^= AConst; \ - hash = (hash << 17) ^ (hash >> (32-17)); \ -} while (0) - -/* - * Start with salt, 32-bit prime number, to avoid getting same hash as phash2 - * which can cause bad hashing in distributed ETS tables for example. - */ -#define INTERNAL_HASH_SALT 3432918353U - -Uint32 -make_internal_hash(Eterm term, Uint32 salt) -{ - Uint32 hash = salt ^ INTERNAL_HASH_SALT; - - /* Optimization. Simple cases before declaration of estack. */ - if (primary_tag(term) == TAG_PRIMARY_IMMED1) { - #if ERTS_SIZEOF_ETERM == 8 - UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); - #elif ERTS_SIZEOF_ETERM == 4 - UINT32_HASH(term, HCONST); - #else - # error "No you don't" - #endif - return hash; - } - { - Eterm tmp; - DECLARE_ESTACK(s); - - for (;;) { - switch (primary_tag(term)) { - case TAG_PRIMARY_LIST: - { - int c = 0; - Uint32 sh = 0; - Eterm* ptr = list_val(term); - while (is_byte(*ptr)) { - /* Optimization for strings. */ - sh = (sh << 8) + unsigned_val(*ptr); - if (c == 3) { - UINT32_HASH(sh, HCONST_4); - c = sh = 0; - } else { - c++; - } - term = CDR(ptr); - if (is_not_list(term)) - break; - ptr = list_val(term); - } - if (c > 0) - UINT32_HASH_2(sh, (Uint32)c, HCONST_22); - - if (is_list(term)) { - tmp = CDR(ptr); - CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ - ESTACK_PUSH(s, tmp); - if (is_not_list(tmp)) { - ESTACK_PUSH(s, HASH_CDR); - } - term = CAR(ptr); - } - } - break; - case TAG_PRIMARY_BOXED: - { - Eterm hdr = *boxed_val(term); - ASSERT(is_header(hdr)); - switch (hdr & _TAG_HEADER_MASK) { - case ARITYVAL_SUBTAG: - { - int i; - int arity = header_arity(hdr); - Eterm* elem = tuple_val(term); - UINT32_HASH(arity, HCONST_9); - if (arity == 0) /* Empty tuple */ - goto pop_next; - for (i = arity; ; i--) { - term = elem[i]; - if (i == 1) - break; - ESTACK_PUSH(s, term); - } - } - break; - - case MAP_SUBTAG: - { - Eterm* ptr = boxed_val(term) + 1; - Uint size; - int i; - - /* - * We rely on key-value iteration order being constant - * for identical maps (in this VM instance). - */ - switch (hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_FLATMAP: - { - flatmap_t *mp = (flatmap_t *)flatmap_val(term); - Eterm *ks = flatmap_get_keys(mp); - Eterm *vs = flatmap_get_values(mp); - size = flatmap_get_size(mp); - UINT32_HASH(size, HCONST_16); - if (size == 0) - goto pop_next; - - for (i = size - 1; i >= 0; i--) { - ESTACK_PUSH(s, vs[i]); - ESTACK_PUSH(s, ks[i]); - } - goto pop_next; - } - case HAMT_SUBTAG_HEAD_ARRAY: - case HAMT_SUBTAG_HEAD_BITMAP: - size = *ptr++; - UINT32_HASH(size, HCONST_16); - if (size == 0) - goto pop_next; - } - switch (hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - i = 16; - break; - case HAMT_SUBTAG_HEAD_BITMAP: - case HAMT_SUBTAG_NODE_BITMAP: - i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); - break; - default: - erts_exit(ERTS_ERROR_EXIT, "bad header"); - } - while (i) { - if (is_list(*ptr)) { - Eterm* cons = list_val(*ptr); - ESTACK_PUSH(s, CDR(cons)); - ESTACK_PUSH(s, CAR(cons)); - } - else { - ASSERT(is_boxed(*ptr)); - ESTACK_PUSH(s, *ptr); - } - i--; ptr++; - } - goto pop_next; - } - break; - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) fun_val(term); - - if (is_local_fun(funp)) { - ErlFunEntry* fe = funp->entry.fun; - Uint num_free = funp->num_free; - UINT32_HASH_2(num_free, fe->module, HCONST_20); - UINT32_HASH_2(fe->index, fe->old_uniq, HCONST_21); - if (num_free == 0) { - goto pop_next; - } else { - Eterm* bptr = funp->env + num_free - 1; - while (num_free-- > 1) { - term = *bptr--; - ESTACK_PUSH(s, term); - } - term = *bptr; - } - } else { - ASSERT(is_external_fun(funp) && funp->next == NULL); - - /* Assumes Export entries never move */ - POINTER_HASH(funp->entry.exp, HCONST_14); - goto pop_next; - } - } - break; - case REFC_BINARY_SUBTAG: - case HEAP_BINARY_SUBTAG: - case SUB_BINARY_SUBTAG: - { - byte* bptr; - Uint sz = binary_size(term); - Uint32 con = HCONST_13 + hash; - Uint bitoffs; - Uint bitsize; - - ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); - if (sz == 0 && bitsize == 0) { - hash = con; - } else { - if (bitoffs == 0) { - hash = block_hash(bptr, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), - HCONST_15); - } - } else { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, - sz + (bitsize != 0)); - erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); - hash = block_hash(buf, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), - HCONST_15); - } - erts_free(ERTS_ALC_T_TMP, (void *) buf); - } - } - goto pop_next; - } - break; - case POS_BIG_SUBTAG: - case NEG_BIG_SUBTAG: - { - Eterm* ptr = big_val(term); - Uint i = 0; - Uint n = BIG_SIZE(ptr); - Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; -#if D_EXP == 16 - do { - Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - UINT32_HASH_2(x, y, con); - } while (i < n); -#elif D_EXP == 32 - do { - Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - UINT32_HASH_2(x, y, con); - } while (i < n); -#elif D_EXP == 64 - do { - Uint t; - Uint32 x, y; - ASSERT(i < n); - t = BIG_DIGIT(ptr, i++); - x = t & 0xffffffff; - y = t >> 32; - UINT32_HASH_2(x, y, con); - } while (i < n); -#else -#error "unsupported D_EXP size" -#endif - goto pop_next; - } - break; - case REF_SUBTAG: - UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); - ASSERT(internal_ref_no_numbers(term) == 3); - UINT32_HASH_2(internal_ref_numbers(term)[1], - internal_ref_numbers(term)[2], HCONST_8); - goto pop_next; - - case EXTERNAL_REF_SUBTAG: - { - ExternalThing* thing = external_thing_ptr(term); - - ASSERT(external_thing_ref_no_numbers(thing) == 3); - /* See limitation #2 */ - #ifdef ARCH_64 - POINTER_HASH(thing->node, HCONST_7); - UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7); - #else - UINT32_HASH_2(thing->node, - external_thing_ref_numbers(thing)[0], HCONST_7); - #endif - UINT32_HASH_2(external_thing_ref_numbers(thing)[1], - external_thing_ref_numbers(thing)[2], HCONST_8); - goto pop_next; - } - case EXTERNAL_PID_SUBTAG: { - ExternalThing* thing = external_thing_ptr(term); - /* See limitation #2 */ - POINTER_HASH(thing->node, HCONST_5); - UINT32_HASH_2(thing->data.pid.num, thing->data.pid.ser, HCONST_5); - goto pop_next; - } - case EXTERNAL_PORT_SUBTAG: { - ExternalThing* thing = external_thing_ptr(term); - /* See limitation #2 */ - POINTER_HASH(thing->node, HCONST_6); - UINT32_HASH_2(thing->data.ui32[0], thing->data.ui32[1], HCONST_6); - goto pop_next; - } - case FLOAT_SUBTAG: - { - FloatDef ff; - GET_DOUBLE(term, ff); - if (ff.fd == 0.0f) { - /* ensure positive 0.0 */ - ff.fd = erts_get_positive_zero_float(); - } - UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); - goto pop_next; - } - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); - } - } - break; - case TAG_PRIMARY_IMMED1: - #if ERTS_SIZEOF_ETERM == 8 - UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); - #else - UINT32_HASH(term, HCONST); - #endif - goto pop_next; - - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); - - pop_next: - if (ESTACK_ISEMPTY(s)) { - DESTROY_ESTACK(s); - - return hash; - } - - term = ESTACK_POP(s); - - switch (term) { - case HASH_CDR: - CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ - goto pop_next; - default: - break; - } - } - } - } - -#undef CONST_HASH -#undef HASH_MAP_TAIL -#undef HASH_MAP_PAIR -#undef HASH_CDR - -#undef UINT32_HASH_2 -#undef UINT32_HASH -#undef SINT32_HASH -} - -#undef HCONST -#undef MIX - /* error_logger ! {log, Level, format, [args], #{ gl, pid, time, error_logger => #{tag, emulator => true} }} */ diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c deleted file mode 100644 index 08ab714153..0000000000 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ /dev/null @@ -1,1606 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2022. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Tty driver that reads one character at the time and provides a - * smart line for output. - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "erl_driver.h" - -static int ttysl_init(void); -static ErlDrvData ttysl_start(ErlDrvPort, char*); - -#ifdef HAVE_TERMCAP /* else make an empty driver that cannot be opened */ - -#ifndef WANT_NONBLOCKING -#define WANT_NONBLOCKING -#endif - -#include "sys.h" -#include <ctype.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <signal.h> -#include <fcntl.h> -#include <limits.h> -#include <locale.h> -#include <unistd.h> -#include <termios.h> -#ifdef HAVE_WCWIDTH -#include <wchar.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_IOCTL_H -#include <sys/ioctl.h> -#endif -#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H) -#define PRIMITIVE_UTF8_CHECK 1 -#else -#include <langinfo.h> -#endif - -#if defined IOV_MAX -#define MAXIOV IOV_MAX -#elif defined UIO_MAXIOV -#define MAXIOV UIO_MAXIOV -#else -#define MAXIOV 16 -#endif - -#define TRUE 1 -#define FALSE 0 - -/* Termcap functions. */ -int tgetent(char* bp, char *name); -int tgetnum(char* cap); -int tgetflag(char* cap); -char *tgetstr(char* cap, char** buf); -char *tgoto(char* cm, int col, int line); -int tputs(char* cp, int affcnt, int (*outc)(int c)); - -/* Terminal capabilities in which we are interested. */ -static char *capbuf; -static char *up, *down, *left, *right; -static int cols, xn; -static volatile int cols_needs_update = FALSE; - -/* The various opcodes. */ -#define OP_PUTC 0 -#define OP_MOVE 1 -#define OP_INSC 2 -#define OP_DELC 3 -#define OP_BEEP 4 -#define OP_PUTC_SYNC 5 -/* Control op */ -#define CTRL_OP_GET_WINSIZE 100 -#define CTRL_OP_GET_UNICODE_STATE 101 -#define CTRL_OP_SET_UNICODE_STATE 102 - -/* We use 1024 as the buf size as that was the default buf size of FILE streams - on all platforms that I checked. */ -#define TTY_BUFFSIZE 1024 - -static int lbuf_size = BUFSIZ; -static Uint32 *lbuf; /* The current line buffer */ -static int llen; /* The current line length */ -static int lpos; /* The current "cursor position" in the line buffer */ - /* NOTE: not the same as column position a char may not take a" - * column to display or it might take many columns - */ -/* - * Tags used in line buffer to show that these bytes represent special characters, - * Max unicode is 0x0010ffff, so we have lots of place for meta tags... - */ -#define CONTROL_TAG 0x10000000U /* Control character, value in first position */ -#define ESCAPED_TAG 0x01000000U /* Escaped character, value in first position */ -#define TAG_MASK 0xFF000000U - -#define MAXSIZE (1 << 16) - -#define COL(_l) ((_l) % cols) -#define LINE(_l) ((_l) / cols) - -#define NL '\n' - -/* Main interface functions. */ -static void ttysl_stop(ErlDrvData); -static void ttysl_from_erlang(ErlDrvData, char*, ErlDrvSizeT); -static void ttysl_to_tty(ErlDrvData, ErlDrvEvent); -static void ttysl_flush_tty(ErlDrvData); -static void ttysl_from_tty(ErlDrvData, ErlDrvEvent); -static void ttysl_stop_select(ErlDrvEvent, void*); -static Sint16 get_sint16(char*); - -static ErlDrvPort ttysl_port; -static int ttysl_fd; -static int ttysl_terminate = 0; -static int ttysl_send_ok = 0; -static ErlDrvBinary *putcbuf; -static int putcpos; -static int putclen; - -/* Functions that work on the line buffer. */ -static int start_lbuf(void); -static int stop_lbuf(void); -static int put_chars(byte*,int); -static int move_rel(int); -static int ins_chars(byte *,int); -static int del_chars(int); -static int step_over_chars(int); -static int insert_buf(byte*,int); -static int write_buf(Uint32 *,int,int); -static int outc(int c); -static int move_cursor(int,int); -static int cp_pos_to_col(int cp_pos); - - -/* Termcap functions. */ -static int start_termcap(void); -static int stop_termcap(void); -static int move_left(int); -static int move_right(int); -static int move_up(int); -static int move_down(int); -static void update_cols(void); - -/* Terminal setting functions. */ -static int tty_init(int,int,int,int); -static int tty_set(int); -static int tty_reset(int); -static ErlDrvSSizeT ttysl_control(ErlDrvData, unsigned int, - char *, ErlDrvSizeT, char **, ErlDrvSizeT); -#ifdef ERTS_NOT_USED -static RETSIGTYPE suspend(int); -#endif -static RETSIGTYPE cont(int); -static RETSIGTYPE winch(int); - -/*#define LOG_DEBUG*/ - -#ifdef LOG_DEBUG -FILE *debuglog = NULL; - -#define DEBUGLOG(X) \ -do { \ - if (debuglog != NULL) { \ - my_debug_printf X; \ - } \ -} while (0) - -static void my_debug_printf(char *fmt, ...) -{ - char buffer[1024]; - va_list args; - - va_start(args, fmt); - erts_vsnprintf(buffer,1024,fmt,args); - va_end(args); - erts_fprintf(debuglog,"%s\n",buffer); - /*erts_printf("Debuglog = %s\n",buffer);*/ -} - -#else - -#define DEBUGLOG(X) - -#endif - -static int utf8_mode = 0; -static byte utf8buf[4]; /* for incomplete input */ -static int utf8buf_size; /* size of incomplete input */ - -# define IF_IMPL(x) x -#else -# define IF_IMPL(x) NULL -#endif /* HAVE_TERMCAP */ - -/* Define the driver table entry. */ -struct erl_drv_entry ttsl_driver_entry = { - ttysl_init, - ttysl_start, - IF_IMPL(ttysl_stop), - IF_IMPL(ttysl_from_erlang), - IF_IMPL(ttysl_from_tty), - IF_IMPL(ttysl_to_tty), - "tty_sl", /* driver_name */ - NULL, /* finish */ - NULL, /* handle */ - IF_IMPL(ttysl_control), - NULL, /* timeout */ - NULL, /* outputv */ - NULL, /* ready_async */ - IF_IMPL(ttysl_flush_tty), - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - 0, /* ERL_DRV_FLAGs */ - NULL, /* handle2 */ - NULL, /* process_exit */ - IF_IMPL(ttysl_stop_select) -}; - - -static int ttysl_init(void) -{ -#ifdef HAVE_TERMCAP - ttysl_port = (ErlDrvPort)-1; - ttysl_fd = -1; - lbuf = NULL; /* For line buffer handling */ - capbuf = NULL; /* For termcap handling */ -#endif -#ifdef LOG_DEBUG - { - char *dl; - if ((dl = getenv("TTYSL_DEBUG_LOG")) != NULL && *dl) { - debuglog = fopen(dl,"w+"); - if (debuglog != NULL) - setbuf(debuglog,NULL); - } - DEBUGLOG(("ttysl_init: Debuglog = %s(0x%ld)\n",dl,(long) debuglog)); - } -#endif - return 0; -} - -static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) -{ -#ifndef HAVE_TERMCAP - return ERL_DRV_ERROR_GENERAL; -#else - char *s, *t, *l; - int canon, echo, sig; /* Terminal characteristics */ - int flag; - extern int using_oldshell; /* set this to let the rest of erts know */ - - DEBUGLOG(("ttysl_start: driver input \"%s\", ttysl_port = %d (-1 expected)", buf, ttysl_port)); - utf8buf_size = 0; - if (ttysl_port != (ErlDrvPort)-1) { - DEBUGLOG(("ttysl_start: failure with ttysl_port = %d, not initialized properly?\n", ttysl_port)); - return ERL_DRV_ERROR_GENERAL; - } - - DEBUGLOG(("ttysl_start: isatty(0) = %d (1 expected), isatty(1) = %d (1 expected)", isatty(0), isatty(1))); - if (!isatty(0) || !isatty(1)) { - DEBUGLOG(("ttysl_start: failure in isatty, isatty(0) = %d, isatty(1) = %d", isatty(0), isatty(1))); - return ERL_DRV_ERROR_GENERAL; - } - - /* Set the terminal modes to default leave as is. */ - canon = echo = sig = 0; - - /* Parse the input parameters. */ - for (s = strchr(buf, ' '); s; s = t) { - s++; - /* Find end of this argument (start of next) and insert NUL. */ - if ((t = strchr(s, ' '))) { - *t = '\0'; - } - if ((flag = ((*s == '+') ? 1 : ((*s == '-') ? -1 : 0)))) { - if (s[1] == 'c') canon = flag; - if (s[1] == 'e') echo = flag; - if (s[1] == 's') sig = flag; - } - else if ((ttysl_fd = open(s, O_RDWR, 0)) < 0) { - DEBUGLOG(("ttysl_start: failed to open ttysl_fd, open(%s, O_RDWR, 0)) = %d\n", s, ttysl_fd)); - return ERL_DRV_ERROR_GENERAL; - } - } - - if (ttysl_fd < 0) - ttysl_fd = 0; - - if (tty_init(ttysl_fd, canon, echo, sig) < 0 || - tty_set(ttysl_fd) < 0) { - DEBUGLOG(("ttysl_start: failed init tty or set tty\n")); - ttysl_port = (ErlDrvPort)-1; - tty_reset(ttysl_fd); - return ERL_DRV_ERROR_GENERAL; - } - - /* Set up smart line and termcap stuff. */ - if (!start_lbuf() || !start_termcap()) { - DEBUGLOG(("ttysl_start: failed to start_lbuf or start_termcap\n")); - stop_lbuf(); /* Must free this */ - tty_reset(ttysl_fd); - return ERL_DRV_ERROR_GENERAL; - } - - SET_NONBLOCKING(ttysl_fd); - -#ifdef PRIMITIVE_UTF8_CHECK - setlocale(LC_CTYPE, ""); /* Set international environment, - ignore result */ - if (((l = getenv("LC_ALL")) && *l) || - ((l = getenv("LC_CTYPE")) && *l) || - ((l = getenv("LANG")) && *l)) { - if (strstr(l, "UTF-8")) - utf8_mode = 1; - } - -#else - l = setlocale(LC_CTYPE, ""); /* Set international environment */ - if (l != NULL) { - utf8_mode = (strcmp(nl_langinfo(CODESET), "UTF-8") == 0); - DEBUGLOG(("ttysl_start: setlocale: %s",l)); - } -#endif - DEBUGLOG(("ttysl_start: utf8_mode is %s",(utf8_mode) ? "on" : "off")); - sys_signal(SIGCONT, cont); - sys_signal(SIGWINCH, winch); - - driver_select(port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1); - ttysl_port = port; - - /* we need to know this when we enter the break handler */ - using_oldshell = 0; - - DEBUGLOG(("ttysl_start: successful start\n")); - return (ErlDrvData)ttysl_port; /* Nothing important to return */ -#endif /* HAVE_TERMCAP */ -} - -#ifdef HAVE_TERMCAP - -#define DEF_HEIGHT 24 -#define DEF_WIDTH 80 -static void ttysl_get_window_size(Uint32 *width, Uint32 *height) -{ -#ifdef TIOCGWINSZ - struct winsize ws; - if (ioctl(ttysl_fd,TIOCGWINSZ,&ws) == 0) { - *width = (Uint32) ws.ws_col; - *height = (Uint32) ws.ws_row; - if (*width <= 0) - *width = DEF_WIDTH; - if (*height <= 0) - *height = DEF_HEIGHT; - return; - } -#endif - *width = DEF_WIDTH; - *height = DEF_HEIGHT; -} - -static ErlDrvSSizeT ttysl_control(ErlDrvData drv_data, - unsigned int command, - char *buf, ErlDrvSizeT len, - char **rbuf, ErlDrvSizeT rlen) -{ - char resbuff[2*sizeof(Uint32)]; - ErlDrvSizeT res_size; - - command -= ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER; - switch (command) { - case CTRL_OP_GET_WINSIZE: - { - Uint32 w,h; - ttysl_get_window_size(&w,&h); - memcpy(resbuff,&w,sizeof(Uint32)); - memcpy(resbuff+sizeof(Uint32),&h,sizeof(Uint32)); - res_size = 2*sizeof(Uint32); - } - break; - case CTRL_OP_GET_UNICODE_STATE: - *resbuff = (utf8_mode) ? 1 : 0; - res_size = 1; - break; - case CTRL_OP_SET_UNICODE_STATE: - if (len > 0) { - int m = (int) *buf; - *resbuff = (utf8_mode) ? 1 : 0; - res_size = 1; - utf8_mode = (m) ? 1 : 0; - } else { - return 0; - } - break; - default: - return -1; - } - if (rlen < res_size) { - *rbuf = driver_alloc(res_size); - } - memcpy(*rbuf,resbuff,res_size); - return res_size; -} - - -static void ttysl_stop(ErlDrvData ttysl_data) -{ - DEBUGLOG(("ttysl_stop: ttysl_port = %d\n",ttysl_port)); - if (ttysl_port != (ErlDrvPort)-1) { - stop_lbuf(); - stop_termcap(); - tty_reset(ttysl_fd); - driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, - ERL_DRV_WRITE|ERL_DRV_READ|ERL_DRV_USE, 0); - sys_signal(SIGCONT, SIG_DFL); - sys_signal(SIGWINCH, SIG_DFL); - } - ttysl_port = (ErlDrvPort)-1; - ttysl_fd = -1; - ttysl_terminate = 0; - /* return TRUE; */ -} - -static int put_utf8(int ch, byte *target, int sz, int *pos) -{ - Uint x = (Uint) ch; - if (x < 0x80) { - if (*pos >= sz) { - return -1; - } - target[(*pos)++] = (byte) x; - } - else if (x < 0x800) { - if (((*pos) + 1) >= sz) { - return -1; - } - target[(*pos)++] = (((byte) (x >> 6)) | - ((byte) 0xC0)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ - return -1; - } - if (((*pos) + 2) >= sz) { - return -1; - } - - target[(*pos)++] = (((byte) (x >> 12)) | - ((byte) 0xE0)); - target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else if (x < 0x110000) { /* Standard imposed max */ - if (((*pos) + 3) >= sz) { - return -1; - } - target[(*pos)++] = (((byte) (x >> 18)) | - ((byte) 0xF0)); - target[(*pos)++] = ((((byte) (x >> 12)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else { - return -1; - } - return 0; -} - - -static int pick_utf8(byte *s, int sz, int *pos) -{ - int size = sz - (*pos); - byte *source; - Uint unipoint; - - if (size > 0) { - source = s + (*pos); - if (((*source) & ((byte) 0x80)) == 0) { - unipoint = (int) *source; - ++(*pos); - return (int) unipoint; - } else if (((*source) & ((byte) 0xE0)) == 0xC0) { - if (size < 2) { - return -2; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((*source) < 0xC2) /* overlong */) { - return -1; - } - (*pos) += 2; - unipoint = - (((Uint) ((*source) & ((byte) 0x1F))) << 6) | - ((Uint) (source[1] & ((byte) 0x3F))); - return (int) unipoint; - } else if (((*source) & ((byte) 0xF0)) == 0xE0) { - if (size < 3) { - return -2; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((source[2] & ((byte) 0xC0)) != 0x80) || - (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return -1; - } - if ((((*source) & ((byte) 0xF)) == 0xD) && - ((source[1] & 0x20) != 0)) { - return -1; - } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return -1; - } - (*pos) += 3; - unipoint = - (((Uint) ((*source) & ((byte) 0xF))) << 12) | - (((Uint) (source[1] & ((byte) 0x3F))) << 6) | - ((Uint) (source[2] & ((byte) 0x3F))); - return (int) unipoint; - } else if (((*source) & ((byte) 0xF8)) == 0xF0) { - if (size < 4) { - return -2 ; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((source[2] & ((byte) 0xC0)) != 0x80) || - ((source[3] & ((byte) 0xC0)) != 0x80) || - (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return -1; - } - if ((((*source) & ((byte)0x7)) > 0x4U) || - ((((*source) & ((byte)0x7)) == 0x4U) && - ((source[1] & ((byte)0x3F)) > 0xFU))) { - return -1; - } - (*pos) += 4; - unipoint = - (((Uint) ((*source) & ((byte) 0x7))) << 18) | - (((Uint) (source[1] & ((byte) 0x3F))) << 12) | - (((Uint) (source[2] & ((byte) 0x3F))) << 6) | - ((Uint) (source[3] & ((byte) 0x3F))); - return (int) unipoint; - } else { - return -1; - } - } else { - return -1; - } -} - -static int octal_or_hex_positions(Uint c) -{ - int x = 0; - Uint ch = c; - if (!ch) { - return 1; - } - while(ch) { - ++x; - ch >>= 3; - } - if (x <= 3) { - return 3; - } - /* \x{H ...} format when larger than \777 */ - x = 0; - ch = c; - while(ch) { - ++x; - ch >>= 4; - } - return x+3; -} - -static void octal_or_hex_format(Uint ch, byte *buf, int *pos) -{ - static byte hex_chars[] = { '0','1','2','3','4','5','6','7','8','9', - 'A','B','C','D','E','F'}; - int num = octal_or_hex_positions(ch); - if (num != 3) { - ASSERT(num > 3); - buf[(*pos)++] = 'x'; - buf[(*pos)++] = '{'; - num -= 3; - while(num--) { - buf[(*pos)++] = hex_chars[((ch >> (4*num)) & 0xFU)]; - } - buf[(*pos)++] = '}'; - } else { - while(num--) { - buf[(*pos)++] = ((byte) ((ch >> (3*num)) & 0x7U) + '0'); - } - } -} - -/* - * Check that there is enough room in all buffers to copy all pad chars - * and stiff we need If not, realloc lbuf. - */ -static int check_buf_size(byte *s, int n) -{ - int pos = 0; - int ch; - int size = 10; - - DEBUGLOG(("check_buf_size: n = %d",n)); - while(pos < n) { - /* Indata is always UTF-8 */ - if ((ch = pick_utf8(s,n,&pos)) < 0) { - /* XXX temporary allow invalid chars */ - ch = (int) s[pos]; - DEBUGLOG(("check_buf_size: Invalid UTF8:%d",ch)); - ++pos; - } - if (utf8_mode) { /* That is, terminal is UTF8 compliant */ - if (ch >= 128 || isprint(ch)) { -#ifdef HAVE_WCWIDTH - int width; -#endif - DEBUGLOG(("check_buf_size: Printable(UTF-8:%d):%d",pos,ch)); - size++; -#ifdef HAVE_WCWIDTH - if ((width = wcwidth(ch)) > 1) { - size += width - 1; - } -#endif - } else if (ch == '\t') { - size += 8; - } else { - DEBUGLOG(("check_buf_size: Magic(UTF-8:%d):%d",pos,ch)); - size += 2; - } - } else { - if (ch <= 255 && isprint(ch)) { - DEBUGLOG(("check_buf_size: Printable:%d",ch)); - size++; - } else if (ch == '\t') - size += 8; - else if (ch >= 128) { - DEBUGLOG(("check_buf_size: Non printable:%d",ch)); - size += (octal_or_hex_positions(ch) + 1); - } - else { - DEBUGLOG(("check_buf_size: Magic:%d",ch)); - size += 2; - } - } - } - - if (size + lpos >= lbuf_size) { - - lbuf_size = size + lpos + BUFSIZ; - if ((lbuf = driver_realloc(lbuf, lbuf_size * sizeof(Uint32))) == NULL) { - DEBUGLOG(("check_buf_size: alloc failure of %d bytes", lbuf_size * sizeof(Uint32))); - driver_failure(ttysl_port, -1); - return(0); - } - } - DEBUGLOG(("check_buf_size: success\n")); - return(1); -} - - -static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT count) -{ - ErlDrvSizeT sz; - - sz = driver_sizeq(ttysl_port); - - putclen = count > TTY_BUFFSIZE ? TTY_BUFFSIZE : count; - putcbuf = driver_alloc_binary(putclen); - putcpos = 0; - - if (lpos > MAXSIZE) - put_chars((byte*)"\n", 1); - - DEBUGLOG(("ttysl_from_erlang: OP = %d", buf[0])); - - switch (buf[0]) { - case OP_PUTC_SYNC: - /* Using sync means that we have to send an ok to the - controlling process for each command call. We delay - sending ok if the driver queue exceeds a certain size. - We do not set ourselves as a busy port, as this - could be very bad for user_drv, if it gets blocked on - the port_command. */ - /* fall through */ - case OP_PUTC: - DEBUGLOG(("ttysl_from_erlang: OP: Putc(%lu)",(unsigned long) count-1)); - if (check_buf_size((byte*)buf+1, count-1) == 0) - return; - put_chars((byte*)buf+1, count-1); - break; - case OP_MOVE: - move_rel(get_sint16(buf+1)); - break; - case OP_INSC: - if (check_buf_size((byte*)buf+1, count-1) == 0) - return; - ins_chars((byte*)buf+1, count-1); - break; - case OP_DELC: - del_chars(get_sint16(buf+1)); - break; - case OP_BEEP: - outc('\007'); - break; - default: - /* Unknown op, just ignore. */ - break; - } - - driver_enq_bin(ttysl_port,putcbuf,0,putcpos); - driver_free_binary(putcbuf); - - if (sz == 0) { - for (;;) { - int written, qlen; - SysIOVec *iov; - - iov = driver_peekq(ttysl_port,&qlen); - if (iov) - written = writev(ttysl_fd, iov, qlen > MAXIOV ? MAXIOV : qlen); - else - written = 0; - if (written < 0) { - if (errno == ERRNO_BLOCK || errno == EINTR) { - driver_select(ttysl_port,(ErlDrvEvent)(long)ttysl_fd, - ERL_DRV_USE|ERL_DRV_WRITE,1); - break; - } else { - DEBUGLOG(("ttysl_from_erlang: driver failure in writev(%d,..) = %d (errno = %d)\n", ttysl_fd, written, errno)); - driver_failure_posix(ttysl_port, errno); - return; - } - } else { - if (driver_deq(ttysl_port, written) == 0) - break; - } - } - } - - if (buf[0] == OP_PUTC_SYNC) { - if (driver_sizeq(ttysl_port) > TTY_BUFFSIZE && !ttysl_terminate) { - /* We delay sending the ack until the buffer has been consumed */ - ttysl_send_ok = 1; - } else { - ErlDrvTermData spec[] = { - ERL_DRV_PORT, driver_mk_port(ttysl_port), - ERL_DRV_ATOM, driver_mk_atom("ok"), - ERL_DRV_TUPLE, 2 - }; - ASSERT(ttysl_send_ok == 0); - erl_drv_output_term(driver_mk_port(ttysl_port), spec, - sizeof(spec) / sizeof(spec[0])); - } - } - - return; /* TRUE; */ -} - -static void ttysl_to_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) { - for (;;) { - int written, qlen; - SysIOVec *iov; - ErlDrvSizeT sz; - - iov = driver_peekq(ttysl_port,&qlen); - - DEBUGLOG(("ttysl_to_tty: qlen = %d", qlen)); - - if (iov) - written = writev(ttysl_fd, iov, qlen > MAXIOV ? MAXIOV : qlen); - else - written = 0; - if (written < 0) { - if (errno == EINTR) { - continue; - } else if (errno != ERRNO_BLOCK){ - DEBUGLOG(("ttysl_to_tty: driver failure in writev(%d,..) = %d (errno = %d)\n", ttysl_fd, written, errno)); - driver_failure_posix(ttysl_port, errno); - } - break; - } else { - sz = driver_deq(ttysl_port, written); - if (sz < TTY_BUFFSIZE && ttysl_send_ok) { - ErlDrvTermData spec[] = { - ERL_DRV_PORT, driver_mk_port(ttysl_port), - ERL_DRV_ATOM, driver_mk_atom("ok"), - ERL_DRV_TUPLE, 2 - }; - ttysl_send_ok = 0; - erl_drv_output_term(driver_mk_port(ttysl_port), spec, - sizeof(spec) / sizeof(spec[0])); - } - if (sz == 0) { - driver_select(ttysl_port,(ErlDrvEvent)(long)ttysl_fd, - ERL_DRV_WRITE,0); - if (ttysl_terminate) { - /* flush has been called, which means we should terminate - when queue is empty. This will not send any exit - message */ - DEBUGLOG(("ttysl_to_tty: ttysl_terminate normal\n")); - driver_failure_atom(ttysl_port, "normal"); - } - break; - } - } - } - - return; -} - -static void ttysl_flush_tty(ErlDrvData ttysl_data) { - DEBUGLOG(("ttysl_flush_tty: ..")); - ttysl_terminate = 1; - return; -} - -static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) -{ - byte b[1024]; - ssize_t i; - int ch = 0, pos = 0; - int left = 1024; - byte *p = b; - byte t[1024]; - int tpos; - - if (utf8buf_size > 0) { - memcpy(b,utf8buf,utf8buf_size); - left -= utf8buf_size; - p += utf8buf_size; - utf8buf_size = 0; - } - - DEBUGLOG(("ttysl_from_tty: remainder = %d", left)); - - if ((i = read((int)(SWord)fd, (char *) p, left)) >= 0) { - if (p != b) { - i += (p - b); - } - if (utf8_mode) { /* Hopefully an UTF8 terminal */ - while(pos < i && (ch = pick_utf8(b,i,&pos)) >= 0) - ; - if (ch == -2 && i - pos <= 4) { - /* bytes left to care for */ - utf8buf_size = i -pos; - memcpy(utf8buf,b+pos,utf8buf_size); - } else if (ch == -1) { - DEBUGLOG(("ttysl_from_tty: Giving up on UTF8 mode, invalid character")); - utf8_mode = 0; - goto latin_terminal; - } - driver_output(ttysl_port, (char *) b, pos); - } else { - latin_terminal: - tpos = 0; - while (pos < i) { - while (tpos < 1020 && pos < i) { /* Max 4 bytes for UTF8 */ - put_utf8((int) b[pos++], t, 1024, &tpos); - } - driver_output(ttysl_port, (char *) t, tpos); - tpos = 0; - } - } - } else if (errno != EAGAIN && errno != EWOULDBLOCK) { - DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d (errno = %d)\n", (int)(SWord)fd, i, errno)); - driver_failure(ttysl_port, -1); - } -} - -static void ttysl_stop_select(ErlDrvEvent e, void* _) -{ - int fd = (int)(long)e; - if (fd != 0) { - close(fd); - } -} - -/* Procedures for putting and getting integers to/from strings. */ -static Sint16 get_sint16(char *s) -{ - return ((*s << 8) | ((byte*)s)[1]); -} - -static int start_lbuf(void) -{ - if (!lbuf && !(lbuf = ( Uint32*) driver_alloc(lbuf_size * sizeof(Uint32)))) - return FALSE; - llen = 0; - lpos = 0; - return TRUE; -} - -static int stop_lbuf(void) -{ - if (lbuf) { - driver_free(lbuf); - lbuf = NULL; - } - return TRUE; -} - -/* Put l bytes (in UTF8) from s into the buffer and output them. */ -static int put_chars(byte *s, int l) -{ - int n; - - n = insert_buf(s, l); - if (lpos > llen) - llen = lpos; - if (n > 0) - write_buf(lbuf + lpos - n, n, 0); - return TRUE; -} - -/* - * Move the current position forwards or backwards within the current - * line. We know about padding. - */ -static int move_rel(int n) -{ - int npos; /* The new position */ - - /* Step forwards or backwards over the buffer. */ - npos = step_over_chars(n); - - /* Calculate move, updates pointers and move the cursor. */ - move_cursor(lpos, npos); - lpos = npos; - return TRUE; -} - -/* Insert characters into the buffer at the current position. */ -static int ins_chars(byte *s, int l) -{ - int n, tl; - Uint32 *tbuf = NULL; /* Suppress warning about use-before-set */ - - /* Move tail of buffer to make space. */ - if ((tl = llen - lpos) > 0) { - if ((tbuf = driver_alloc(tl * sizeof(Uint32))) == NULL) - return FALSE; - memcpy(tbuf, lbuf + lpos, tl * sizeof(Uint32)); - } - n = insert_buf(s, l); - if (tl > 0) { - memcpy(lbuf + lpos, tbuf, tl * sizeof(Uint32)); - driver_free(tbuf); - } - llen += n; - write_buf(lbuf + (lpos - n), llen - (lpos - n), 0); - move_cursor(llen, lpos); - return TRUE; -} - -/* - * Delete characters in the buffer. Can delete characters before (n < 0) - * and after (n > 0) the current position. Cursor left at beginning of - * deleted block. - */ -static int del_chars(int n) -{ - int i, l, r; - int pos; - int gcs; /* deleted grapheme characters */ - - update_cols(); - - /* Step forward or backwards over n logical characters. */ - pos = step_over_chars(n); - DEBUGLOG(("del_chars: %d from %d %d %d\n", n, lpos, pos, llen)); - if (pos > lpos) { - l = pos - lpos; /* Buffer characters to delete */ - r = llen - lpos - l; /* Characters after deleted */ - gcs = cp_pos_to_col(pos) - cp_pos_to_col(lpos); - /* Fix up buffer and buffer pointers. */ - if (r > 0) - memmove(lbuf + lpos, lbuf + pos, r * sizeof(Uint32)); - llen -= l; - /* Write out characters after, blank the tail and jump back to lpos. */ - write_buf(lbuf + lpos, r, 0); - for (i = gcs ; i > 0; --i) - outc(' '); - if (xn && COL(cp_pos_to_col(llen)+gcs) == 0) - { - outc(' '); - move_left(1); - } - move_cursor(llen + gcs, lpos); - } - else if (pos < lpos) { - l = lpos - pos; /* Buffer characters */ - r = llen - lpos; /* Characters after deleted */ - gcs = -move_cursor(lpos, lpos-l); /* Move back */ - /* Fix up buffer and buffer pointers. */ - if (r > 0) - memmove(lbuf + pos, lbuf + lpos, r * sizeof(Uint32)); - lpos -= l; - llen -= l; - /* Write out characters after, blank the tail and jump back to lpos. */ - write_buf(lbuf + lpos, r, 0); - for (i = gcs ; i > 0; --i) - outc(' '); - if (xn && COL(cp_pos_to_col(llen)+gcs) == 0) - { - outc(' '); - move_left(1); - } - move_cursor(llen + gcs, lpos); - } - return TRUE; -} - -/* Step over n logical characters, check for overflow. */ -static int step_over_chars(int n) -{ - Uint32 *c, *beg, *end; - - beg = lbuf; - end = lbuf + llen; - c = lbuf + lpos; - for ( ; n > 0 && c < end; --n) { - c++; - while (c < end && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) - c++; - } - for ( ; n < 0 && c > beg; n++) { - --c; - while (c > beg && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) - --c; - } - return c - lbuf; -} - -/* - * Insert n characters into the buffer at lpos. - * Know about pad characters and treat \n specially. - */ - -static int insert_buf(byte *s, int n) -{ - int pos = 0; - int buffpos = lpos; - int ch; - - while (pos < n) { - if ((ch = pick_utf8(s,n,&pos)) < 0) { - /* XXX temporary allow invalid chars */ - ch = (int) s[pos]; - DEBUGLOG(("insert_buf: Invalid UTF8:%d",ch)); - ++pos; - } - if ((utf8_mode && (ch >= 128 || isprint(ch))) || (ch <= 255 && isprint(ch))) { - DEBUGLOG(("insert_buf: Printable(UTF-8):%d",ch)); - lbuf[lpos++] = (Uint32) ch; - } else if (ch >= 128) { /* not utf8 mode */ - int nc = octal_or_hex_positions(ch); - lbuf[lpos++] = ((Uint32) ch) | ESCAPED_TAG; - while (nc--) { - lbuf[lpos++] = ESCAPED_TAG; - } - } else if (ch == '\t') { - do { - lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); - ch = 0; - } while (lpos % 8); - } else if (ch == '\e') { - DEBUGLOG(("insert_buf: ANSI Escape: \\e")); - lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); - } else if (ch == '\n' || ch == '\r') { - write_buf(lbuf + buffpos, lpos - buffpos, 1); - outc('\r'); - if (ch == '\n') - outc('\n'); - if (llen > lpos) { - memmove(lbuf, lbuf + lpos, llen - lpos); - } - llen -= lpos; - lpos = buffpos = 0; - } else { - DEBUGLOG(("insert_buf: Magic(UTF-8):%d",ch)); - lbuf[lpos++] = ch | CONTROL_TAG; - lbuf[lpos++] = CONTROL_TAG; - } - } - return lpos - buffpos; /* characters "written" into - current buffer (may be less due to newline) */ -} - - - -/* - * Write n characters in line buffer starting at s. Be smart about - * non-printables. Know about pad characters and that \n can never - * occur normally. - */ - -static int write_buf(Uint32 *s, int n, int next_char_is_crnl) -{ - byte ubuf[4]; - int ubytes = 0, i; - byte lastput = ' '; - - update_cols(); - - DEBUGLOG(("write_buf(%d, %d)",n,next_char_is_crnl)); - - while (n > 0) { - if (!(*s & TAG_MASK) ) { - if (utf8_mode) { - ubytes = 0; - if (put_utf8((int) *s, ubuf, 4, &ubytes) == 0) { - for (i = 0; i < ubytes; ++i) { - outc(ubuf[i]); - } - lastput = 0; /* Means the last written character was multibyte UTF8 */ - } - } else { - outc((byte) *s); - lastput = (byte) *s; - } - --n; - ++s; - } else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { - outc(lastput = ' '); - --n; s++; - while (n > 0 && *s == CONTROL_TAG) { - outc(lastput = ' '); - --n; s++; - } - } else if (*s == (CONTROL_TAG | ((Uint32) '\e'))) { - outc(lastput = '\e'); - --n; - ++s; - } else if (*s & CONTROL_TAG) { - outc('^'); - outc(lastput = ((byte) ((*s == 0177) ? '?' : *s | 0x40))); - n -= 2; - s += 2; - } else if (*s & ESCAPED_TAG) { - Uint32 ch = *s & ~(TAG_MASK); - byte *octbuff; - byte octtmp[256]; - int octbytes; - DEBUGLOG(("write_buf: Escaped: %d", ch)); - octbytes = octal_or_hex_positions(ch); - if (octbytes > 256) { - octbuff = driver_alloc(octbytes); - } else { - octbuff = octtmp; - } - octbytes = 0; - octal_or_hex_format(ch, octbuff, &octbytes); - DEBUGLOG(("write_buf: octbytes: %d", octbytes)); - outc('\\'); - for (i = 0; i < octbytes; ++i) { - outc(lastput = octbuff[i]); - DEBUGLOG(("write_buf: outc: %d", (int) lastput)); - } - n -= octbytes+1; - s += octbytes+1; - if (octbuff != octtmp) { - driver_free(octbuff); - } - } else { - DEBUGLOG(("write_buf: Very unexpected character %d",(int) *s)); - ++n; - --s; - } - } - /* Check landed in first column of new line and have 'xn' bug. - * https://www.gnu.org/software/termutils/manual/termcap-1.3/html_node/termcap_27.html - * - * The 'xn' bugs (from what I understand) is that the terminal cursor does - * not wrap to the next line when the current line is full. For example: - * - * If the terminal column size is 20 and we output 20 'a' the cursor will be - * on row 1, column 21. While we actually want it at row 2 column 0. So to - * achieve this the code below emits " \b", which will move the cursor to the - * correct place. - * - * We should not apply this 'xn' workaround if we know that the next character - * to be emitted is a cr|nl as that will wrap by itself. - */ - n = s - lbuf; - if (!next_char_is_crnl && xn && n != 0 && COL(cp_pos_to_col(n)) == 0) { - if (n >= llen) { - outc(' '); - } else if (lastput == 0) { /* A multibyte UTF8 character */ - for (i = 0; i < ubytes; ++i) { - outc(ubuf[i]); - } - } else { - outc(lastput); - } - move_left(1); - } - return TRUE; -} - - -/* The basic procedure for outputting one character. */ -static int outc(int c) -{ - putcbuf->orig_bytes[putcpos++] = c; - if (putcpos == putclen) { - driver_enq_bin(ttysl_port,putcbuf,0,putclen); - driver_free_binary(putcbuf); - putcpos = 0; - putclen = TTY_BUFFSIZE; - putcbuf = driver_alloc_binary(BUFSIZ); - } - return 1; -} - -static int move_cursor(int from_pos, int to_pos) -{ - int from_col, to_col; - int dc, dl; - update_cols(); - - from_col = cp_pos_to_col(from_pos); - to_col = cp_pos_to_col(to_pos); - - dc = COL(to_col) - COL(from_col); - dl = LINE(to_col) - LINE(from_col); - DEBUGLOG(("move_cursor: from %d %d to %d %d => %d %d\n", - from_pos, from_col, to_pos, to_col, dl, dc)); - if (dl > 0) - move_down(dl); - else if (dl < 0) - move_up(-dl); - if (dc > 0) - move_right(dc); - else if (dc < 0) - move_left(-dc); - return to_col-from_col; -} - -/* - * Returns the length of an ANSI escape code in a buffer, this function only consider - * color escape sequences like `\e[33m` or `\e[21;33m`. If a sequence has no valid - * terminator, the length is equal the number of characters between `\e` and the first - * invalid character, inclusive. - */ - -static int ansi_escape_width(Uint32 *s, int max_length) -{ - int i; - - if (*s != (CONTROL_TAG | ((Uint32) '\e'))) { - return 0; - } else if (max_length <= 1) { - return 1; - } else if (s[1] != '[') { - return 2; - } - - for (i = 2; i < max_length && (s[i] == ';' || (s[i] >= '0' && s[i] <= '9')); i++); - - return i + 1; -} - -static int cp_pos_to_col(int cp_pos) -{ - /* - * If we don't have any character width information. Assume that - * code points are one column wide - */ - int w = 1; - int col = 0; - int i = 0; - int j; - - if (cp_pos > llen) { - col += cp_pos - llen; - cp_pos = llen; - } - - while (i < cp_pos) { - j = ansi_escape_width(lbuf + i, llen - i); - - if (j > 0) { - i += j; - } else { -#ifdef HAVE_WCWIDTH - w = wcwidth(lbuf[i]); -#endif - if (w > 0) { - col += w; - } - i++; - } - } - - return col; -} - -static int start_termcap(void) -{ - int eres; - size_t envsz = 1024; - char *env = NULL; - char *c; - int tres; - - DEBUGLOG(("start_termcap: ..")); - - capbuf = driver_alloc(1024); - if (!capbuf) - goto termcap_false; - eres = erl_drv_getenv("TERM", capbuf, &envsz); - if (eres == 0) - env = capbuf; - else if (eres < 0) { - DEBUGLOG(("start_termcap: failure in erl_drv_getenv(\"TERM\", ..) = %d\n", eres)); - goto termcap_false; - } else /* if (eres > 1) */ { - char *envbuf = driver_alloc(envsz); - if (!envbuf) - goto termcap_false; - while (1) { - char *newenvbuf; - eres = erl_drv_getenv("TERM", envbuf, &envsz); - if (eres == 0) - break; - newenvbuf = driver_realloc(envbuf, envsz); - if (eres < 0 || !newenvbuf) { - DEBUGLOG(("start_termcap: failure in erl_drv_getenv(\"TERM\", ..) = %d or realloc buf == %p\n", eres, newenvbuf)); - env = newenvbuf ? newenvbuf : envbuf; - goto termcap_false; - } - envbuf = newenvbuf; - } - env = envbuf; - } - if ((tres = tgetent((char*)lbuf, env)) <= 0) { - DEBUGLOG(("start_termcap: failure in tgetent(..) = %d\n", tres)); - goto termcap_false; - } - if (env != capbuf) { - env = NULL; - driver_free(env); - } - c = capbuf; - cols = tgetnum("co"); - if (cols <= 0) - cols = DEF_WIDTH; - xn = tgetflag("xn"); - up = tgetstr("up", &c); - if (!(down = tgetstr("do", &c))) - down = "\n"; - if (!(left = tgetflag("bs") ? "\b" : tgetstr("bc", &c))) - left = "\b"; /* Can't happen - but does on Solaris 2 */ - right = tgetstr("nd", &c); - if (up && down && left && right) { - DEBUGLOG(("start_termcap: successful start\n")); - return TRUE; - } - DEBUGLOG(("start_termcap: failed start\n")); - termcap_false: - if (env && env != capbuf) - driver_free(env); - if (capbuf) - driver_free(capbuf); - capbuf = NULL; - return FALSE; -} - -static int stop_termcap(void) -{ - if (capbuf) driver_free(capbuf); - capbuf = NULL; - return TRUE; -} - -static int move_left(int n) -{ - while (n-- > 0) - tputs(left, 1, outc); - return TRUE; -} - -static int move_right(int n) -{ - while (n-- > 0) - tputs(right, 1, outc); - return TRUE; -} - -static int move_up(int n) -{ - while (n-- > 0) - tputs(up, 1, outc); - return TRUE; -} - -static int move_down(int n) -{ - while (n-- > 0) - tputs(down, 1, outc); - return TRUE; -} - - -/* - * Updates cols if terminal has resized (SIGWINCH). Should be called - * at the start of any function that uses the COL or LINE macros. If - * the terminal is resized after calling this function but before use - * of the macros, then we may write to the wrong screen location. - * - * We cannot call this from the SIGWINCH handler because it uses - * ioctl() which is not a safe function as listed in the signal(7) - * man page. - */ -static void update_cols(void) -{ - Uint32 width, height; - - if (cols_needs_update) { - cols_needs_update = FALSE; - ttysl_get_window_size(&width, &height); - cols = width; - } -} - - -/* - * Put a terminal device into non-canonical mode with ECHO off. - * Before doing so we first save the terminal's current mode, - * assuming the caller will call the tty_reset() function - * (also in this file) when it's done with raw mode. - */ - -static struct termios tty_smode, tty_rmode; - -static int tty_init(int fd, int canon, int echo, int sig) { - int tres; - DEBUGLOG(("tty_init: fd = %d, canon = %d, echo = %d, sig = %d", fd, canon, echo, sig)); - if ((tres = tcgetattr(fd, &tty_rmode)) < 0) { - DEBUGLOG(("tty_init: failure in tcgetattr(%d,..) = %d\n", fd, tres)); - return -1; - } - tty_smode = tty_rmode; - - /* Default characteristics for all usage including termcap output. */ - tty_smode.c_iflag &= ~ISTRIP; - - /* Turn canonical (line mode) on off. */ - if (canon > 0) { - tty_smode.c_iflag |= ICRNL; - tty_smode.c_lflag |= ICANON; - tty_smode.c_oflag |= OPOST; - tty_smode.c_cc[VEOF] = tty_rmode.c_cc[VEOF]; -#ifdef VDSUSP - tty_smode.c_cc[VDSUSP] = tty_rmode.c_cc[VDSUSP]; -#endif - } - if (canon < 0) { - tty_smode.c_iflag &= ~ICRNL; - tty_smode.c_lflag &= ~ICANON; - tty_smode.c_oflag &= ~OPOST; - /* Must get these really right or funny effects can occur. */ - tty_smode.c_cc[VMIN] = 1; - tty_smode.c_cc[VTIME] = 0; -#ifdef VDSUSP - tty_smode.c_cc[VDSUSP] = 0; -#endif - } - - /* Turn echo on or off. */ - if (echo > 0) - tty_smode.c_lflag |= ECHO; - if (echo < 0) - tty_smode.c_lflag &= ~ECHO; - - /* Set extra characteristics for "RAW" mode, no signals. */ - if (sig > 0) { - /* Ignore IMAXBEL as not POSIX. */ -#ifndef QNX - tty_smode.c_iflag |= (BRKINT|IGNPAR|ICRNL|IXON|IXANY); -#else - tty_smode.c_iflag |= (BRKINT|IGNPAR|ICRNL|IXON); -#endif - tty_smode.c_lflag |= (ISIG|IEXTEN); - } - if (sig < 0) { - /* Ignore IMAXBEL as not POSIX. */ -#ifndef QNX - tty_smode.c_iflag &= ~(BRKINT|IGNPAR|ICRNL|IXON|IXANY); -#else - tty_smode.c_iflag &= ~(BRKINT|IGNPAR|ICRNL|IXON); -#endif - tty_smode.c_lflag &= ~(ISIG|IEXTEN); - } - DEBUGLOG(("tty_init: successful init\n")); - return 0; -} - -/* - * Set/restore a terminal's mode to whatever it was on the most - * recent call to the tty_init() function above. - */ - -static int tty_set(int fd) -{ - int tres; - DEBUGF(("tty_set: Setting tty...\n")); - - if ((tres = tcsetattr(fd, TCSANOW, &tty_smode)) < 0) { - DEBUGLOG(("tty_set: failure in tcgetattr(%d,..) = %d\n", fd, tres)); - return(-1); - } - return(0); -} - -static int tty_reset(int fd) /* of terminal device */ -{ - int tres; - DEBUGF(("tty_reset: Resetting tty...\n")); - - if ((tres = tcsetattr(fd, TCSANOW, &tty_rmode)) < 0) { - DEBUGLOG(("tty_reset: failure in tcsetattr(%d,..) = %d\n", fd, tres)); - return(-1); - } - return(0); -} - -/* - * Signal handler to cope with signals so that we can reset the tty - * to the original settings - */ - -#ifdef ERTS_NOT_USED -/* XXX: A mistake that it isn't used, or should it be removed? */ - -static RETSIGTYPE suspend(int sig) -{ - if (tty_reset(ttysl_fd) < 0) { - DEBUGLOG(("signal: failure in suspend(%d), can't reset tty %d\n", sig, ttysl_fd)); - fprintf(stderr,"Can't reset tty \n"); - exit(1); - } - - sys_signal(sig, SIG_DFL); /* Set signal handler to default */ - sys_sigrelease(sig); /* Allow 'sig' to come through */ - kill(getpid(), sig); /* Send ourselves the signal */ - sys_sigblock(sig); /* Reset to old mask */ - sys_signal(sig, suspend); /* Reset signal handler */ - - if (tty_set(ttysl_fd) < 0) { - DEBUGLOG(("signal: failure in suspend(%d), can't set tty %d\n", sig, ttysl_fd)); - fprintf(stderr,"Can't set tty raw \n"); - exit(1); - } -} - -#endif - -static RETSIGTYPE cont(int sig) -{ - if (tty_set(ttysl_fd) < 0) { - DEBUGLOG(("signal: failure in cont(%d), can't set tty raw %d\n", sig, ttysl_fd)); - fprintf(stderr,"Can't set tty raw\n"); - exit(1); - } -} - -static RETSIGTYPE winch(int sig) -{ - cols_needs_update = TRUE; -} -#endif /* HAVE_TERMCAP */ diff --git a/erts/emulator/drivers/win32/ttsl_drv.c b/erts/emulator/drivers/win32/ttsl_drv.c deleted file mode 100644 index 8917e48919..0000000000 --- a/erts/emulator/drivers/win32/ttsl_drv.c +++ /dev/null @@ -1,786 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2021. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Tty driver that reads one character at the time and provides a - * smart line for output. - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#include <ctype.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <signal.h> - -#include "erl_driver.h" -#include "win_con.h" - -#define TRUE 1 -#define FALSE 0 - -static int cols; /* Number of columns available. */ -static int rows; /* Number of rows available. */ - -/* The various opcodes. */ -#define OP_PUTC 0 -#define OP_MOVE 1 -#define OP_INSC 2 -#define OP_DELC 3 -#define OP_BEEP 4 -#define OP_PUTC_SYNC 5 - -/* Control op */ -#define CTRL_OP_GET_WINSIZE 100 -#define CTRL_OP_GET_UNICODE_STATE 101 -#define CTRL_OP_SET_UNICODE_STATE 102 - -static int lbuf_size = BUFSIZ; -Uint32 *lbuf; /* The current line buffer */ -int llen; /* The current line length */ -int lpos; /* The current "cursor position" in the line buffer */ - -/* - * Tags used in line buffer to show that these bytes represent special characters, - * Max unicode is 0x0010ffff, so we have lots of place for meta tags... - */ -#define CONTROL_TAG 0x10000000U /* Control character, value in first position */ -#define ESCAPED_TAG 0x01000000U /* Escaped character, value in first position */ -#define TAG_MASK 0xFF000000U - -#define MAXSIZE (1 << 16) - -#define ISPRINT(c) (isprint(c) || (128+32 <= (c) && (c) < 256)) - -#define DEBUGLOG(X) /* nothing */ - -/* - * XXX These are used by win_con.c (for command history). - * Should be cleaned up. - */ - - -#define NL '\n' - -/* Main interface functions. */ -static int ttysl_init(void); -static ErlDrvData ttysl_start(ErlDrvPort, char*); -static void ttysl_stop(ErlDrvData); -static ErlDrvSSizeT ttysl_control(ErlDrvData, unsigned int, - char *, ErlDrvSizeT, char **, ErlDrvSizeT); -static void ttysl_from_erlang(ErlDrvData, char*, ErlDrvSizeT); -static void ttysl_from_tty(ErlDrvData, ErlDrvEvent); -static Sint16 get_sint16(char *s); - -static ErlDrvPort ttysl_port; - -extern ErlDrvEvent console_input_event; -extern HANDLE console_thread; - -static HANDLE ttysl_in = INVALID_HANDLE_VALUE; /* Handle for console input. */ -static HANDLE ttysl_out = INVALID_HANDLE_VALUE; /* Handle for console output */ - -/* Functions that work on the line buffer. */ -static int start_lbuf(); -static int stop_lbuf(); -static int put_chars(); -static int move_rel(); -static int ins_chars(); -static int del_chars(); -static int step_over_chars(int n); -static int insert_buf(); -static int write_buf(); -static void move_cursor(int, int); - -/* Define the driver table entry. */ -struct erl_drv_entry ttsl_driver_entry = { - ttysl_init, - ttysl_start, - ttysl_stop, - ttysl_from_erlang, - ttysl_from_tty, - NULL, - "tty_sl", - NULL, - NULL, - ttysl_control, - NULL, /* timeout */ - NULL, /* outputv */ - NULL, /* ready_async */ - NULL, /* flush */ - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - 0, - NULL, - NULL, - NULL, -}; - -static int utf8_mode = 0; - -static int ttysl_init() -{ - lbuf = NULL; /* For line buffer handling */ - ttysl_port = (ErlDrvPort)-1; - return 0; -} - -static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) -{ - if ((SWord)ttysl_port != -1 || console_thread == NULL) { - return ERL_DRV_ERROR_GENERAL; - } - start_lbuf(); - utf8_mode = 1; - driver_select(port, console_input_event, ERL_DRV_READ, 1); - ttysl_port = port; - return (ErlDrvData)ttysl_port;/* Nothing important to return */ -} - -#define DEF_HEIGHT 24 -#define DEF_WIDTH 80 - -static void ttysl_get_window_size(Uint32 *width, Uint32 *height) -{ - *width = ConGetColumns(); - *height = ConGetRows(); -} - - -static ErlDrvSSizeT ttysl_control(ErlDrvData drv_data, - unsigned int command, - char *buf, ErlDrvSizeT len, - char **rbuf, ErlDrvSizeT rlen) -{ - char resbuff[2*sizeof(Uint32)]; - ErlDrvSizeT res_size; - - command -= ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER; - switch (command) { - case CTRL_OP_GET_WINSIZE: - { - Uint32 w,h; - ttysl_get_window_size(&w,&h); - memcpy(resbuff,&w,sizeof(Uint32)); - memcpy(resbuff+sizeof(Uint32),&h,sizeof(Uint32)); - res_size = 2*sizeof(Uint32); - } - break; - case CTRL_OP_GET_UNICODE_STATE: - *resbuff = (utf8_mode) ? 1 : 0; - res_size = 1; - break; - case CTRL_OP_SET_UNICODE_STATE: - if (len != 0) { - int m = (int) *buf; - *resbuff = (utf8_mode) ? 1 : 0; - res_size = 1; - utf8_mode = (m) ? 1 : 0; - } else { - return 0; - } - break; - default: - return -1; - } - if (rlen < res_size) { - *rbuf = driver_alloc(res_size); - } - memcpy(*rbuf,resbuff,res_size); - return res_size; -} - - -static void ttysl_stop(ErlDrvData ttysl_data) -{ - if ((SWord)ttysl_port != -1) { - driver_select(ttysl_port, console_input_event, ERL_DRV_READ, 0); - } - - ttysl_in = ttysl_out = INVALID_HANDLE_VALUE; - stop_lbuf(); - ttysl_port = (ErlDrvPort)-1; -} - -static int put_utf8(int ch, byte *target, int sz, int *pos) -{ - Uint x = (Uint) ch; - if (x < 0x80) { - if (*pos >= sz) { - return -1; - } - target[(*pos)++] = (byte) x; - } - else if (x < 0x800) { - if (((*pos) + 1) >= sz) { - return -1; - } - target[(*pos)++] = (((byte) (x >> 6)) | - ((byte) 0xC0)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ - return -1; - } - if (((*pos) + 2) >= sz) { - return -1; - } - - target[(*pos)++] = (((byte) (x >> 12)) | - ((byte) 0xE0)); - target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else if (x < 0x110000) { /* Standard imposed max */ - if (((*pos) + 3) >= sz) { - return -1; - } - target[(*pos)++] = (((byte) (x >> 18)) | - ((byte) 0xF0)); - target[(*pos)++] = ((((byte) (x >> 12)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F) | - ((byte) 0x80)); - target[(*pos)++] = (((byte) (x & 0x3F)) | - ((byte) 0x80)); - } else { - return -1; - } - return 0; -} - - -static int pick_utf8(byte *s, int sz, int *pos) -{ - int size = sz - (*pos); - byte *source; - Uint unipoint; - - if (size > 0) { - source = s + (*pos); - if (((*source) & ((byte) 0x80)) == 0) { - unipoint = (int) *source; - ++(*pos); - return (int) unipoint; - } else if (((*source) & ((byte) 0xE0)) == 0xC0) { - if (size < 2) { - return -2; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((*source) < 0xC2) /* overlong */) { - return -1; - } - (*pos) += 2; - unipoint = - (((Uint) ((*source) & ((byte) 0x1F))) << 6) | - ((Uint) (source[1] & ((byte) 0x3F))); - return (int) unipoint; - } else if (((*source) & ((byte) 0xF0)) == 0xE0) { - if (size < 3) { - return -2; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((source[2] & ((byte) 0xC0)) != 0x80) || - (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return -1; - } - if ((((*source) & ((byte) 0xF)) == 0xD) && - ((source[1] & 0x20) != 0)) { - return -1; - } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return -1; - } - (*pos) += 3; - unipoint = - (((Uint) ((*source) & ((byte) 0xF))) << 12) | - (((Uint) (source[1] & ((byte) 0x3F))) << 6) | - ((Uint) (source[2] & ((byte) 0x3F))); - return (int) unipoint; - } else if (((*source) & ((byte) 0xF8)) == 0xF0) { - if (size < 4) { - return -2 ; - } - if (((source[1] & ((byte) 0xC0)) != 0x80) || - ((source[2] & ((byte) 0xC0)) != 0x80) || - ((source[3] & ((byte) 0xC0)) != 0x80) || - (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return -1; - } - if ((((*source) & ((byte)0x7)) > 0x4U) || - ((((*source) & ((byte)0x7)) == 0x4U) && - ((source[1] & ((byte)0x3F)) > 0xFU))) { - return -1; - } - (*pos) += 4; - unipoint = - (((Uint) ((*source) & ((byte) 0x7))) << 18) | - (((Uint) (source[1] & ((byte) 0x3F))) << 12) | - (((Uint) (source[2] & ((byte) 0x3F))) << 6) | - ((Uint) (source[3] & ((byte) 0x3F))); - return (int) unipoint; - } else { - return -1; - } - } else { - return -1; - } -} - -static int octal_or_hex_positions(Uint c) -{ - int x = 0; - Uint ch = c; - if (!ch) { - return 1; - } - while(ch) { - ++x; - ch >>= 3; - } - if (x <= 3) { - return 3; - } - /* \x{H ...} format when larger than \777 */ - x = 0; - ch = c; - while(ch) { - ++x; - ch >>= 4; - } - return x+3; -} - -static void octal_or_hex_format(Uint ch, byte *buf, int *pos) -{ - static byte hex_chars[] = { '0','1','2','3','4','5','6','7','8','9', - 'A','B','C','D','E','F'}; - int num = octal_or_hex_positions(ch); - if (num != 3) { - buf[(*pos)++] = 'x'; - buf[(*pos)++] = '{'; - num -= 3; - while(num--) { - buf[(*pos)++] = hex_chars[((ch >> (4*num)) & 0xFU)]; - } - buf[(*pos)++] = '}'; - } else { - while(num--) { - buf[(*pos)++] = ((byte) ((ch >> (3*num)) & 0x7U) + '0'); - } - } -} - -/* - * Check that there is enough room in all buffers to copy all pad chars - * and stiff we need If not, realloc lbuf. - */ -static int check_buf_size(byte *s, int n) -{ - int pos = 0; - int ch; - int size = 10; - - while(pos < n) { - /* Indata is always UTF-8 */ - if ((ch = pick_utf8(s,n,&pos)) < 0) { - /* XXX temporary allow invalid chars */ - ch = (int) s[pos]; - DEBUGLOG(("Invalid UTF8:%d",ch)); - ++pos; - } - if (utf8_mode) { /* That is, terminal is UTF8 compliant */ - if (ch >= 128 || isprint(ch)) { - DEBUGLOG(("Printable(UTF-8:%d):%d",pos,ch)); - size++; /* Buffer contains wide characters... */ - } else if (ch == '\t') { - size += 8; - } else { - DEBUGLOG(("Magic(UTF-8:%d):%d",pos,ch)); - size += 2; - } - } else { - if (ch <= 255 && isprint(ch)) { - DEBUGLOG(("Printable:%d",ch)); - size++; - } else if (ch == '\t') - size += 8; - else if (ch >= 128) { - DEBUGLOG(("Non printable:%d",ch)); - size += (octal_or_hex_positions(ch) + 1); - } - else { - DEBUGLOG(("Magic:%d",ch)); - size += 2; - } - } - } - - if (size + lpos >= lbuf_size) { - - lbuf_size = size + lpos + BUFSIZ; - if ((lbuf = driver_realloc(lbuf, lbuf_size * sizeof(Uint32))) == NULL) { - driver_failure(ttysl_port, -1); - return(0); - } - } - return(1); -} - - -static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT count) -{ - if (lpos > MAXSIZE) - put_chars((byte*)"\n", 1); - - switch (buf[0]) { - case OP_PUTC: - case OP_PUTC_SYNC: - DEBUGLOG(("OP: Putc(%I64u)",(unsigned long long)count-1)); - if (check_buf_size((byte*)buf+1, count-1) == 0) - return; - put_chars((byte*)buf+1, count-1); - break; - case OP_MOVE: - move_rel(get_sint16(buf+1)); - break; - case OP_INSC: - if (check_buf_size((byte*)buf+1, count-1) == 0) - return; - ins_chars((byte*)buf+1, count-1); - break; - case OP_DELC: - del_chars(get_sint16(buf+1)); - break; - case OP_BEEP: - ConBeep(); - break; - default: - /* Unknown op, just ignore. */ - break; - } - - if (buf[0] == OP_PUTC_SYNC) { - /* On windows we do a blocking write to the tty so we just - send the ack immediately. If at some point in the future - someone has a problem with tty output being blocking - this has to be changed. */ - ErlDrvTermData spec[] = { - ERL_DRV_PORT, driver_mk_port(ttysl_port), - ERL_DRV_ATOM, driver_mk_atom("ok"), - ERL_DRV_TUPLE, 2 - }; - erl_drv_output_term(driver_mk_port(ttysl_port), spec, - sizeof(spec) / sizeof(spec[0])); - } - return; -} - -extern int read_inbuf(char *data, int n); - -static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) -{ - Uint32 inbuf[64]; - byte t[1024]; - int i,pos,tpos; - - i = ConReadInput(inbuf,1); - - pos = 0; - tpos = 0; - - while (pos < i) { - while (tpos < 1020 && pos < i) { /* Max 4 bytes for UTF8 */ - put_utf8((int) inbuf[pos++], t, 1024, &tpos); - } - driver_output(ttysl_port, (char *) t, tpos); - tpos = 0; - } -} - -/* - * Gets signed 16 bit integer from binary buffer. - */ -static Sint16 -get_sint16(char *s) -{ - return ((*s << 8) | ((byte*)s)[1]); -} - - -static int start_lbuf(void) -{ - if (!lbuf && !(lbuf = ( Uint32*) driver_alloc(lbuf_size * sizeof(Uint32)))) - return FALSE; - llen = 0; - lpos = 0; - return TRUE; -} - -static int stop_lbuf(void) -{ - if (lbuf) { - driver_free(lbuf); - lbuf = NULL; - } - llen = 0; /* To avoid access error in win_con:AddToCmdHistory during exit*/ - return TRUE; -} - -/* Put l bytes (in UTF8) from s into the buffer and output them. */ -static int put_chars(byte *s, int l) -{ - int n; - - n = insert_buf(s, l); - if (n > 0) - write_buf(lbuf + lpos - n, n); - if (lpos > llen) - llen = lpos; - return TRUE; -} - -/* - * Move the current position forwards or backwards within the current - * line. We know about padding. - */ -static int move_rel(int n) -{ - int npos; /* The new position */ - - /* Step forwards or backwards over the buffer. */ - npos = step_over_chars(n); - - /* Calculate move, updates pointers and move the cursor. */ - move_cursor(lpos, npos); - lpos = npos; - return TRUE; -} - -/* Insert characters into the buffer at the current position. */ -static int ins_chars(byte *s, int l) -{ - int n, tl; - Uint32 *tbuf = NULL; /* Suppress warning about use-before-set */ - - /* Move tail of buffer to make space. */ - if ((tl = llen - lpos) > 0) { - if ((tbuf = driver_alloc(tl * sizeof(Uint32))) == NULL) - return FALSE; - memcpy(tbuf, lbuf + lpos, tl * sizeof(Uint32)); - } - n = insert_buf(s, l); - if (tl > 0) { - memcpy(lbuf + lpos, tbuf, tl * sizeof(Uint32)); - driver_free(tbuf); - } - llen += n; - write_buf(lbuf + (lpos - n), llen - (lpos - n)); - move_cursor(llen, lpos); - return TRUE; -} - -/* - * Delete characters in the buffer. Can delete characters before (n < 0) - * and after (n > 0) the current position. Cursor left at beginning of - * deleted block. - */ -static int del_chars(int n) -{ - int i, l, r; - int pos; - - /*update_cols();*/ - - /* Step forward or backwards over n logical characters. */ - pos = step_over_chars(n); - - if (pos > lpos) { - l = pos - lpos; /* Buffer characters to delete */ - r = llen - lpos - l; /* Characters after deleted */ - /* Fix up buffer and buffer pointers. */ - if (r > 0) - memcpy(lbuf + lpos, lbuf + pos, r * sizeof(Uint32)); - llen -= l; - /* Write out characters after, blank the tail and jump back to lpos. */ - write_buf(lbuf + lpos, r); - for (i = l ; i > 0; --i) - ConPutChar(' '); - move_cursor(llen + l, lpos); - } - else if (pos < lpos) { - l = lpos - pos; /* Buffer characters */ - r = llen - lpos; /* Characters after deleted */ - move_cursor(lpos, lpos-l); /* Move back */ - /* Fix up buffer and buffer pointers. */ - if (r > 0) - memcpy(lbuf + pos, lbuf + lpos, r * sizeof(Uint32)); - lpos -= l; - llen -= l; - /* Write out characters after, blank the tail and jump back to lpos. */ - write_buf(lbuf + lpos, r); - for (i = l ; i > 0; --i) - ConPutChar(' '); - move_cursor(llen + l, lpos); - } - return TRUE; -} - - -/* Step over n logical characters, check for overflow. */ -static int step_over_chars(int n) -{ - Uint32 *c, *beg, *end; - - beg = lbuf; - end = lbuf + llen; - c = lbuf + lpos; - for ( ; n > 0 && c < end; --n) { - c++; - while (c < end && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) - c++; - } - for ( ; n < 0 && c > beg; n++) { - --c; - while (c > beg && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) - --c; - } - return c - lbuf; -} - -static int insert_buf(byte *s, int n) -{ - int pos = 0; - int buffpos = lpos; - int ch; - - while (pos < n) { - if ((ch = pick_utf8(s,n,&pos)) < 0) { - /* XXX temporary allow invalid chars */ - ch = (int) s[pos]; - DEBUGLOG(("insert_buf: Invalid UTF8:%d",ch)); - ++pos; - } - if ((utf8_mode && (ch >= 128 || isprint(ch))) || (ch <= 255 && isprint(ch))) { - DEBUGLOG(("insert_buf: Printable(UTF-8):%d",ch)); - lbuf[lpos++] = (Uint32) ch; - } else if (ch >= 128) { /* not utf8 mode */ - int nc = octal_or_hex_positions(ch); - lbuf[lpos++] = ((Uint32) ch) | ESCAPED_TAG; - while (nc--) { - lbuf[lpos++] = ESCAPED_TAG; - } - } else if (ch == '\t') { - do { - lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); - ch = 0; - } while (lpos % 8); - } else if (ch == '\n' || ch == '\r') { - write_buf(lbuf + buffpos, lpos - buffpos); - ConPutChar('\r'); - if (ch == '\n') - ConPutChar('\n'); - if (llen > lpos) { - memcpy(lbuf, lbuf + lpos, llen - lpos); - } - llen -= lpos; - lpos = buffpos = 0; - } else { - DEBUGLOG(("insert_buf: Magic(UTF-8):%d",ch)); - lbuf[lpos++] = ch | CONTROL_TAG; - lbuf[lpos++] = CONTROL_TAG; - } - } - return lpos - buffpos; /* characters "written" into - current buffer (may be less due to newline) */ -} -static int write_buf(Uint32 *s, int n) -{ - int i; - - /*update_cols();*/ - - while (n > 0) { - if (!(*s & TAG_MASK) ) { - ConPutChar(*s); - --n; - ++s; - } - else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { - ConPutChar(' '); - --n; s++; - while (n > 0 && *s == CONTROL_TAG) { - ConPutChar(' '); - --n; s++; - } - } else if (*s & CONTROL_TAG) { - ConPutChar('^'); - ConPutChar((*s == 0177) ? '?' : *s | 0x40); - n -= 2; - s += 2; - } else if (*s & ESCAPED_TAG) { - Uint32 ch = *s & ~(TAG_MASK); - byte *octbuff; - byte octtmp[256]; - int octbytes; - DEBUGLOG(("Escaped: %d", ch)); - octbytes = octal_or_hex_positions(ch); - if (octbytes > 256) { - octbuff = driver_alloc(octbytes); - } else { - octbuff = octtmp; - } - octbytes = 0; - octal_or_hex_format(ch, octbuff, &octbytes); - DEBUGLOG(("octbytes: %d", octbytes)); - ConPutChar('\\'); - for (i = 0; i < octbytes; ++i) { - ConPutChar(octbuff[i]); - } - n -= octbytes+1; - s += octbytes+1; - if (octbuff != octtmp) { - driver_free(octbuff); - } - } else { - DEBUGLOG(("Very unexpected character %d",(int) *s)); - ++n; - --s; - } - } - return TRUE; -} - - -static void -move_cursor(int from, int to) -{ - ConSetCursor(from,to); -} diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c deleted file mode 100644 index 2e3c12dc58..0000000000 --- a/erts/emulator/drivers/win32/win_con.c +++ /dev/null @@ -1,2355 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2021. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#define UNICODE 1 -#define _UNICODE 1 -#include <tchar.h> -#include <stdio.h> -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#include <windowsx.h> -#include "resource.h" -#include "erl_version.h" -#include <commdlg.h> -#include <commctrl.h> -#include "erl_driver.h" -#include "win_con.h" - -#define ALLOC(X) malloc(X) -#define REALLOC(X,Y) realloc(X,Y) -#define FREE(X) free(X) - -#if SIZEOF_VOID_P == 8 -#define WIN64 1 -#ifndef GCL_HBRBACKGROUND -#define GCL_HBRBACKGROUND GCLP_HBRBACKGROUND -#endif -#define DIALOG_PROC_RET INT_PTR -#define CF_HOOK_RET INT_PTR -#define CC_HOOK_RET INT_PTR -#define OFN_HOOK_RET INT_PTR -#else -#define DIALOG_PROC_RET BOOL -#define CF_HOOK_RET UINT -#define CC_HOOK_RET UINT -#define OFN_HOOK_RET UINT -#endif - - -#ifndef STATE_SYSTEM_INVISIBLE -/* Mingw problem with oleacc.h and WIN32_LEAN_AND_MEAN */ -#define STATE_SYSTEM_INVISIBLE 0x00008000 -#endif - -#define WM_CONTEXT (0x0401) -#define WM_CONBEEP (0x0402) -#define WM_SAVE_PREFS (0x0403) - -#define USER_KEY TEXT("Software\\Ericsson\\Erlang\\") TEXT(ERLANG_VERSION) - -#define FRAME_HEIGHT ((2*GetSystemMetrics(SM_CYEDGE))+(2*GetSystemMetrics(SM_CYFRAME))+GetSystemMetrics(SM_CYCAPTION)) -#define FRAME_WIDTH (2*GetSystemMetrics(SM_CXFRAME)+(2*GetSystemMetrics(SM_CXFRAME))+GetSystemMetrics(SM_CXVSCROLL)) - -#define LINE_LENGTH canvasColumns -#define COL(_l) ((_l) % LINE_LENGTH) -#define LINE(_l) ((_l) / LINE_LENGTH) - -#ifdef UNICODE -/* - * We use a character in the invalid unicode range - */ -#define SET_CURSOR (0xD8FF) -#else -/* - * XXX There is no escape to send a character 0x80. Fortunately, - * the ttsl driver currently replaces 0x80 with an octal sequence. - */ -#define SET_CURSOR (0x80) -#endif - -#define SCAN_CODE_BREAK 0x46 /* scan code for Ctrl-Break */ - - -typedef struct ScreenLine_s { - struct ScreenLine_s* next; - struct ScreenLine_s* prev; - int width; -#ifdef HARDDEBUG - int allocated; -#endif - int newline; /* Ends with hard newline: 1, wrapped at end: 0 */ - TCHAR *text; -} ScreenLine_t; - -extern Uint32 *lbuf; /* The current line buffer */ -extern int llen; /* The current line length */ -extern int lpos; - -HANDLE console_input_event; -HANDLE console_thread = NULL; - -#define DEF_CANVAS_COLUMNS 80 -#define DEF_CANVAS_ROWS 26 - -#define BUFSIZE 4096 -#define MAXBUFSIZE 32768 -typedef struct { - TCHAR *data; - int size; - int wrPos; - int rdPos; -} buffer_t; - -static buffer_t inbuf; -static buffer_t outbuf; - -static CHOOSEFONT cf; - -static TCHAR szFrameClass[] = TEXT("FrameClass"); -static TCHAR szClientClass[] = TEXT("ClientClass"); -static HWND hFrameWnd; -static HWND hClientWnd; -static HWND hTBWnd; -static HWND hComboWnd; -static HANDLE console_input; -static HANDLE console_output; -static int cxChar,cyChar, cxCharMax; -static int cxClient,cyClient; -static int cyToolBar; -static int iVscrollPos,iHscrollPos; -static int iVscrollMax,iHscrollMax; -static int nBufLines; -static int cur_x; -static int cur_y; -static int canvasColumns = DEF_CANVAS_COLUMNS; -static int canvasRows = DEF_CANVAS_ROWS; -static ScreenLine_t *buffer_top,*buffer_bottom; -static ScreenLine_t* cur_line; -static POINT editBeg,editEnd; -static BOOL fSelecting = FALSE; -static BOOL fTextSelected = FALSE; -static HKEY key; -static BOOL has_key = FALSE; -static LOGFONT logfont; -static DWORD fgColor; -static DWORD bkgColor; -static FILE *logfile = NULL; -static RECT winPos; -static BOOL toolbarVisible; -static BOOL destroyed = FALSE; - -static int lines_to_save = 10000; /* Maximum number of screen lines to save. */ - -#define TITLE_BUF_SZ 256 - -struct title_buf { - TCHAR *name; - TCHAR buf[TITLE_BUF_SZ]; -}; - -static TCHAR *erlang_window_title = TEXT("Erlang"); - -static unsigned __stdcall ConThreadInit(LPVOID param); -static LRESULT CALLBACK ClientWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam); -static DIALOG_PROC_RET CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam); -static ScreenLine_t *ConNewLine(void); -static void DeleteTopLine(void); -static void ensure_line_below(void); -static ScreenLine_t *GetLineFromY(int y); -static void LoadUserPreferences(void); -static void SaveUserPreferences(void); -static void set_scroll_info(HWND hwnd); -static void ConCarriageFeed(int); -static void ConScrollScreen(void); -static BOOL ConChooseFont(HWND hwnd); -static void ConFontInitialize(HWND hwnd); -static void ConSetFont(HWND hwnd); -static void ConChooseColor(HWND hwnd); -static void DrawSelection(HWND hwnd, POINT pt1, POINT pt2); -static void InvertSelectionArea(HWND hwnd); -static void OnEditCopy(HWND hwnd); -static void OnEditPaste(HWND hwnd); -static void OnEditSelAll(HWND hwnd); -static void GetFileName(HWND hwnd, TCHAR *pFile); -static void OpenLogFile(HWND hwnd); -static void CloseLogFile(HWND hwnd); -static void LogFileWrite(TCHAR *buf, int n); -static int write_inbuf(TCHAR *data, int n); -static void init_buffers(void); -static void AddToCmdHistory(void); -static int write_outbuf(TCHAR *data, int num_chars); -static void ConDrawText(HWND hwnd); -static BOOL (WINAPI *ctrl_handler)(DWORD); -static HWND InitToolBar(HWND hwndParent); -static void window_title(struct title_buf *); -static void free_window_title(struct title_buf *); -static void Client_OnMouseMove(HWND hwnd, int x, int y, UINT keyFlags); - -#ifdef HARDDEBUG -/* For really hard GUI startup debugging, place DEBUGBOX() macros in code - and get modal message boxes with the line number. */ -static void debug_box(int line) { - TCHAR buff[1024]; - swprintf(buff,1024,TEXT("DBG:%d"),line); - MessageBox(NULL,buff,TEXT("DBG"),MB_OK|MB_APPLMODAL); -} - -#define DEBUGBOX() debug_box(__LINE__) -#endif - -#define CON_VPRINTF_BUF_INC_SIZE 1024 - -static erts_dsprintf_buf_t * -grow_con_vprintf_buf(erts_dsprintf_buf_t *dsbufp, size_t need) -{ - char *buf; - size_t size; - - ASSERT(dsbufp); - - if (!dsbufp->str) { - size = (((need + CON_VPRINTF_BUF_INC_SIZE - 1) - / CON_VPRINTF_BUF_INC_SIZE) - * CON_VPRINTF_BUF_INC_SIZE); - buf = (char *) ALLOC(size * sizeof(char)); - } - else { - size_t free_size = dsbufp->size - dsbufp->str_len; - - if (need <= free_size) - return dsbufp; - - size = need - free_size + CON_VPRINTF_BUF_INC_SIZE; - size = (((size + CON_VPRINTF_BUF_INC_SIZE - 1) - / CON_VPRINTF_BUF_INC_SIZE) - * CON_VPRINTF_BUF_INC_SIZE); - size += dsbufp->size; - buf = (char *) REALLOC((void *) dsbufp->str, - size * sizeof(char)); - } - if (!buf) - return NULL; - if (buf != dsbufp->str) - dsbufp->str = buf; - dsbufp->size = size; - return dsbufp; -} - -static int con_vprintf(char *format, va_list arg_list) -{ - int res,i; - erts_dsprintf_buf_t dsbuf = ERTS_DSPRINTF_BUF_INITER(grow_con_vprintf_buf); - res = erts_vdsprintf(&dsbuf, format, arg_list); - if (res >= 0) { - TCHAR *tmp = ALLOC(dsbuf.str_len*sizeof(TCHAR)); - for (i=0;i<dsbuf.str_len;++i) { - tmp[i] = dsbuf.str[i]; - } - write_outbuf(tmp, dsbuf.str_len); - FREE(tmp); - } - if (dsbuf.str) - FREE((void *) dsbuf.str); - return res; -} - -void -ConInit(void) -{ - unsigned tid; - - console_input = CreateSemaphore(NULL, 0, 1, NULL); - console_output = CreateSemaphore(NULL, 0, 1, NULL); - console_input_event = CreateManualEvent(FALSE); - console_thread = (HANDLE *) _beginthreadex(NULL, 0, - ConThreadInit, - 0, 0, &tid); - - /* Make all erts_*printf on stdout and stderr use con_vprintf */ - erts_printf_stdout_func = con_vprintf; - erts_printf_stderr_func = con_vprintf; -} - -/* - ConNormalExit() is called from erts_exit() when the emulator - is stopping. If the exit has not been initiated by this - console thread (WM_DESTROY or ID_BREAK), the function must - invoke the console thread to save the user preferences. -*/ -void -ConNormalExit(void) -{ - if (!destroyed) - SendMessage(hFrameWnd, WM_SAVE_PREFS, 0L, 0L); -} - -void -ConWaitForExit(void) -{ - ConPrintf("\n\nAbnormal termination\n"); - WaitForSingleObject(console_thread, INFINITE); -} - -void ConSetCtrlHandler(BOOL (WINAPI *handler)(DWORD)) -{ - ctrl_handler = handler; -} - -int ConPutChar(Uint32 c) -{ - TCHAR sbuf[1]; -#ifdef HARDDEBUG - fprintf(stderr,"ConPutChar: %d\n",(int) c); - fflush(stderr); -#endif - sbuf[0] = c; - write_outbuf(sbuf, 1); - return 1; -} - -static int GetXFromLine(HDC hdc, int hscroll, int xpos,ScreenLine_t *pLine) -{ - SIZE size; - int hscrollPix = hscroll * cxChar; - - if (pLine == NULL) { - return 0; - } - - if (pLine->width < xpos) { - return (canvasColumns-hscroll)*cxChar; - } - /* Not needed (?): SelectObject(hdc,CreateFontIndirect(&logfont)); */ - if (GetTextExtentPoint32(hdc,pLine->text,xpos,&size)) { -#ifdef HARDDEBUG - fprintf(stderr,"size.cx:%d\n",(int)size.cx); - fflush(stderr); -#endif - if (hscrollPix >= size.cx) { - return 0; - } - return ((int) size.cx) - hscrollPix; - } else { - return (xpos-hscroll)*cxChar; - } -} - -static int GetXFromCurrentY(HDC hdc, int hscroll, int xpos) { - return GetXFromLine(hdc, hscroll, xpos, GetLineFromY(cur_y)); -} - -void ConSetCursor(int from, int to) -{ TCHAR cmd[9]; - int *p; - //DebugBreak(); - cmd[0] = SET_CURSOR; - /* - * XXX Expect trouble on CPUs which don't allow misaligned read and writes. - */ - p = (int *)&cmd[1]; - *p++ = from; - *p = to; - write_outbuf(cmd, 1 + (2*sizeof(int)/sizeof(TCHAR))); -} - -void ConPrintf(char *format, ...) -{ - va_list va; - - va_start(va, format); - (void) con_vprintf(format, va); - va_end(va); -} - -void ConBeep(void) -{ - SendMessage(hClientWnd, WM_CONBEEP, 0L, 0L); -} - -int ConReadInput(Uint32 *data, int num_chars) -{ - TCHAR *buf; - int nread; - WaitForSingleObject(console_input,INFINITE); - nread = num_chars = min(num_chars,inbuf.wrPos-inbuf.rdPos); - buf = &inbuf.data[inbuf.rdPos]; - inbuf.rdPos += nread; - while (nread--) - *data++ = *buf++; - if (inbuf.rdPos >= inbuf.wrPos) { - inbuf.rdPos = 0; - inbuf.wrPos = 0; - ResetEvent(console_input_event); - } - ReleaseSemaphore(console_input,1,NULL); - return num_chars; -} - -int ConGetKey(void) -{ - Uint32 c; - WaitForSingleObject(console_input,INFINITE); - ResetEvent(console_input_event); - inbuf.rdPos = inbuf.wrPos = 0; - ReleaseSemaphore(console_input,1,NULL); - WaitForSingleObject(console_input_event,INFINITE); - ConReadInput(&c, 1); - return (int) c; -} - -int ConGetColumns(void) -{ - return (int) canvasColumns; /* 32bit atomic on windows */ -} - -int ConGetRows(void) { - return (int) canvasRows; -} - - -static HINSTANCE hInstance; -extern HMODULE beam_module; - -static unsigned __stdcall -ConThreadInit(LPVOID param) -{ - MSG msg; - WNDCLASSEX wndclass; - int iCmdShow; - STARTUPINFO StartupInfo; - HACCEL hAccel; - int x, y, w, h; - struct title_buf title; - - /*DebugBreak();*/ -#ifdef HARDDEBUG - if(AttachConsole(ATTACH_PARENT_PROCESS) || AllocConsole()) { - freopen("CONOUT$", "w", stdout); - freopen("CONOUT$", "w", stderr); - } -#endif - - hInstance = GetModuleHandle(NULL); - StartupInfo.dwFlags = 0; - GetStartupInfo(&StartupInfo); - iCmdShow = StartupInfo.dwFlags & STARTF_USESHOWWINDOW ? - StartupInfo.wShowWindow : SW_SHOWDEFAULT; - - LoadUserPreferences(); - - /* frame window class */ - wndclass.cbSize = sizeof (wndclass); - wndclass.style = CS_HREDRAW | CS_VREDRAW | CS_BYTEALIGNCLIENT; - wndclass.lpfnWndProc = FrameWndProc; - wndclass.cbClsExtra = 0; - wndclass.cbWndExtra = 0; - wndclass.hInstance = hInstance; - wndclass.hIcon = LoadIcon (hInstance, MAKEINTRESOURCE(1)); - wndclass.hCursor = LoadCursor (NULL, IDC_ARROW); - wndclass.hbrBackground = NULL; - wndclass.lpszMenuName = NULL; - wndclass.lpszClassName = szFrameClass; - wndclass.hIconSm = LoadIcon (hInstance, MAKEINTRESOURCE(1)); - RegisterClassExW (&wndclass); - - /* client window class */ - wndclass.cbSize = sizeof (wndclass); - wndclass.style = CS_HREDRAW | CS_VREDRAW | CS_OWNDC; - wndclass.lpfnWndProc = ClientWndProc; - wndclass.cbClsExtra = 0; - wndclass.cbWndExtra = 0; - wndclass.hInstance = hInstance; - wndclass.hIcon = LoadIcon (hInstance, MAKEINTRESOURCE(1)); - wndclass.hCursor = LoadCursor (NULL, IDC_ARROW); - wndclass.hbrBackground = CreateSolidBrush(bkgColor); - wndclass.lpszMenuName = NULL; - wndclass.lpszClassName = szClientClass; - wndclass.hIconSm = LoadIcon (hInstance, MAKEINTRESOURCE(1)); - RegisterClassExW (&wndclass); - - InitCommonControls(); - init_buffers(); - - nBufLines = 0; - buffer_top = cur_line = ConNewLine(); - cur_line->next = buffer_bottom = ConNewLine(); - buffer_bottom->prev = cur_line; - - /* Create Frame Window */ - window_title(&title); - hFrameWnd = CreateWindowEx(0, szFrameClass, title.name, - WS_OVERLAPPEDWINDOW,CW_USEDEFAULT, - CW_USEDEFAULT,CW_USEDEFAULT,CW_USEDEFAULT, - NULL,LoadMenu(beam_module,MAKEINTRESOURCE(1)), - hInstance,NULL); - free_window_title(&title); - - /* XXX OTP-5522: - The window position is not saved correctly and if the window - is closed when minimized, it's not possible to start werl again - with the window open. Temporary fix so far is to ignore saved values - and always start with initial settings. */ - /* Original: if (winPos.left == -1) { */ - /* Temporary: if (1) { */ - if (1) { - - /* initial window position */ - x = 0; - y = 0; - w = cxChar*LINE_LENGTH+FRAME_WIDTH+GetSystemMetrics(SM_CXVSCROLL); - h = cyChar*30+FRAME_HEIGHT; - } else { - /* saved window position */ - x = winPos.left; - y = winPos.top; - w = winPos.right - x; - h = winPos.bottom - y; - } - SetWindowPos(hFrameWnd, NULL, x, y, w, h, SWP_NOZORDER); - - ShowWindow(hFrameWnd, iCmdShow); - UpdateWindow(hFrameWnd); - - hAccel = LoadAccelerators(beam_module,MAKEINTRESOURCE(1)); - - ReleaseSemaphore(console_input, 1, NULL); - ReleaseSemaphore(console_output, 1, NULL); - - - /* Main message loop */ - while (GetMessage (&msg, NULL, 0, 0)) - { - if (!TranslateAccelerator(hFrameWnd,hAccel,&msg)) - { - TranslateMessage (&msg); - DispatchMessage (&msg); - } - } - /* - PostQuitMessage() results in WM_QUIT which makes GetMessage() - return 0 (which stops the main loop). Before we return from - the console thread, the ctrl_handler is called to do erts_exit. - */ - (*ctrl_handler)(CTRL_CLOSE_EVENT); - return msg.wParam; -} - -static LRESULT CALLBACK -FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam) -{ - RECT r; - int cy,i,bufsize; - TCHAR c; - unsigned long l; - TCHAR buf[128]; - struct title_buf title; - - switch (iMsg) { - case WM_CREATE: - /* client window creation */ - window_title(&title); - hClientWnd = CreateWindowEx(0, szClientClass, title.name, - WS_CHILD|WS_VISIBLE|WS_VSCROLL|WS_HSCROLL, - CW_USEDEFAULT, CW_USEDEFAULT, - CW_USEDEFAULT, CW_USEDEFAULT, - hwnd, (HMENU)0, hInstance, NULL); - free_window_title(&title); - hTBWnd = InitToolBar(hwnd); - UpdateWindow (hClientWnd); - return 0; - case WM_SIZE : - if (IsWindowVisible(hTBWnd)) { - SendMessage(hTBWnd,TB_AUTOSIZE,0,0L); - GetWindowRect(hTBWnd,&r); - cy = r.bottom-r.top; - } else cy = 0; - MoveWindow(hClientWnd,0,cy,LOWORD(lParam),HIWORD(lParam)-cy,TRUE); - return 0; - case WM_ERASEBKGND: - return 1; - case WM_SETFOCUS : - CreateCaret(hClientWnd, NULL, cxChar, cyChar); - SetCaretPos(GetXFromCurrentY(GetDC(hClientWnd),iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); - ShowCaret(hClientWnd); - return 0; - case WM_KILLFOCUS: - HideCaret(hClientWnd); - DestroyCaret(); - return 0; - case WM_INITMENUPOPUP : - if (lParam == 0) /* File popup menu */ - { - EnableMenuItem((HMENU)wParam, IDMENU_STARTLOG, - logfile ? MF_GRAYED : MF_ENABLED); - EnableMenuItem((HMENU)wParam, IDMENU_STOPLOG, - logfile ? MF_ENABLED : MF_GRAYED); - return 0; - } - else if (lParam == 1) /* Edit popup menu */ - { - EnableMenuItem((HMENU)wParam, IDMENU_COPY, - fTextSelected ? MF_ENABLED : MF_GRAYED); - EnableMenuItem((HMENU)wParam, IDMENU_PASTE, - IsClipboardFormatAvailable(CF_TEXT) ? MF_ENABLED : MF_GRAYED); - return 0; - } - else if (lParam == 3) /* View popup menu */ - { - CheckMenuItem((HMENU)wParam,IDMENU_TOOLBAR, - IsWindowVisible(hTBWnd) ? MF_CHECKED : MF_UNCHECKED); - return 0; - } - break; - case WM_NOTIFY: - switch (((LPNMHDR) lParam)->code) { - case TTN_NEEDTEXT: - { - LPTOOLTIPTEXT lpttt; - lpttt = (LPTOOLTIPTEXT) lParam; - lpttt->hinst = hInstance; - /* check for combobox handle */ - if (lpttt->uFlags&TTF_IDISHWND) { - if ((lpttt->hdr.idFrom == (UINT_PTR) hComboWnd)) { - lstrcpy(lpttt->lpszText,TEXT("Command History")); - break; - } - } - /* check for toolbar buttons */ - switch (lpttt->hdr.idFrom) { - case IDMENU_COPY: - lstrcpy(lpttt->lpszText,TEXT("Copy (Ctrl+C)")); - break; - case IDMENU_PASTE: - lstrcpy(lpttt->lpszText,TEXT("Paste (Ctrl+V)")); - break; - case IDMENU_FONT: - lstrcpy(lpttt->lpszText,TEXT("Fonts")); - break; - case IDMENU_ABOUT: - lstrcpy(lpttt->lpszText,TEXT("Help")); - break; - } - } - } - break; - case WM_COMMAND: - switch(LOWORD(wParam)) - { - case IDMENU_STARTLOG: - OpenLogFile(hwnd); - return 0; - case IDMENU_STOPLOG: - CloseLogFile(hwnd); - return 0; - case IDMENU_EXIT: - SendMessage(hwnd, WM_CLOSE, 0, 0L); - return 0; - case IDMENU_COPY: - if (fTextSelected) - OnEditCopy(hClientWnd); - return 0; - case IDMENU_PASTE: - OnEditPaste(hClientWnd); - return 0; - case IDMENU_SELALL: - OnEditSelAll(hClientWnd); - return 0; - case IDMENU_FONT: - if (ConChooseFont(hClientWnd)) { - ConSetFont(hClientWnd); - } - SaveUserPreferences(); - return 0; - case IDMENU_SELECTBKG: - ConChooseColor(hClientWnd); - SaveUserPreferences(); - return 0; - case IDMENU_TOOLBAR: - if (toolbarVisible) { - ShowWindow(hTBWnd,SW_HIDE); - toolbarVisible = FALSE; - } else { - ShowWindow(hTBWnd,SW_SHOW); - toolbarVisible = TRUE; - } - GetClientRect(hwnd,&r); - PostMessage(hwnd,WM_SIZE,0,MAKELPARAM(r.right,r.bottom)); - return 0; - case IDMENU_ABOUT: - DialogBox(beam_module,TEXT("AboutBox"),hwnd,AboutDlgProc); - return 0; - case ID_COMBOBOX: - switch (HIWORD(wParam)) { - case CBN_SELENDOK: - i = SendMessage(hComboWnd,CB_GETCURSEL,0,0); - if (i != CB_ERR) { - buf[0] = 0x01; /* CTRL+A */ - buf[1] = 0x0B; /* CTRL+K */ - bufsize = SendMessage(hComboWnd,CB_GETLBTEXT,i,(LPARAM)&buf[2]); - if (bufsize != CB_ERR) - write_inbuf(buf,bufsize+2); - SetFocus(hwnd); - } - break; - case CBN_SELENDCANCEL: - break; - } - break; - case ID_BREAK: /* CTRL+BRK */ - /* pass on break char if the ctrl_handler is disabled */ - if ((*ctrl_handler)(CTRL_C_EVENT) == FALSE) { - c = 0x03; - write_inbuf(&c,1); - } - return 0; - } - break; - case WM_KEYDOWN : - switch (wParam) { - case VK_UP: c = 'P'-'@'; break; - case VK_DOWN : c = 'N'-'@'; break; - case VK_RIGHT : c = 'F'-'@'; break; - case VK_LEFT : c = 'B'-'@'; break; - case VK_DELETE : c = 'D' -'@'; break; - case VK_HOME : c = 'A'-'@'; break; - case VK_END : c = 'E'-'@'; break; - case VK_RETURN : AddToCmdHistory(); return 0; - case VK_PRIOR : /* PageUp */ - PostMessage(hClientWnd, WM_VSCROLL, SB_PAGEUP, 0); - return 0; - case VK_NEXT : /* PageDown */ - PostMessage(hClientWnd, WM_VSCROLL, SB_PAGEDOWN, 0); - return 0; - default: return 0; - } - write_inbuf(&c, 1); - return 0; - case WM_MOUSEWHEEL: - { - int delta = GET_WHEEL_DELTA_WPARAM(wParam); - if (delta < 0) { - PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK, - (iVscrollPos + 5)),0); - } else { - WORD pos = ((iVscrollPos - 5) < 0) ? 0 : (iVscrollPos - 5); - PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,pos),0); - } - return 0; - } - case WM_CHAR: - c = (TCHAR)wParam; - write_inbuf(&c,1); - return 0; - case WM_CLOSE : - break; - case WM_DESTROY : - SaveUserPreferences(); - destroyed = TRUE; - PostQuitMessage(0); - return 0; - case WM_SAVE_PREFS : - SaveUserPreferences(); - return 0; - } - return DefWindowProc(hwnd, iMsg, wParam, lParam); -} - -static BOOL -Client_OnCreate(HWND hwnd, LPCREATESTRUCT lpCreateStruct) -{ - ConFontInitialize(hwnd); - cur_x = cur_y = 0; - iVscrollPos = 0; - iHscrollPos = 0; - return TRUE; -} - -static void -Client_OnPaint(HWND hwnd) -{ - ScreenLine_t *pLine; - int x,y,i,iTop,iBot; - PAINTSTRUCT ps; - RECT rcInvalid; - HDC hdc; - - hdc = BeginPaint(hwnd, &ps); - rcInvalid = ps.rcPaint; - hdc = ps.hdc; - iTop = max(0, iVscrollPos + rcInvalid.top/cyChar); - iBot = min(nBufLines, iVscrollPos + rcInvalid.bottom/cyChar+1); - pLine = GetLineFromY(iTop); - for (i = iTop; i < iBot && pLine != NULL; i++) { - y = cyChar*(i-iVscrollPos); - x = -cxChar*iHscrollPos; - TextOut(hdc, x, y, &pLine->text[0], pLine->width); - pLine = pLine->next; - } - if (fTextSelected || fSelecting) { - InvertSelectionArea(hwnd); - } - SetCaretPos(GetXFromCurrentY(hdc,iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); - EndPaint(hwnd, &ps); -} -#ifdef HARDDEBUG -static void dump_linebufs(void) { - char *buff; - ScreenLine_t *s = buffer_top; - fprintf(stderr,"LinebufDump------------------------\n"); - while(s) { - if (s == buffer_top) fprintf(stderr,"BT-> "); - if (s == buffer_bottom) fprintf(stderr,"BB-> "); - if (s == cur_line) fprintf(stderr,"CL-> "); - - buff = (char *) ALLOC(s->width+1); - memcpy(buff,s->text,s->width); - buff[s->width] = '\0'; - fprintf(stderr,"{\"%s\",%d,%d}\n",buff,s->newline,s->allocated); - FREE(buff); - s = s->next; - } - fprintf(stderr,"LinebufDumpEnd---------------------\n"); - fflush(stderr); -} -#endif - -static void reorganize_linebufs(HWND hwnd) { - ScreenLine_t *otop = buffer_top; - ScreenLine_t *obot = buffer_bottom; - ScreenLine_t *next; - int i,cpos; - - cpos = 0; - i = nBufLines - cur_y; - while (i > 1) { - cpos += obot->width; - obot = obot->prev; - i--; - } - cpos += (obot->width - cur_x); -#ifdef HARDDEBUG - fprintf(stderr,"nBufLines = %d, cur_x = %d, cur_y = %d, cpos = %d\n", - nBufLines,cur_x,cur_y,cpos); - fflush(stderr); -#endif - - - nBufLines = 0; - buffer_top = cur_line = ConNewLine(); - cur_line->next = buffer_bottom = ConNewLine(); - buffer_bottom->prev = cur_line; - - cur_x = cur_y = 0; - iVscrollPos = 0; - iHscrollPos = 0; - - while(otop) { - for(i=0;i<otop->width;++i) { - cur_line->text[cur_x] = otop->text[i]; - cur_x++; - if (cur_x > cur_line->width) - cur_line->width = cur_x; - if (GetXFromCurrentY(GetDC(hwnd),0,cur_x) + cxChar > - (LINE_LENGTH * cxChar)) { - ConCarriageFeed(0); - } - } - if (otop->newline) { - ConCarriageFeed(1); - /*ConScrollScreen();*/ - } - next = otop->next; - FREE(otop->text); - FREE(otop); - otop = next; - } - while (cpos) { - cur_x--; - if (cur_x < 0) { - cur_y--; - cur_line = cur_line->prev; - cur_x = cur_line->width-1; - } - cpos--; - } - SetCaretPos(GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); -#ifdef HARDDEBUG - fprintf(stderr,"canvasColumns = %d,nBufLines = %d, cur_x = %d, cur_y = %d\n", - canvasColumns,nBufLines,cur_x,cur_y); - fflush(stderr); -#endif -} - - -static void -Client_OnSize(HWND hwnd, UINT state, int cx, int cy) -{ - RECT r; - SCROLLBARINFO sbi; - int w,h,columns; - int scrollheight; - cxClient = cx; - cyClient = cy; - set_scroll_info(hwnd); - GetClientRect(hwnd,&r); - w = r.right - r.left; - h = r.bottom - r.top; - sbi.cbSize = sizeof(SCROLLBARINFO); - if (!GetScrollBarInfo(hwnd, OBJID_HSCROLL,&sbi) || - (sbi.rgstate[0] & STATE_SYSTEM_INVISIBLE)) { - scrollheight = 0; - } else { - scrollheight = sbi.rcScrollBar.bottom - sbi.rcScrollBar.top; - } - canvasRows = (h - scrollheight) / cyChar; - if (canvasRows < DEF_CANVAS_ROWS) { - canvasRows = DEF_CANVAS_ROWS; - } - columns = (w - GetSystemMetrics(SM_CXVSCROLL)) /cxChar; - if (columns < DEF_CANVAS_COLUMNS) - columns = DEF_CANVAS_COLUMNS; - if (columns != canvasColumns) { - canvasColumns = columns; - /*dump_linebufs();*/ - reorganize_linebufs(hwnd); - fSelecting = fTextSelected = FALSE; - InvalidateRect(hwnd, NULL, TRUE); -#ifdef HARDDEBUG - fprintf(stderr,"Paint: cols = %d, rows = %d\n",canvasColumns,canvasRows); - fflush(stderr); -#endif - } - - SetCaretPos(GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); -} - -static void calc_charpoint_from_point(HDC dc, int x, int y, int y_offset, POINT *pt) -{ - int r; - int hscrollPix = iHscrollPos * cxChar; - - pt->y = y/cyChar + iVscrollPos + y_offset; - - if (x > (LINE_LENGTH-iHscrollPos) * cxChar) { - x = (LINE_LENGTH-iHscrollPos) * cxChar; - } - if (pt->y - y_offset > 0 && GetLineFromY(pt->y - y_offset) == NULL) { - pt->y = nBufLines - 1 + y_offset; - pt->x = GetLineFromY(pt->y - y_offset)->width; - } else { - for (pt->x = 1; - (r = GetXFromLine(dc, 0, pt->x, GetLineFromY(pt->y - y_offset))) != 0 && - (r - hscrollPix) < x; - ++(pt->x)) - ; - if ((r - hscrollPix) > x) - --(pt->x); -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"pt->x = %d, iHscrollPos = %d\n",(int) pt->x, iHscrollPos); - fflush(stderr); -#endif - if (pt->x <= 0) { - pt->x = x/cxChar + iHscrollPos; - } - } -} - - -static void -Client_OnLButtonDown(HWND hwnd, BOOL fDoubleClick, int x, int y, UINT keyFlags) -{ - int r; - SetFocus(GetParent(hwnd)); /* In case combobox steals the focus */ -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"OnLButtonDown fSelecting = %d, fTextSelected = %d:\n", - fSelecting,fTextSelected); - fflush(stderr); -#endif - if (fTextSelected) { - InvertSelectionArea(hwnd); - } - fTextSelected = FALSE; - - calc_charpoint_from_point(GetDC(hwnd), x, y, 0, &editBeg); - - editEnd.x = editBeg.x; - editEnd.y = editBeg.y + 1; - fSelecting = TRUE; - SetCapture(hwnd); -} - -static void -Client_OnRButtonDown(HWND hwnd, BOOL fDoubleClick, int x, int y, UINT keyFlags) -{ - if (fTextSelected) { - fSelecting = TRUE; - Client_OnMouseMove(hwnd,x,y,keyFlags); - fSelecting = FALSE; - } -} - -static void -Client_OnLButtonUp(HWND hwnd, int x, int y, UINT keyFlags) -{ -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"OnLButtonUp fSelecting = %d, fTextSelected = %d:\n", - fSelecting,fTextSelected); - fprintf(stderr,"(Beg.x = %d, Beg.y = %d, " - "End.x = %d, End.y = %d)\n",editBeg.x,editBeg.y, - editEnd.x,editEnd.y); -#endif - if (fSelecting && - !(editBeg.x == editEnd.x && editBeg.y == (editEnd.y - 1))) { - fTextSelected = TRUE; - } -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"OnLButtonUp fTextSelected = %d:\n", - fTextSelected); - fflush(stderr); -#endif - fSelecting = FALSE; - ReleaseCapture(); -} - -#define EMPTY_RECT(R) \ -(((R).bottom - (R).top == 0) || ((R).right - (R).left == 0)) -#define ABS(X) (((X)< 0) ? -1 * (X) : X) -#define DIFF(A,B) ABS(((int)(A)) - ((int)(B))) - -static int diff_sel_area(RECT old[3], RECT new[3], RECT result[6]) -{ - int absposold = old[0].left + old[0].top * canvasColumns; - int absposnew = new[0].left + new[0].top * canvasColumns; - int absendold = absposold, absendnew = absposnew; - int i, x, ret = 0; - int abspos[2],absend[2]; - for(i = 0; i < 3; ++i) { - if (!EMPTY_RECT(old[i])) { - absendold += (old[i].right - old[i].left) * - (old[i].bottom - old[i].top); - } - if (!EMPTY_RECT(new[i])) { - absendnew += (new[i].right - new[i].left) * - (new[i].bottom - new[i].top); - } - } - abspos[0] = min(absposold, absposnew); - absend[0] = DIFF(absposold, absposnew) + abspos[0]; - abspos[1] = min(absendold, absendnew); - absend[1] = DIFF(absendold, absendnew) + abspos[1]; -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"abspos[0] = %d, absend[0] = %d, abspos[1] = %d, absend[1] = %d\n",abspos[0],absend[0],abspos[1],absend[1]); - fflush(stderr); -#endif - i = 0; - for (x = 0; x < 2; ++x) { - if (abspos[x] != absend[x]) { - int consumed = 0; - result[i].left = abspos[x] % canvasColumns; - result[i].top = abspos[x] / canvasColumns; - result[i].bottom = result[i].top + 1; - if ((absend[x] - abspos[x]) + result[i].left < canvasColumns) { -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"Nowrap, %d < canvasColumns\n", - (absend[x] - abspos[x]) + result[i].left); - fflush(stderr); -#endif - result[i].right = (absend[x] - abspos[x]) + result[i].left; - consumed += result[i].right - result[i].left; - } else { -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"Wrap, %d >= canvasColumns\n", - (absend[x] - abspos[x]) + result[i].left); - fflush(stderr); -#endif - result[i].right = canvasColumns; - consumed += result[i].right - result[i].left; - if (absend[x] - abspos[x] - consumed >= canvasColumns) { - ++i; - result[i].top = result[i-1].bottom; - result[i].left = 0; - result[i].right = canvasColumns; - result[i].bottom = (absend[x] - abspos[x] - consumed) / canvasColumns + result[i].top; - consumed += (result[i].bottom - result[i].top) * canvasColumns; - } - if (absend[x] - abspos[x] - consumed > 0) { - ++i; - result[i].top = result[i-1].bottom; - result[i].bottom = result[i].top + 1; - result[i].left = 0; - result[i].right = absend[x] - abspos[x] - consumed; - } - } - ++i; - } - } -#ifdef HARD_SEL_DEBUG - if (i > 2) { - int x; - fprintf(stderr,"i = %d\n",i); - fflush(stderr); - for (x = 0; x < i; ++x) { - fprintf(stderr, "result[%d]: top = %d, left = %d, " - "bottom = %d. right = %d\n", - x, result[x].top, result[x].left, - result[x].bottom, result[x].right); - } - } -#endif - return i; -} - - - -static void calc_sel_area(RECT rects[3], POINT beg, POINT end) -{ - /* These are not really rects and points, these are character - based positions, need to be multiplied by cxChar and cyChar to - make up canvas coordinates */ - memset(rects,0,3*sizeof(RECT)); - rects[0].left = beg.x; - rects[0].top = beg.y; - rects[0].bottom = beg.y+1; - if (end.y - beg.y == 1) { /* Only one row */ - rects[0].right = end.x; - goto out; - } - rects[0].right = canvasColumns; - if (end.y - beg.y > 2) { - rects[1].left = 0; - rects[1].top = rects[0].bottom; - rects[1].right = canvasColumns; - rects[1].bottom = end.y - 1; - } - rects[2].left = 0; - rects[2].top = end.y - 1; - rects[2].bottom = end.y; - rects[2].right = end.x; - - out: -#ifdef HARD_SEL_DEBUG - { - int i; - fprintf(stderr,"beg.x = %d, beg.y = %d, end.x = %d, end.y = %d\n", - beg.x,beg.y,end.x,end.y); - for (i = 0; i < 3; ++i) { - fprintf(stderr,"[%d] left = %d, top = %d, " - "right = %d, bottom = %d\n", - i, rects[i].left, rects[i].top, - rects[i].right, rects[i].bottom); - } - fflush(stderr); - } -#endif - return; -} - -static void calc_sel_area_turned(RECT rects[3], POINT eBeg, POINT eEnd) { - POINT from,to; - if (eBeg.y >= eEnd.y || - (eBeg.y == eEnd.y - 1 && eBeg.x > eEnd.x)) { -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"Reverting (Beg.x = %d, Beg.y = %d, " - "End.x = %d, End.y = %d)\n",eBeg.x,eBeg.y, - eEnd.x,eEnd.y); - fflush(stderr); -#endif - from.x = eEnd.x; - from.y = eEnd.y - 1; - to.x = eBeg.x; - to.y = eBeg.y + 1; - calc_sel_area(rects,from,to); - } else { - calc_sel_area(rects,eBeg,eEnd); - } -} - - -static void InvertSelectionArea(HWND hwnd) -{ - RECT rects[3]; - POINT from,to; - int i; - calc_sel_area_turned(rects,editBeg,editEnd); - for (i = 0; i < 3; ++i) { - if (!EMPTY_RECT(rects[i])) { - from.x = rects[i].left; - to.x = rects[i].right; - from.y = rects[i].top; - to.y = rects[i].bottom; - DrawSelection(hwnd,from,to); - } - } -} - -static void -Client_OnMouseMove(HWND hwnd, int x, int y, UINT keyFlags) -{ - if (fSelecting) { - RECT rold[3], rnew[3], rupdate[6]; - int num_updates,i,r; - POINT from,to; - calc_sel_area_turned(rold,editBeg,editEnd); - - calc_charpoint_from_point(GetDC(hwnd), x, y, 1, &editEnd); - - calc_sel_area_turned(rnew,editBeg,editEnd); - num_updates = diff_sel_area(rold,rnew,rupdate); - for (i = 0; i < num_updates;++i) { - from.x = rupdate[i].left; - to.x = rupdate[i].right; - from.y = rupdate[i].top; - to.y = rupdate[i].bottom; -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"from: x=%d,y=%d, to: x=%d, y=%d\n", - from.x, from.y,to.x,to.y); - fflush(stderr); -#endif - DrawSelection(hwnd,from,to); - } - } -} - -static void -Client_OnVScroll(HWND hwnd, HWND hwndCtl, UINT code, int pos) -{ - int iVscroll; - - switch(code) { - case SB_LINEDOWN: - iVscroll = 1; - break; - case SB_LINEUP: - iVscroll = -1; - break; - case SB_PAGEDOWN: - iVscroll = max(1, cyClient/cyChar); - break; - case SB_PAGEUP: - iVscroll = min(-1, -cyClient/cyChar); - break; - case SB_THUMBTRACK: - iVscroll = pos - iVscrollPos; - break; - default: - iVscroll = 0; - } - iVscroll = max(-iVscrollPos, min(iVscroll, iVscrollMax-iVscrollPos)); - if (iVscroll != 0) { - iVscrollPos += iVscroll; - ScrollWindowEx(hwnd, 0, -cyChar*iVscroll, NULL, NULL, - NULL, NULL, SW_ERASE | SW_INVALIDATE); - SetScrollPos(hwnd, SB_VERT, iVscrollPos, TRUE); - iVscroll = GetScrollPos(hwnd, SB_VERT); - UpdateWindow(hwnd); - } -} - -static void -Client_OnHScroll(HWND hwnd, HWND hwndCtl, UINT code, int pos) -{ - int iHscroll, curCharWidth = cxClient/cxChar; - - switch(code) { - case SB_LINEDOWN: - iHscroll = 1; - break; - case SB_LINEUP: - iHscroll = -1; - break; - case SB_PAGEDOWN: - iHscroll = max(1,curCharWidth-1); - break; - case SB_PAGEUP: - iHscroll = min(-1,-(curCharWidth-1)); - break; - case SB_THUMBTRACK: - iHscroll = pos - iHscrollPos; - break; - default: - iHscroll = 0; - } - iHscroll = max(-iHscrollPos, min(iHscroll, iHscrollMax-iHscrollPos-(curCharWidth-1))); - if (iHscroll != 0) { - iHscrollPos += iHscroll; - ScrollWindow(hwnd, -cxChar*iHscroll, 0, NULL, NULL); - SetScrollPos(hwnd, SB_HORZ, iHscrollPos, TRUE); - UpdateWindow(hwnd); - } -} - -static LRESULT CALLBACK -ClientWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam) -{ - switch (iMsg) { - HANDLE_MSG(hwnd, WM_CREATE, Client_OnCreate); - HANDLE_MSG(hwnd, WM_SIZE, Client_OnSize); - HANDLE_MSG(hwnd, WM_PAINT, Client_OnPaint); - HANDLE_MSG(hwnd, WM_LBUTTONDOWN, Client_OnLButtonDown); - HANDLE_MSG(hwnd, WM_RBUTTONDOWN, Client_OnRButtonDown); - HANDLE_MSG(hwnd, WM_LBUTTONUP, Client_OnLButtonUp); - HANDLE_MSG(hwnd, WM_MOUSEMOVE, Client_OnMouseMove); - HANDLE_MSG(hwnd, WM_VSCROLL, Client_OnVScroll); - HANDLE_MSG(hwnd, WM_HSCROLL, Client_OnHScroll); - case WM_CONBEEP: - if (0) Beep(440, 400); - return 0; - case WM_CONTEXT: - ConDrawText(hwnd); - return 0; - case WM_CLOSE: - break; - case WM_DESTROY: - PostQuitMessage(0); - return 0; - } - return DefWindowProc (hwnd, iMsg, wParam, lParam); -} - -static void -LoadUserPreferences(void) -{ - DWORD size; - DWORD res; - DWORD type; - HFONT hfont; - /* default prefs */ - hfont = CreateFont(0,0, 0,0, 0, FALSE,FALSE,FALSE, - ANSI_CHARSET, OUT_TT_ONLY_PRECIS, CLIP_DEFAULT_PRECIS, - CLEARTYPE_QUALITY, FIXED_PITCH, TEXT("Consolas")); - if(hfont) { - GetObject(hfont, sizeof(LOGFONT), (PSTR)&logfont); - DeleteObject(hfont); - } else { - GetObject(GetStockObject(SYSTEM_FIXED_FONT),sizeof(LOGFONT),(PSTR)&logfont); - } - fgColor = GetSysColor(COLOR_WINDOWTEXT); - bkgColor = GetSysColor(COLOR_WINDOW); - winPos.left = -1; - toolbarVisible = FALSE; - - if (RegCreateKeyEx(HKEY_CURRENT_USER, USER_KEY, 0, 0, - REG_OPTION_NON_VOLATILE, KEY_ALL_ACCESS, NULL, - &key, &res) != ERROR_SUCCESS) - return; - has_key = TRUE; - if (res == REG_CREATED_NEW_KEY) - return; - size = sizeof(logfont); - res = RegQueryValueEx(key,TEXT("Font"),NULL,&type,(LPBYTE)&logfont,&size); - size = sizeof(fgColor); - res = RegQueryValueEx(key,TEXT("FgColor"),NULL,&type,(LPBYTE)&fgColor,&size); - size = sizeof(bkgColor); - res = RegQueryValueEx(key,TEXT("BkColor"),NULL,&type,(LPBYTE)&bkgColor,&size); - size = sizeof(winPos); - res = RegQueryValueEx(key,TEXT("Pos"),NULL,&type,(LPBYTE)&winPos,&size); - size = sizeof(toolbarVisible); - res = RegQueryValueEx(key,TEXT("Toolbar"),NULL,&type,(LPBYTE)&toolbarVisible,&size); -} - -static void -SaveUserPreferences(void) -{ - WINDOWPLACEMENT wndPlace; - - if (has_key == TRUE) { - RegSetValueEx(key,TEXT("Font"),0,REG_BINARY,(CONST BYTE *)&logfont,sizeof(LOGFONT)); - RegSetValueEx(key,TEXT("FgColor"),0,REG_DWORD,(CONST BYTE *)&fgColor,sizeof(fgColor)); - RegSetValueEx(key,TEXT("BkColor"),0,REG_DWORD,(CONST BYTE *)&bkgColor,sizeof(bkgColor)); - RegSetValueEx(key,TEXT("Toolbar"),0,REG_DWORD,(CONST BYTE *)&toolbarVisible,sizeof(toolbarVisible)); - - wndPlace.length = sizeof(WINDOWPLACEMENT); - GetWindowPlacement(hFrameWnd,&wndPlace); - /* If wndPlace.showCmd == SW_MINIMIZE, then the window is minimized. - We don't care, wndPlace.rcNormalPosition always holds the last known position. */ - winPos = wndPlace.rcNormalPosition; - RegSetValueEx(key,TEXT("Pos"),0,REG_BINARY,(CONST BYTE *)&winPos,sizeof(winPos)); - } -} - - -static void -set_scroll_info(HWND hwnd) -{ - SCROLLINFO info; - int hScrollBy; - /* - * Set vertical scrolling range and scroll box position. - */ - - iVscrollMax = nBufLines-1; - iVscrollPos = min(iVscrollPos, iVscrollMax); - info.cbSize = sizeof(info); - info.fMask = SIF_PAGE|SIF_RANGE|SIF_POS; - info.nMin = 0; - info.nPos = iVscrollPos; - info.nPage = min(cyClient/cyChar, iVscrollMax); - info.nMax = iVscrollMax; - SetScrollInfo(hwnd, SB_VERT, &info, TRUE); - - /* - * Set horizontal scrolling range and scroll box position. - */ - - iHscrollMax = LINE_LENGTH-1; - hScrollBy = max(0, (iHscrollPos - (iHscrollMax-cxClient/cxChar))*cxChar); - iHscrollPos = min(iHscrollPos, iHscrollMax); - info.nPos = iHscrollPos; - info.nPage = cxClient/cxChar; - info.nMax = iHscrollMax; - SetScrollInfo(hwnd, SB_HORZ, &info, TRUE); - /*ScrollWindow(hwnd, hScrollBy, 0, NULL, NULL);*/ -} - - -static void -ensure_line_below(void) -{ - if (cur_line->next == NULL) { - if (nBufLines >= lines_to_save) { - ScreenLine_t* pLine = buffer_top->next; - FREE(buffer_top->text); - FREE(buffer_top); - buffer_top = pLine; - buffer_top->prev = NULL; - nBufLines--; - } - cur_line->next = ConNewLine(); - cur_line->next->prev = cur_line; - buffer_bottom = cur_line->next; - set_scroll_info(hClientWnd); - } -} - -static ScreenLine_t* -ConNewLine(void) -{ - ScreenLine_t *pLine; - - pLine = (ScreenLine_t *)ALLOC(sizeof(ScreenLine_t)); - if (!pLine) - return NULL; - pLine->text = (TCHAR *) ALLOC(canvasColumns * sizeof(TCHAR)); -#ifdef HARDDEBUG - pLine->allocated = canvasColumns; -#endif - pLine->width = 0; - pLine->prev = pLine->next = NULL; - pLine->newline = 0; - nBufLines++; - return pLine; -} - -static ScreenLine_t* -GetLineFromY(int y) -{ - ScreenLine_t *pLine = buffer_top; - int i; - - for (i = 0; i < nBufLines && pLine != NULL; i++) { - if (i == y) - return pLine; - pLine = pLine->next; - } - return NULL; -} - -void ConCarriageFeed(int hard_newline) -{ - cur_x = 0; - ensure_line_below(); - cur_line->newline = hard_newline; - cur_line = cur_line->next; - if (cur_y < nBufLines-1) { - cur_y++; - } else if (iVscrollPos > 0) { - iVscrollPos--; - } -} - -/* - * Scroll screen if cursor is not visible. - */ -static void -ConScrollScreen(void) -{ - if (cur_y >= iVscrollPos + cyClient/cyChar) { - int iVscroll; - - iVscroll = cur_y - iVscrollPos - cyClient/cyChar + 1; - iVscrollPos += iVscroll; - ScrollWindowEx(hClientWnd, 0, -cyChar*iVscroll, NULL, NULL, - NULL, NULL, SW_ERASE | SW_INVALIDATE); - SetScrollPos(hClientWnd, SB_VERT, iVscrollPos, TRUE); - UpdateWindow(hClientWnd); - } -} - -static void -DrawSelection(HWND hwnd, POINT pt1, POINT pt2) -{ - HDC hdc; - int width,height; -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"pt1.x = %d, pt1.y = %d, pt2.x = %d, pt2.y = %d\n", - (int) pt1.x, (int) pt1.y, (int) pt2.x, (int) pt2.y); -#endif - pt1.x = GetXFromLine(GetDC(hwnd),iHscrollPos,pt1.x,GetLineFromY(pt1.y)); - pt2.x = GetXFromLine(GetDC(hwnd),iHscrollPos,pt2.x,GetLineFromY(pt2.y-1)); - pt1.y -= iVscrollPos; - pt2.y -= iVscrollPos; - pt1.y *= cyChar; - pt2.y *= cyChar; -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"pt1.x = %d, pt1.y = %d, pt2.x = %d, pt2.y = %d\n", - (int) pt1.x, (int) pt1.y, (int) pt2.x, (int) pt2.y); - fflush(stderr); -#endif - width = pt2.x-pt1.x; - height = pt2.y - pt1.y; - hdc = GetDC(hwnd); - PatBlt(hdc,pt1.x,pt1.y,width,height,DSTINVERT); - ReleaseDC(hwnd,hdc); -} - -static void -OnEditCopy(HWND hwnd) -{ - HGLOBAL hMem; - TCHAR *pMem; - ScreenLine_t *pLine; - RECT rects[3]; - POINT from,to; - int i,j,sum,len; - if (editBeg.y >= editEnd.y || - (editBeg.y == editEnd.y - 1 && editBeg.x > editEnd.x)) { -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"CopyReverting (Beg.x = %d, Beg.y = %d, " - "End.x = %d, End.y = %d)\n",editBeg.x,editBeg.y, - editEnd.x,editEnd.y); - fflush(stderr); -#endif - from.x = editEnd.x; - from.y = editEnd.y - 1; - to.x = editBeg.x; - to.y = editBeg.y + 1; - calc_sel_area(rects,from,to); - } else { - calc_sel_area(rects,editBeg,editEnd); - } - sum = 1; - for (i = 0; i < 3; ++i) { - if (!EMPTY_RECT(rects[i])) { - pLine = GetLineFromY(rects[i].top); - for (j = rects[i].top; j < rects[i].bottom ;++j) { - if (pLine == NULL) { - sum += 2; - break; - } - if (pLine->width > rects[i].left) { - sum += (pLine->width < rects[i].right) ? - pLine->width - rects[i].left : - rects[i].right - rects[i].left; - } - if(pLine->newline && rects[i].right >= pLine->width) { - sum += 2; - } - pLine = pLine->next; - } - } - } -#ifdef HARD_SEL_DEBUG - fprintf(stderr,"sum = %d\n",sum); - fflush(stderr); -#endif - hMem = GlobalAlloc(GHND, sum * sizeof(TCHAR)); - pMem = GlobalLock(hMem); - for (i = 0; i < 3; ++i) { - if (!EMPTY_RECT(rects[i])) { - pLine = GetLineFromY(rects[i].top); - for (j = rects[i].top; j < rects[i].bottom; ++j) { - if (pLine == NULL) { - memcpy(pMem,TEXT("\r\n"),2 * sizeof(TCHAR)); - pMem += 2; - break; - } - if (pLine->width > rects[i].left) { - len = (pLine->width < rects[i].right) ? - pLine->width - rects[i].left : - rects[i].right - rects[i].left; - memcpy(pMem,pLine->text + rects[i].left,len * sizeof(TCHAR)); - pMem +=len; - } - if(pLine->newline && rects[i].right >= pLine->width) { - memcpy(pMem,TEXT("\r\n"),2 * sizeof(TCHAR)); - pMem += 2; - } - pLine = pLine->next; - } - } - } - *pMem = TEXT('\0'); - /* Flash de selection area to give user feedback about copying */ - InvertSelectionArea(hwnd); - Sleep(100); - InvertSelectionArea(hwnd); - - OpenClipboard(hwnd); - EmptyClipboard(); - GlobalUnlock(hMem); - SetClipboardData(CF_UNICODETEXT,hMem); - CloseClipboard(); -} - -/* XXX:PaN Tchar or char? */ -static void -OnEditPaste(HWND hwnd) -{ - HANDLE hClipMem; - TCHAR *pClipMem,*pMem,*pMem2; - if (!OpenClipboard(hwnd)) - return; - if ((hClipMem = GetClipboardData(CF_UNICODETEXT)) != NULL) { - pClipMem = GlobalLock(hClipMem); - pMem = (TCHAR *)ALLOC(GlobalSize(hClipMem) * sizeof(TCHAR)); - pMem2 = pMem; - while ((*pMem2 = *pClipMem) != TEXT('\0')) { - if (*pClipMem == TEXT('\r')) - *pMem2 = TEXT('\n'); - ++pMem2; - ++pClipMem; - } - GlobalUnlock(hClipMem); - write_inbuf(pMem, _tcsclen(pMem)); - } - CloseClipboard(); -} - -static void -OnEditSelAll(HWND hwnd) -{ - editBeg.x = 0; - editBeg.y = 0; - editEnd.x = LINE_LENGTH-1; - editEnd.y = cur_y; - fTextSelected = TRUE; - InvalidateRect(hwnd, NULL, TRUE); -} - -CF_HOOK_RET APIENTRY CFHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam) -{ - /* Hook procedure for font dialog box */ - HWND hOwner; - RECT rc,rcOwner,rcDlg; - switch (iMsg) { - case WM_INITDIALOG: - /* center dialogbox within its owner window */ - if ((hOwner = GetParent(hDlg)) == NULL) - hOwner = GetDesktopWindow(); - GetWindowRect(hOwner, &rcOwner); - GetWindowRect(hDlg, &rcDlg); - CopyRect(&rc, &rcOwner); - OffsetRect(&rcDlg, -rcDlg.left, -rcDlg.top); - OffsetRect(&rc, -rc.left, -rc.top); - OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); - SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), - rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); - return (CF_HOOK_RET) 1; - default: - break; - } - return (CF_HOOK_RET) 0; /* Let the default procedure process the message */ -} - -static BOOL -ConChooseFont(HWND hwnd) -{ - HDC hdc; - hdc = GetDC(hwnd); - cf.lStructSize = sizeof(CHOOSEFONT); - cf.hwndOwner = hwnd; - cf.hDC = NULL; - cf.lpLogFont = &logfont; - cf.iPointSize = 0; - cf.Flags = CF_INITTOLOGFONTSTRUCT|CF_SCREENFONTS|CF_FIXEDPITCHONLY|CF_EFFECTS|CF_ENABLEHOOK; - cf.rgbColors = GetTextColor(hdc); - cf.lCustData = 0L; - cf.lpfnHook = CFHookProc; - cf.lpTemplateName = NULL; - cf.hInstance = NULL; - cf.lpszStyle = NULL; - cf.nFontType = 0; - cf.nSizeMin = 0; - cf.nSizeMax = 0; - ReleaseDC(hwnd,hdc); - return ChooseFont(&cf); -} - -static void -ConFontInitialize(HWND hwnd) -{ - HDC hdc; - TEXTMETRIC tm; - HFONT hFont; - - hFont = CreateFontIndirect(&logfont); - hdc = GetDC(hwnd); - SelectObject(hdc, hFont); - SetTextColor(hdc,fgColor); - SetBkColor(hdc,bkgColor); - GetTextMetrics(hdc, &tm); - cxChar = tm.tmAveCharWidth; - cxCharMax = tm.tmMaxCharWidth; - cyChar = tm.tmHeight + tm.tmExternalLeading; - ReleaseDC(hwnd, hdc); -} - -static void -ConSetFont(HWND hwnd) -{ - HDC hdc; - TEXTMETRIC tm; - HFONT hFontNew; - - hFontNew = CreateFontIndirect(&logfont); - SendMessage(hComboWnd,WM_SETFONT,(WPARAM)hFontNew, - MAKELPARAM(1,0)); - hdc = GetDC(hwnd); - DeleteObject(SelectObject(hdc, hFontNew)); - GetTextMetrics(hdc, &tm); - cxChar = tm.tmAveCharWidth; - cxCharMax = tm.tmMaxCharWidth; - cyChar = tm.tmHeight + tm.tmExternalLeading; - fgColor = cf.rgbColors; - SetTextColor(hdc,fgColor); - ReleaseDC(hwnd, hdc); - set_scroll_info(hwnd); - HideCaret(hwnd); - if (DestroyCaret()) { - CreateCaret(hwnd, NULL, cxChar, cyChar); - SetCaretPos(GetXFromCurrentY(hdc,iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); - } - ShowCaret(hwnd); - InvalidateRect(hwnd, NULL, TRUE); -} - -CC_HOOK_RET APIENTRY -CCHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam) -{ - /* Hook procedure for choose color dialog box */ - HWND hOwner; - RECT rc,rcOwner,rcDlg; - switch (iMsg) { - case WM_INITDIALOG: - /* center dialogbox within its owner window */ - if ((hOwner = GetParent(hDlg)) == NULL) - hOwner = GetDesktopWindow(); - GetWindowRect(hOwner, &rcOwner); - GetWindowRect(hDlg, &rcDlg); - CopyRect(&rc, &rcOwner); - OffsetRect(&rcDlg, -rcDlg.left, -rcDlg.top); - OffsetRect(&rc, -rc.left, -rc.top); - OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); - SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), - rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); - return (CC_HOOK_RET) 1; - default: - break; - } - return (CC_HOOK_RET) 0; /* Let the default procedure process the message */ -} - -void ConChooseColor(HWND hwnd) -{ - CHOOSECOLOR cc; - static COLORREF acrCustClr[16]; - HBRUSH hbrush; - HDC hdc; - - /* Initialize CHOOSECOLOR */ - ZeroMemory(&cc, sizeof(CHOOSECOLOR)); - cc.lStructSize = sizeof(CHOOSECOLOR); - cc.hwndOwner = hwnd; - cc.lpCustColors = (LPDWORD) acrCustClr; - cc.rgbResult = bkgColor; - cc.lpfnHook = CCHookProc; - cc.Flags = CC_FULLOPEN|CC_RGBINIT|CC_SOLIDCOLOR|CC_ENABLEHOOK; - - if (ChooseColor(&cc)==TRUE) { - bkgColor = cc.rgbResult; - hdc = GetDC(hwnd); - SetBkColor(hdc,bkgColor); - ReleaseDC(hwnd,hdc); - hbrush = CreateSolidBrush(bkgColor); - DeleteObject((HBRUSH)SetClassLongPtr(hClientWnd,GCL_HBRBACKGROUND,(LONG_PTR)hbrush)); - InvalidateRect(hwnd,NULL,TRUE); - } -} - -OFN_HOOK_RET APIENTRY OFNHookProc(HWND hwndDlg,UINT iMsg, - WPARAM wParam,LPARAM lParam) -{ - /* Hook procedure for open file dialog box */ - HWND hOwner,hDlg; - RECT rc,rcOwner,rcDlg; - hDlg = GetParent(hwndDlg); - switch (iMsg) { - case WM_INITDIALOG: - /* center dialogbox within its owner window */ - if ((hOwner = GetParent(hDlg)) == NULL) - hOwner = GetDesktopWindow(); - GetWindowRect(hOwner, &rcOwner); - GetWindowRect(hDlg, &rcDlg); - CopyRect(&rc, &rcOwner); - OffsetRect(&rcDlg, -rcDlg.left, -rcDlg.top); - OffsetRect(&rc, -rc.left, -rc.top); - OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); - SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), - rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); - return (OFN_HOOK_RET) 1; - default: - break; - } - return (OFN_HOOK_RET) 0; /* the let default procedure process the message */ -} - -static void -GetFileName(HWND hwnd, TCHAR *pFile) -{ - /* Open the File Open dialog box and */ - /* retrieve the file name */ - OPENFILENAME ofn; - TCHAR szFilterSpec [128] = TEXT("logfiles (*.log)\0*.log\0All files (*.*)\0*.*\0\0"); - #define MAXFILENAME 256 - TCHAR szFileName[MAXFILENAME]; - TCHAR szFileTitle[MAXFILENAME]; - - /* these need to be filled in */ - _tcscpy(szFileName, TEXT("erlshell.log")); - _tcscpy(szFileTitle, TEXT("")); /* must be NULL */ - - ofn.lStructSize = sizeof(OPENFILENAME); - ofn.hwndOwner = NULL; - ofn.lpstrFilter = szFilterSpec; - ofn.lpstrCustomFilter = NULL; - ofn.nMaxCustFilter = 0; - ofn.nFilterIndex = 0; - ofn.lpstrFile = szFileName; - ofn.nMaxFile = MAXFILENAME; - ofn.lpstrInitialDir = NULL; - ofn.lpstrFileTitle = szFileTitle; - ofn.nMaxFileTitle = MAXFILENAME; - ofn.lpstrTitle = TEXT("Open logfile"); - ofn.lpstrDefExt = TEXT("log"); - ofn.Flags = OFN_CREATEPROMPT|OFN_HIDEREADONLY|OFN_EXPLORER|OFN_ENABLEHOOK|OFN_NOCHANGEDIR; /* OFN_NOCHANGEDIR only works in Vista :( */ - ofn.lpfnHook = OFNHookProc; - - if (!GetOpenFileName ((LPOPENFILENAME)&ofn)){ - *pFile = TEXT('\0'); - } else { - _tcscpy(pFile, ofn.lpstrFile); - } -} - -void OpenLogFile(HWND hwnd) -{ - /* open a file for logging */ - TCHAR filename[_MAX_PATH]; - - GetFileName(hwnd, filename); - if (filename[0] == '\0') - return; - if (NULL == (logfile = _tfopen(filename,TEXT("w,ccs=UNICODE")))) - return; -} - -void CloseLogFile(HWND hwnd) -{ - /* close log file */ - fclose(logfile); - logfile = NULL; -} - -void LogFileWrite(TCHAR *buf, int num_chars) -{ - /* write to logfile */ - int from,to; - while (num_chars-- > 0) { - switch (*buf) { - case SET_CURSOR: - buf++; - from = *((int *)buf); - buf += sizeof(int)/sizeof(TCHAR); - to = *((int *)buf); - buf += (sizeof(int)/sizeof(TCHAR))-1; - num_chars -= 2 * (sizeof(int)/sizeof(TCHAR)); - // Won't seek in Unicode file, sorry... - // fseek(logfile,to-from *sizeof(TCHAR),SEEK_CUR); - break; - default: - _fputtc(*buf,logfile); - break; - } - buf++; - } -} - -static void -init_buffers(void) -{ - inbuf.data = (TCHAR *) ALLOC(BUFSIZE * sizeof(TCHAR)); - outbuf.data = (TCHAR *) ALLOC(BUFSIZE * sizeof(TCHAR)); - inbuf.size = BUFSIZE; - inbuf.rdPos = inbuf.wrPos = 0; - outbuf.size = BUFSIZE; - outbuf.rdPos = outbuf.wrPos = 0; -} - -static int -check_realloc(buffer_t *buf, int num_chars) -{ - if (buf->wrPos + num_chars >= buf->size) { - if (buf->size > MAXBUFSIZE) - return 0; - buf->size += num_chars + BUFSIZE; - if (!(buf->data = (TCHAR *)REALLOC(buf->data, buf->size * sizeof(TCHAR)))) { - buf->size = buf->rdPos = buf->wrPos = 0; - return 0; - } - } - return 1; -} - -static int -write_inbuf(TCHAR *data, int num_chars) -{ - TCHAR *buf; - int nwrite; - WaitForSingleObject(console_input,INFINITE); - if (!check_realloc(&inbuf,num_chars)) { - ReleaseSemaphore(console_input,1,NULL); - return -1; - } - buf = &inbuf.data[inbuf.wrPos]; - inbuf.wrPos += num_chars; - nwrite = num_chars; - while (nwrite--) - *buf++ = *data++; - SetEvent(console_input_event); - ReleaseSemaphore(console_input,1,NULL); - return num_chars; -} - -static int -write_outbuf(TCHAR *data, int num_chars) -{ - TCHAR *buf; - int nwrite; - - WaitForSingleObject(console_output,INFINITE); - if (!check_realloc(&outbuf, num_chars)) { - ReleaseSemaphore(console_output,1,NULL); - return -1; - } - if (outbuf.rdPos == outbuf.wrPos) - PostMessage(hClientWnd, WM_CONTEXT, 0L, 0L); - buf = &outbuf.data[outbuf.wrPos]; - outbuf.wrPos += num_chars; - nwrite = num_chars; - while (nwrite--) - *buf++ = *data++; - ReleaseSemaphore(console_output,1,NULL); - return num_chars; -} - -DIALOG_PROC_RET CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam) -{ - HWND hOwner; - RECT rc,rcOwner,rcDlg; - - switch (iMsg) { - case WM_INITDIALOG: - /* center dialogbox within its owner window */ - if ((hOwner = GetParent(hDlg)) == NULL) - hOwner = GetDesktopWindow(); - GetWindowRect(hOwner, &rcOwner); - GetWindowRect(hDlg, &rcDlg); - CopyRect(&rc, &rcOwner); - OffsetRect(&rcDlg, -rcDlg.left, -rcDlg.top); - OffsetRect(&rc, -rc.left, -rc.top); - OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); - SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), - rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); - SetDlgItemText(hDlg, ID_OTP_VERSIONSTRING, - TEXT("OTP version ") TEXT(ERLANG_OTP_VERSION)); - SetDlgItemText(hDlg, ID_ERTS_VERSIONSTRING, - TEXT("Erlang emulator version ") TEXT(ERLANG_VERSION)); - return (DIALOG_PROC_RET) TRUE; - case WM_COMMAND: - switch (LOWORD(wParam)) { - case IDOK: - case IDCANCEL: - EndDialog(hDlg,0); - return (DIALOG_PROC_RET) TRUE; - } - break; - } - return (DIALOG_PROC_RET) FALSE; -} - -static void -ConDrawText(HWND hwnd) -{ - int num_chars; - int nchars; - TCHAR *buf; - int from, to; - int dl; - int dc; - RECT rc; - - WaitForSingleObject(console_output, INFINITE); - nchars = 0; - num_chars = outbuf.wrPos - outbuf.rdPos; - buf = &outbuf.data[outbuf.rdPos]; - if (logfile != NULL) - LogFileWrite(buf, num_chars); - - -#ifdef HARDDEBUG - { - TCHAR *bu = (TCHAR *) ALLOC((num_chars+1) * sizeof(TCHAR)); - memcpy(bu,buf,num_chars * sizeof(TCHAR)); - bu[num_chars]='\0'; - fprintf(stderr,"ConDrawText\"%S\"\n",bu); - FREE(bu); - fflush(stderr); - } -#endif - /* - * Don't draw any text in the window; just update the line buffers - * and invalidate the appropriate part of the window. The window - * will be updated on the next WM_PAINT message. - */ - - while (num_chars-- > 0) { - switch (*buf) { - case '\r': - break; - case '\n': - if (nchars > 0) { - rc.left = GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x - nchars); - rc.right = rc.left + cxCharMax*nchars; - rc.top = cyChar * (cur_y-iVscrollPos); - rc.bottom = rc.top + cyChar; - InvalidateRect(hwnd, &rc, TRUE); - nchars = 0; - } - ConCarriageFeed(1); - ConScrollScreen(); - break; - case SET_CURSOR: - if (nchars > 0) { - rc.left = GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x - nchars); - rc.right = rc.left + cxCharMax*nchars; - rc.top = cyChar * (cur_y-iVscrollPos); - rc.bottom = rc.top + cyChar; - InvalidateRect(hwnd, &rc, TRUE); - nchars = 0; - } - buf++; - from = *((int *)buf); - buf += sizeof(int)/sizeof(TCHAR); - to = *((int *)buf); - buf += (sizeof(int)/sizeof(TCHAR))-1; - num_chars -= 2 * (sizeof(int)/sizeof(TCHAR)); - while (to > from) { - cur_x++; - if (GetXFromCurrentY(GetDC(hwnd),0,cur_x)+cxChar > - (LINE_LENGTH * cxChar)) { - cur_x = 0; - cur_y++; - ensure_line_below(); - cur_line = cur_line->next; - } - from++; - } - while (to < from) { - cur_x--; - if (cur_x < 0) { - cur_y--; - cur_line = cur_line->prev; - cur_x = cur_line->width-1; - } - from--; - } - - break; - default: - nchars++; - cur_line->text[cur_x] = *buf; - cur_x++; - if (cur_x > cur_line->width) - cur_line->width = cur_x; - if (GetXFromCurrentY(GetDC(hwnd),0,cur_x)+cxChar > - (LINE_LENGTH * cxChar)) { - if (nchars > 0) { - rc.left = GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x - nchars); - rc.right = rc.left + cxCharMax*nchars; - rc.top = cyChar * (cur_y-iVscrollPos); - rc.bottom = rc.top + cyChar; - InvalidateRect(hwnd, &rc, TRUE); - } - ConCarriageFeed(0); - nchars = 0; - } - } - buf++; - } - if (nchars > 0) { - rc.left = GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x - nchars); - rc.right = rc.left + cxCharMax*nchars; - rc.top = cyChar * (cur_y-iVscrollPos); - rc.bottom = rc.top + cyChar; - InvalidateRect(hwnd, &rc, TRUE); - } - ConScrollScreen(); - SetCaretPos(GetXFromCurrentY(GetDC(hwnd),iHscrollPos,cur_x), (cur_y-iVscrollPos)*cyChar); - outbuf.wrPos = outbuf.rdPos = 0; - ReleaseSemaphore(console_output, 1, NULL); -} - -static void -AddToCmdHistory(void) -{ - int i; - int size; - Uint32 *buf; - wchar_t cmdBuf[128]; - - if (llen != 0) { - for (i = 0, size = 0; i < llen-1; i++) { - /* - * Find end of prompt. - */ - if ((lbuf[i] == '>') && lbuf[i+1] == ' ') { - buf = &lbuf[i+2]; - size = llen-i-2; - break; - } - } - if (size > 0 && size < 128) { - for (i = 0;i < size; ++i) { - cmdBuf[i] = (wchar_t) buf[i]; - } - cmdBuf[size] = 0; - SendMessage(hComboWnd,CB_INSERTSTRING,0,(LPARAM)cmdBuf); - } - } -} - -/*static TBBUTTON tbb[] = -{ - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - 0, IDMENU_COPY, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0, - 1, IDMENU_PASTE, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0, - 2, IDMENU_FONT, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0, - 3, IDMENU_ABOUT, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0, - 0, 0, TBSTATE_ENABLED, TBSTYLE_SEP, 0, 0, 0, 0, - };*/ -static TBBUTTON tbb[] = -{ - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP}, - {0, IDMENU_COPY, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE}, - {1, IDMENU_PASTE, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE}, - {2, IDMENU_FONT, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE}, - {3, IDMENU_ABOUT, TBSTATE_ENABLED, TBSTYLE_AUTOSIZE}, - {0, 0, TBSTATE_ENABLED, TBSTYLE_SEP} -}; - -static TBADDBITMAP tbbitmap = -{ - HINST_COMMCTRL, IDB_STD_SMALL_COLOR, -}; - - -static HWND -InitToolBar(HWND hwndParent) -{ - int x,y,cx; - HWND hwndTB,hwndTT; - RECT r; - TOOLINFO ti; - HFONT hFontNew; - DWORD backgroundColor = GetSysColor(COLOR_BTNFACE); - COLORMAP colorMap; - colorMap.from = RGB(192, 192, 192); - colorMap.to = backgroundColor; - /* Create toolbar window with tooltips */ - hwndTB = CreateWindowEx(0,TOOLBARCLASSNAME,(TCHAR *)NULL, - WS_CHILD|CCS_TOP|WS_CLIPSIBLINGS|TBSTYLE_TOOLTIPS, - 0,0,0,0,hwndParent, - (HMENU)2,hInstance,NULL); - SendMessage(hwndTB,TB_BUTTONSTRUCTSIZE, - (WPARAM) sizeof(TBBUTTON),0); - tbbitmap.hInst = NULL; - tbbitmap.nID = (UINT_PTR) CreateMappedBitmap(beam_module, 1,0, &colorMap, 1); - SendMessage(hwndTB, TB_ADDBITMAP, (WPARAM) 4, - (LPARAM) &tbbitmap); - - SendMessage(hwndTB,TB_ADDBUTTONS, (WPARAM) 30, - (LPARAM) tbb); - if (toolbarVisible) - ShowWindow(hwndTB, SW_SHOW); - - /* Create combobox window */ - SendMessage(hwndTB,TB_GETITEMRECT,0,(LPARAM)&r); - x = r.left; y = r.top; - SendMessage(hwndTB,TB_GETITEMRECT,23,(LPARAM)&r); - cx = r.right - x + 1; - hComboWnd = CreateWindow(TEXT("combobox"),NULL,WS_VSCROLL|WS_CHILD|WS_VISIBLE|CBS_DROPDOWNLIST, - x,y,cx,100,hwndParent,(HMENU)ID_COMBOBOX, hInstance,NULL); - SetParent(hComboWnd,hwndTB); - hFontNew = CreateFontIndirect(&logfont); - SendMessage(hComboWnd,WM_SETFONT,(WPARAM)hFontNew, - MAKELPARAM(1,0)); - - /* Add tooltip for combo box */ - ZeroMemory(&ti,sizeof(TOOLINFO)); - ti.cbSize = sizeof(TOOLINFO); - ti.uFlags = TTF_IDISHWND|TTF_CENTERTIP|TTF_SUBCLASS; - ti.hwnd = hwndTB;; - ti.uId = (UINT_PTR)hComboWnd; - ti.lpszText = LPSTR_TEXTCALLBACK; - hwndTT = (HWND)SendMessage(hwndTB,TB_GETTOOLTIPS,0,0); - SendMessage(hwndTT,TTM_ADDTOOL,0,(LPARAM)&ti); - - return hwndTB; -} - -static void -window_title(struct title_buf *tbuf) -{ - int res, i; - size_t bufsz = TITLE_BUF_SZ; - unsigned char charbuff[TITLE_BUF_SZ]; - - res = erl_drv_getenv("ERL_WINDOW_TITLE", charbuff, &bufsz); - if (res < 0) - tbuf->name = erlang_window_title; - else if (res == 0) { - for (i = 0; i < bufsz; ++i) { - tbuf->buf[i] = charbuff[i]; - } - tbuf->buf[bufsz - 1] = 0; - tbuf->name = &tbuf->buf[0]; - } else { - char *buf = ALLOC(bufsz); - if (!buf) - tbuf->name = erlang_window_title; - else { - while (1) { - char *newbuf; - res = erl_drv_getenv("ERL_WINDOW_TITLE", buf, &bufsz); - if (res <= 0) { - if (res == 0) { - TCHAR *wbuf = ALLOC(bufsz *sizeof(TCHAR)); - for (i = 0; i < bufsz ; ++i) { - wbuf[i] = buf[i]; - } - wbuf[bufsz - 1] = 0; - FREE(buf); - tbuf->name = wbuf; - } else { - tbuf->name = erlang_window_title; - FREE(buf); - } - break; - } - newbuf = REALLOC(buf, bufsz); - if (newbuf) - buf = newbuf; - else { - tbuf->name = erlang_window_title; - FREE(buf); - break; - } - } - } - } -} - -static void -free_window_title(struct title_buf *tbuf) -{ - if (tbuf->name != erlang_window_title && tbuf->name != &tbuf->buf[0]) - FREE(tbuf->name); -} diff --git a/erts/emulator/drivers/win32/win_con.h b/erts/emulator/drivers/win32/win_con.h deleted file mode 100644 index 7a642cd7ed..0000000000 --- a/erts/emulator/drivers/win32/win_con.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -/* - * External API for the windows console (aka werl window) - * used by ttsl_drv.c - */ -#ifndef _WIN_CON_H_VISITED -#define _WIN_CON_H_VISITED 1 -void ConNormalExit(void); -void ConWaitForExit(void); -void ConSetCtrlHandler(BOOL (WINAPI *handler)(DWORD)); -int ConPutChar(Uint32 c); -void ConSetCursor(int from, int to); -void ConPrintf(char *format, ...); -void ConVprintf(char *format, va_list va); -void ConBeep(void); -int ConReadInput(Uint32 *data, int nbytes); -int ConGetKey(void); -int ConGetColumns(void); -int ConGetRows(void); -void ConInit(void); -#endif /* _WIN_CON_H_VISITED */ diff --git a/erts/emulator/nifs/common/prim_tty_nif.c b/erts/emulator/nifs/common/prim_tty_nif.c new file mode 100644 index 0000000000..83279c832f --- /dev/null +++ b/erts/emulator/nifs/common/prim_tty_nif.c @@ -0,0 +1,1036 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2015-2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: NIF library for interacting with the tty + * + */ + +#define STATIC_ERLANG_NIF 1 + +#ifndef WANT_NONBLOCKING +#define WANT_NONBLOCKING +#endif + +#include "config.h" +#include "sys.h" +#include "erl_nif.h" +#include "erl_driver.h" + +#include <errno.h> +#include <string.h> +#include <ctype.h> +#include <wchar.h> +#include <stdio.h> +#include <signal.h> +#include <locale.h> +#ifdef HAVE_TERMCAP + #include <termios.h> + #include <curses.h> + #include <term.h> +#endif +#ifndef __WIN32__ + #include <unistd.h> + #include <sys/ioctl.h> +#endif +#ifdef HAVE_SYS_UIO_H + #include <sys/uio.h> +#endif + +#if defined IOV_MAX +#define MAXIOV IOV_MAX +#elif defined UIO_MAXIOV +#define MAXIOV UIO_MAXIOV +#else +#define MAXIOV 16 +#endif + +#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H) +#define PRIMITIVE_UTF8_CHECK 1 +#else +#include <langinfo.h> +#endif + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#define DEF_HEIGHT 24 +#define DEF_WIDTH 80 + +typedef struct { +#ifdef __WIN32__ + HANDLE ofd; + HANDLE ifd; + DWORD dwOriginalOutMode; + DWORD dwOriginalInMode; + DWORD dwOutMode; + DWORD dwInMode; +#else + int ofd; /* stdout */ + int ifd; /* stdin */ +#endif + ErlNifPid self; + ErlNifPid reader; + int tty; /* if the tty is initialized */ +#ifdef THREADED_READER + ErlNifTid reader_tid; +#endif +#ifndef __WIN32__ + int signal[2]; /* Pipe used for signal (winch + cont) notifications */ +#endif +#ifdef HAVE_TERMCAP + struct termios tty_smode; + struct termios tty_rmode; +#endif +} TTYResource; + +static ErlNifResourceType *tty_rt; + +/* The NIFs: */ +static ERL_NIF_TERM isatty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_create_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_set_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM setlocale_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_read_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM isprint_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM wcwidth_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM wcswidth_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM sizeof_wchar_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_window_size_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_tgetent_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_tgetnum_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_tgetflag_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_tgetstr_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_tgoto_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM tty_read_signal_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +static ErlNifFunc nif_funcs[] = { + {"isatty", 1, isatty_nif}, + {"tty_create", 0, tty_create_nif}, + {"tty_init", 3, tty_init_nif}, + {"tty_set", 1, tty_set_nif}, + {"tty_read_signal", 2, tty_read_signal_nif}, + {"setlocale", 1, setlocale_nif}, + {"tty_select", 3, tty_select_nif}, + {"tty_window_size", 1, tty_window_size_nif}, + {"write_nif", 2, tty_write_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_nif", 2, tty_read_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"isprint", 1, isprint_nif}, + {"wcwidth", 1, wcwidth_nif}, + {"wcswidth", 1, wcswidth_nif}, + {"sizeof_wchar", 0, sizeof_wchar_nif}, + {"tgetent_nif", 1, tty_tgetent_nif}, + {"tgetnum_nif", 1, tty_tgetnum_nif}, + {"tgetflag_nif", 1, tty_tgetflag_nif}, + {"tgetstr_nif", 1, tty_tgetstr_nif}, + {"tgoto_nif", 2, tty_tgoto_nif}, + {"tgoto_nif", 3, tty_tgoto_nif} +}; + +/* NIF interface declarations */ +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); +static void unload(ErlNifEnv* env, void* priv_data); + +ERL_NIF_INIT(prim_tty, nif_funcs, load, NULL, upgrade, unload) + +#define ATOMS \ + ATOM_DECL(canon); \ + ATOM_DECL(echo); \ + ATOM_DECL(ebadf); \ + ATOM_DECL(undefined); \ + ATOM_DECL(error); \ + ATOM_DECL(true); \ + ATOM_DECL(ok); \ + ATOM_DECL(input); \ + ATOM_DECL(false); \ + ATOM_DECL(stdin); \ + ATOM_DECL(stdout); \ + ATOM_DECL(stderr); \ + ATOM_DECL(sig); + + +#define ATOM_DECL(A) static ERL_NIF_TERM atom_##A +ATOMS +#undef ATOM_DECL + +static ERL_NIF_TERM make_error(ErlNifEnv *env, ERL_NIF_TERM reason) { + return enif_make_tuple2(env, atom_error, reason); +} + +static ERL_NIF_TERM make_enotsup(ErlNifEnv *env) { + return make_error(env, enif_make_atom(env, "enotsup")); +} + +static ERL_NIF_TERM make_errno(ErlNifEnv *env) { +#ifdef __WIN32__ + return enif_make_atom(env, last_error()); +#else + return enif_make_atom(env, erl_errno_id(errno)); +#endif +} + +static ERL_NIF_TERM make_errno_error(ErlNifEnv *env, const char *function) { + return make_error( + env, enif_make_tuple2( + env, enif_make_atom(env, function), make_errno(env))); +} + +static int tty_get_fd(ErlNifEnv *env, ERL_NIF_TERM atom, int *fd) { + if (enif_is_identical(atom, atom_stdout)) { + *fd = fileno(stdout); + } else if (enif_is_identical(atom, atom_stdin)) { + *fd = fileno(stdin); + } else if (enif_is_identical(atom, atom_stderr)) { + *fd = fileno(stderr); + } else { + return 0; + } + return 1; +} + +static ERL_NIF_TERM isatty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + int fd; + if (tty_get_fd(env, argv[0], &fd)) { + if (isatty(fd)) { + return atom_true; + } else if (errno == EINVAL || errno == ENOTTY) { + return atom_false; + } else { + return atom_ebadf; + } + } + return enif_make_badarg(env); +} + +static ERL_NIF_TERM isprint_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + int i; + if (enif_get_int(env, argv[0], &i)) { + ASSERT(i > 0 && i < 256); + return isprint((char)i) ? atom_true : atom_false; + } + return enif_make_badarg(env); +} + +static ERL_NIF_TERM wcwidth_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + int i; + if (enif_get_int(env, argv[0], &i)) { +#ifndef __WIN32__ + int width; + ASSERT(i > 0 && i < (1l << 21)); + width = wcwidth((wchar_t)i); + if (width == -1) { + return make_error(env, enif_make_atom(env, "not_printable")); + } + return enif_make_int(env, width); +#else + return make_enotsup(env); +#endif + } + return enif_make_badarg(env); +} + +static ERL_NIF_TERM wcswidth_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifBinary bin; + if (enif_inspect_iolist_as_binary(env, argv[0], &bin)) { + wchar_t *chars = (wchar_t*)bin.data; + int width; +#ifdef DEBUG + for (int i = 0; i < bin.size / sizeof(wchar_t); i++) { + ASSERT(chars[i] >= 0 && chars[i] < (1l << 21)); + } +#endif +#ifndef __WIN32__ + width = wcswidth(chars, bin.size / sizeof(wchar_t)); +#else + width = bin.size / sizeof(wchar_t); +#endif + if (width == -1) { + return make_error(env, enif_make_atom(env, "not_printable")); + } + return enif_make_tuple2(env, atom_ok, enif_make_int(env, width)); + } + return enif_make_badarg(env); +} + +static ERL_NIF_TERM sizeof_wchar_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + return enif_make_int(env, sizeof(wchar_t)); +} + +static ERL_NIF_TERM tty_write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM head = argv[1], tail; + ErlNifIOQueue *q = NULL; + ErlNifIOVec vec, *iovec = &vec; + SysIOVec *iov; + int iovcnt; + TTYResource *tty; + ssize_t res = 0; + size_t size; + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + + while (!enif_is_identical(head, enif_make_list(env, 0))) { + if (!enif_inspect_iovec(env, MAXIOV, head, &tail, &iovec)) + return enif_make_badarg(env); + + head = tail; + + iov = iovec->iov; + size = iovec->size; + iovcnt = iovec->iovcnt; + + do { +#ifndef __WIN32__ + do { + res = writev(tty->ofd, iov, iovcnt); + } while(res < 0 && (errno == EINTR || errno == EAGAIN)); +#else + for (int i = 0; i < iovec->iovcnt; i++) { + ssize_t written; + BOOL r = WriteFile(tty->ofd, iovec->iov[i].iov_base, + iovec->iov[i].iov_len, &written, NULL); + if (!r) { + res = -1; + break; + } + res += written; + } +#endif + if (res < 0) { + if (q) enif_ioq_destroy(q); + return make_error(env, make_errno(env)); + } + if (res != size) { + if (!q) { + q = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + enif_ioq_enqv(q, iovec, 0); + } + } + + if (q) { + enif_ioq_deq(q, res, &size); + if (size == 0) { + enif_ioq_destroy(q); + q = NULL; + } else { + iov = enif_ioq_peek(q, &iovcnt); + } + } + } while(q); + + }; + return atom_ok; +} + +static ERL_NIF_TERM tty_read_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + TTYResource *tty; + ErlNifBinary bin; + ERL_NIF_TERM res_term; + ssize_t res = 0; + + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + +#ifdef __WIN32__ + if (tty->dwInMode) { + ssize_t inputs_read, num_characters = 0; + wchar_t *characters = NULL; + INPUT_RECORD inputs[128]; + if (!ReadConsoleInputW(tty->ifd, inputs, sizeof(inputs)/sizeof(*inputs), + &inputs_read)) { + return make_errno_error(env, "ReadConsoleInput"); + } + for (int i = 0; i < inputs_read; i++) { + if (inputs[i].EventType == KEY_EVENT) { + if (inputs[i].Event.KeyEvent.bKeyDown && + inputs[i].Event.KeyEvent.uChar.UnicodeChar < 256 && + inputs[i].Event.KeyEvent.uChar.UnicodeChar != 0) { + num_characters++; + } + if (!inputs[i].Event.KeyEvent.bKeyDown && + inputs[i].Event.KeyEvent.uChar.UnicodeChar > 255 && + inputs[i].Event.KeyEvent.uChar.UnicodeChar != 0) { + num_characters++; + } + } + } + enif_alloc_binary(num_characters * sizeof(wchar_t), &bin); + characters = (wchar_t*)bin.data; + for (int i = 0; i < inputs_read; i++) { + switch (inputs[i].EventType) + { + case KEY_EVENT: + if (inputs[i].Event.KeyEvent.bKeyDown && + inputs[i].Event.KeyEvent.uChar.UnicodeChar < 256 && + inputs[i].Event.KeyEvent.uChar.UnicodeChar != 0) { + characters[res++] = inputs[i].Event.KeyEvent.uChar.UnicodeChar; + } + if (!inputs[i].Event.KeyEvent.bKeyDown && + inputs[i].Event.KeyEvent.uChar.UnicodeChar > 255 && + inputs[i].Event.KeyEvent.uChar.UnicodeChar != 0) { + characters[res++] = inputs[i].Event.KeyEvent.uChar.UnicodeChar; + } + break; + case WINDOW_BUFFER_SIZE_EVENT: + enif_send(env, &tty->self, NULL, + enif_make_tuple2(env, enif_make_atom(env, "resize"), + enif_make_tuple2(env, + enif_make_int(env, inputs[i].Event.WindowBufferSizeEvent.dwSize.Y), + enif_make_int(env, inputs[i].Event.WindowBufferSizeEvent.dwSize.X)))); + break; + case MENU_EVENT: + case FOCUS_EVENT: + /* Should be ignored according to + https://docs.microsoft.com/en-us/windows/console/input-record-str */ + break; + default: + fprintf(stderr,"Unknown event: %d\r\n", inputs[i].EventType); + break; + } + } + res *= sizeof(wchar_t); + } else { + DWORD bytesTransferred; + enif_alloc_binary(1024, &bin); + if (ReadFile(tty->ifd, bin.data, bin.size, + &bytesTransferred, NULL)) { + res = bytesTransferred; + if (res == 0) { + enif_release_binary(&bin); + return make_error(env, enif_make_atom(env, "closed")); + } + } else { + DWORD error = GetLastError(); + enif_release_binary(&bin); + if (error == ERROR_BROKEN_PIPE) + return make_error(env, enif_make_atom(env, "closed")); + return make_errno_error(env, "ReadFile"); + } + } +#else + enif_alloc_binary(1024, &bin); + res = read(tty->ifd, bin.data, bin.size); + if (res < 0) { + if (errno != EAGAIN && errno != EINTR) { + enif_release_binary(&bin); + return make_errno_error(env, "read"); + } + res = 0; + } else if (res == 0) { + enif_release_binary(&bin); + return make_error(env, enif_make_atom(env, "closed")); + } +#endif + enif_select(env, tty->ifd, ERL_NIF_SELECT_READ, tty, NULL, argv[1]); + if (res == bin.size) { + res_term = enif_make_binary(env, &bin); + } else if (res < bin.size / 2) { + unsigned char *buff = enif_make_new_binary(env, res, &res_term); + if (res > 0) { + memcpy(buff, bin.data, res); + } + enif_release_binary(&bin); + } else { + enif_realloc_binary(&bin, res); + res_term = enif_make_binary(env, &bin); + } + + return enif_make_tuple2(env, atom_ok, res_term); +} + +static ERL_NIF_TERM setlocale_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef __WIN32__ + TTYResource *tty; + + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + + if (tty->dwOutMode) + { + if (!SetConsoleOutputCP(CP_UTF8)) { + return make_errno_error(env, "SetConsoleOutputCP"); + } + } + return atom_true; +#elif defined(PRIMITIVE_UTF8_CHECK) + setlocale(LC_CTYPE, ""); /* Set international environment, + ignore result */ + return enif_make_atom(env, "primitive"); +#else + char *l = setlocale(LC_CTYPE, ""); /* Set international environment */ + if (l != NULL) { + if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) + return atom_true; + } + return atom_false; +#endif +} + +static ERL_NIF_TERM tty_tgetent_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef HAVE_TERMCAP + ErlNifBinary TERM; + if (!enif_inspect_iolist_as_binary(env, argv[0], &TERM)) + return enif_make_badarg(env); + if (tgetent((char *)NULL /* ignored */, (char *)TERM.data) <= 0) { + return make_errno_error(env, "tgetent"); + } + return atom_ok; +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_tgetnum_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef HAVE_TERMCAP + ErlNifBinary TERM; + if (!enif_inspect_iolist_as_binary(env, argv[0], &TERM)) + return enif_make_badarg(env); + return enif_make_int(env, tgetnum((char*)TERM.data)); +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_tgetflag_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef HAVE_TERMCAP + ErlNifBinary TERM; + if (!enif_inspect_iolist_as_binary(env, argv[0], &TERM)) + return enif_make_badarg(env); + if (tgetflag((char*)TERM.data)) + return atom_true; + return atom_false; +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_tgetstr_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef HAVE_TERMCAP + ErlNifBinary TERM, ret; + /* tgetstr seems to use a lot of stack buffer space, + so buff needs to be relatively "small" */ + char *str = NULL; + char buff[BUFSIZ] = {0}; + + if (!enif_inspect_iolist_as_binary(env, argv[0], &TERM)) + return enif_make_badarg(env); + str = tgetstr((char*)TERM.data, (char**)&buff); + if (!str) return atom_false; + enif_alloc_binary(strlen(str), &ret); + memcpy(ret.data, str, strlen(str)); + return enif_make_tuple2( + env, atom_ok, enif_make_binary(env, &ret)); +#else + return make_enotsup(env); +#endif +} + +#ifdef HAVE_TERMCAP +static int tputs_buffer_index; +static unsigned char tputs_buffer[1024]; + +#if defined(__sun) && defined(__SVR4) /* Solaris */ +static int tty_puts_putc(char c) { +#else +static int tty_puts_putc(int c) { +#endif + tputs_buffer[tputs_buffer_index++] = (unsigned char)c; + return 0; +} +#endif + +static ERL_NIF_TERM tty_tgoto_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#ifdef HAVE_TERMCAP + ErlNifBinary TERM; + char *ent; + int value1, value2 = 0; + if (!enif_inspect_iolist_as_binary(env, argv[0], &TERM) || + !enif_get_int(env, argv[1], &value1)) + return enif_make_badarg(env); + if (argc == 2) { + ent = tgoto((char*)TERM.data, 0, value1); + } else { + ASSERT(argc == 3); + ent = tgoto((char*)TERM.data, value1, value2); + } + if (!ent) return make_errno_error(env, "tgoto"); + + tputs_buffer_index = 0; + if (tputs(ent, 1, tty_puts_putc)) { + return make_errno_error(env, "tputs"); + } else { + ERL_NIF_TERM ret; + unsigned char *buff = enif_make_new_binary(env, tputs_buffer_index, &ret); + memcpy(buff, tputs_buffer, tputs_buffer_index); + return enif_make_tuple2(env, atom_ok, ret); + } +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_create_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + + TTYResource *tty = enif_alloc_resource(tty_rt, sizeof(TTYResource)); + ERL_NIF_TERM tty_term; + memset(tty, 0, sizeof(*tty)); +#ifndef __WIN32__ + tty->ifd = 0; + tty->ofd = 1; +#else + tty->ifd = GetStdHandle(STD_INPUT_HANDLE); + if (tty->ifd == INVALID_HANDLE_VALUE || tty->ifd == NULL) { + tty->ifd = CreateFile("nul", GENERIC_READ, 0, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + } + tty->ofd = GetStdHandle(STD_OUTPUT_HANDLE); + if (tty->ofd == INVALID_HANDLE_VALUE || tty->ofd == NULL) { + tty->ofd = CreateFile("nul", GENERIC_WRITE, 0, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + } + if (GetConsoleMode(tty->ofd, &tty->dwOriginalOutMode)) + { + tty->dwOutMode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | tty->dwOriginalOutMode; + if (!SetConsoleMode(tty->ofd, tty->dwOutMode)) { + /* Failed to set any VT mode, can't do anything here. */ + return make_errno_error(env, "SetConsoleMode"); + } + } + if (GetConsoleMode(tty->ifd, &tty->dwOriginalInMode)) + { + tty->dwInMode = ENABLE_VIRTUAL_TERMINAL_INPUT | tty->dwOriginalInMode; + if (!SetConsoleMode(tty->ifd, tty->dwInMode)) { + /* Failed to set any VT mode, can't do anything here. */ + return make_errno_error(env, "SetConsoleMode"); + } + } +#endif + + tty_term = enif_make_resource(env, tty); + enif_release_resource(tty); + + enif_set_pid_undefined(&tty->self); + enif_set_pid_undefined(&tty->reader); + + return enif_make_tuple2(env, atom_ok, tty_term); +} + +static ERL_NIF_TERM tty_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + +#if defined(HAVE_TERMCAP) || defined(__WIN32__) + ERL_NIF_TERM canon, echo, sig; + TTYResource *tty; + int fd; + + if (argc != 3 || + !tty_get_fd(env, argv[1], &fd) || + !enif_is_map(env, argv[2])) { + return enif_make_badarg(env); + } + + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + + if (!enif_get_map_value(env, argv[2], enif_make_atom(env,"canon"), &canon)) + canon = enif_make_atom(env, "undefined"); + if (!enif_get_map_value(env, argv[2], enif_make_atom(env,"echo"), &echo)) + echo = enif_make_atom(env, "undefined"); + if (!enif_get_map_value(env, argv[2], enif_make_atom(env,"sig"), &sig)) + sig = enif_make_atom(env, "undefined"); + +#ifndef __WIN32__ + if (tcgetattr(fd, &tty->tty_rmode) < 0) { + return make_errno_error(env, "tcgetattr"); + } + + tty->tty_smode = tty->tty_rmode; + + /* Default characteristics for all usage including termcap output. */ + tty->tty_smode.c_iflag &= ~ISTRIP; + + /* erts_fprintf(stderr,"canon %T\r\n", canon); */ + /* Turn canonical (line mode) on off. */ + if (enif_is_identical(canon, atom_true)) { + tty->tty_smode.c_iflag |= ICRNL; + tty->tty_smode.c_lflag |= ICANON; + tty->tty_smode.c_oflag |= OPOST; + tty->tty_smode.c_cc[VEOF] = tty->tty_rmode.c_cc[VEOF]; +#ifdef VDSUSP + tty->tty_smode.c_cc[VDSUSP] = tty->tty_rmode.c_cc[VDSUSP]; +#endif + } + if (enif_is_identical(canon, atom_false)) { + tty->tty_smode.c_iflag &= ~ICRNL; + tty->tty_smode.c_lflag &= ~ICANON; + tty->tty_smode.c_oflag &= ~OPOST; + + tty->tty_smode.c_cc[VMIN] = 1; + tty->tty_smode.c_cc[VTIME] = 0; +#ifdef VDSUSP + tty->tty_smode.c_cc[VDSUSP] = 0; +#endif + } + + /* Turn echo on or off. */ + /* erts_fprintf(stderr,"echo %T\r\n", echo); */ + if (enif_is_identical(echo, atom_true)) + tty->tty_smode.c_lflag |= ECHO; + if (enif_is_identical(echo, atom_false)) + tty->tty_smode.c_lflag &= ~ECHO; + + /* erts_fprintf(stderr,"sig %T\r\n", sig); */ + /* Set extra characteristics for "RAW" mode, no signals. */ + if (enif_is_identical(sig, atom_true)) { + /* Ignore IMAXBEL as not POSIX. */ +#ifndef QNX + tty->tty_smode.c_iflag |= (BRKINT|IGNPAR|ICRNL|IXON|IXANY); +#else + tty->tty_smode.c_iflag |= (BRKINT|IGNPAR|ICRNL|IXON); +#endif + tty->tty_smode.c_lflag |= (ISIG|IEXTEN); + } + if (enif_is_identical(sig, atom_false)) { + /* Ignore IMAXBEL as not POSIX. */ +#ifndef QNX + tty->tty_smode.c_iflag &= ~(BRKINT|IGNPAR|ICRNL|IXON|IXANY); +#else + tty->tty_smode.c_iflag &= ~(BRKINT|IGNPAR|ICRNL|IXON); +#endif + tty->tty_smode.c_lflag &= ~(ISIG|IEXTEN); + } + +#else + /* fprintf(stderr, "origOutMode: %x origInMode: %x\r\n", */ + /* tty->dwOriginalOutMode, tty->dwOriginalInMode); */ + + /* If we cannot disable NEWLINE_AUTO_RETURN we continue anyway as things work */ + if (SetConsoleMode(tty->ofd, tty->dwOutMode | DISABLE_NEWLINE_AUTO_RETURN)) { + tty->dwOutMode |= DISABLE_NEWLINE_AUTO_RETURN; + } + + tty->dwInMode &= ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT); + if (!SetConsoleMode(tty->ifd, tty->dwInMode)) + { + /* Failed to set disable echo or line input mode */ + return make_errno_error(env, "SetConsoleMode"); + } + +#endif /* __WIN32__ */ + + tty->tty = 1; + + return atom_ok; +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_set_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { +#if defined(HAVE_TERMCAP) || defined(__WIN32__) + TTYResource *tty; + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); +#ifdef HAVE_TERMCAP + if (tty->tty && tcsetattr(tty->ifd, TCSANOW, &tty->tty_smode) < 0) { + return make_errno_error(env, "tcsetattr"); + } +#endif + enif_self(env, &tty->self); + enif_monitor_process(env, tty, &tty->self, NULL); + return atom_ok; +#else + return make_enotsup(env); +#endif +} + +static ERL_NIF_TERM tty_window_size_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + TTYResource *tty; + int width = -1, height = -1; + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + { +#ifdef TIOCGWINSZ + struct winsize ws; + if (ioctl(tty->ifd,TIOCGWINSZ,&ws) == 0) { + if (ws.ws_col > 0) + width = ws.ws_col; + if (ws.ws_row > 0) + height = ws.ws_row; + } else if (ioctl(tty->ofd,TIOCGWINSZ,&ws) == 0) { + if (ws.ws_col > 0) + width = ws.ws_col; + if (ws.ws_row > 0) + height = ws.ws_row; + } +#elif defined(__WIN32__) + CONSOLE_SCREEN_BUFFER_INFOEX buffer_info; + buffer_info.cbSize = sizeof(buffer_info); + if (GetConsoleScreenBufferInfoEx(tty->ofd, &buffer_info)) { + height = buffer_info.dwSize.Y; + width = buffer_info.dwSize.X; + } else { + return make_errno_error(env,"GetConsoleScreenBufferInfoEx"); + } +#endif + } + if (width == -1 && height == -1) { + return make_enotsup(env); + } + return enif_make_tuple2( + env, atom_ok, + enif_make_tuple2( + env, + enif_make_int(env, width), + enif_make_int(env, height) + )); +} + +#ifndef __WIN32__ + +static int tty_signal_fd = -1; + +static RETSIGTYPE tty_cont(int sig) +{ + if (tty_signal_fd != 1) { + while (write(tty_signal_fd, "c", 1) < 0 && errno == EINTR) { }; + } +} + + +static RETSIGTYPE tty_winch(int sig) +{ + if (tty_signal_fd != 1) { + while (write(tty_signal_fd, "w", 1) < 0 && errno == EINTR) { }; + } +} + +#endif + +static ERL_NIF_TERM tty_read_signal_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + TTYResource *tty; + char buff[1]; + ssize_t ret; + ERL_NIF_TERM res; + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); +#ifndef __WIN32__ + do { + ret = read(tty->signal[0], buff, 1); + } while (ret < 0 && errno == EAGAIN); + + if (ret < 0) { + return make_errno_error(env, "read"); + } else if (ret == 0) { + return make_error(env, enif_make_atom(env,"empty")); + } + + enif_select(env, tty->signal[0], ERL_NIF_SELECT_READ, tty, NULL, argv[1]); + + if (buff[0] == 'w') { + res = enif_make_atom(env, "winch"); + } else if (buff[0] == 'c') { + res = enif_make_atom(env, "cont"); + } else { + res = enif_make_string_len(env, buff, 1, ERL_NIF_LATIN1); + } + return enif_make_tuple2(env, atom_ok, res); +#else + return make_enotsup(env); +#endif +} + +#ifdef THREADED_READED +struct tty_reader_init { + ErlNifEnv *env; + ERL_NIF_TERM tty; +}; + +#define TTY_READER_BUF_SIZE 1024 + +static void *tty_reader_thread(void *args) { + struct tty_reader_init *tty_reader_init = (struct tty_reader_init*)args; + TTYResource *tty; + ErlNifBinary binary; + ErlNifEnv *env = NULL; + ERL_NIF_TERM data[10]; + int cnt = 0; + + enif_alloc_binary(TTY_READER_BUF_SIZE, &binary); + + enif_get_resource(tty_reader_init->env, tty_reader_init->tty, tty_rt, (void **)&tty); + + SET_BLOCKING(tty->ifd); + + while(true) { + ssize_t i = read(tty->ifd, binary.data, TTY_READER_BUF_SIZE); + /* fprintf(stderr,"Read: %ld bytes from %d\r\n", i, tty->ifd); */ + if (i < 0) { + int saved_errno = errno; + if (env) { + ERL_NIF_TERM msg = enif_make_list_from_array(env, data, cnt); + enif_send(env, &tty->self, NULL, enif_make_tuple2(env, atom_input, msg)); + cnt = 0; + env = NULL; + } + if (saved_errno != EAGAIN) { + env = enif_alloc_env(); + errno = saved_errno; + enif_send(env, &tty->self, NULL, make_errno_error(env, "read")); + break; + } + } else { + if (!env) { + env = enif_alloc_env(); + } + enif_realloc_binary(&binary, i); + data[cnt++] = enif_make_binary(env, &binary); + if (cnt == 10 || i != TTY_READER_BUF_SIZE) { + ERL_NIF_TERM msg = enif_make_list_from_array(env, data, cnt); + enif_send(env, &tty->self, NULL, enif_make_tuple2(env, atom_input, msg)); + cnt = 0; + env = NULL; + } + enif_alloc_binary(TTY_READER_BUF_SIZE, &binary); + } + } + + enif_free_env(tty_reader_init->env); + enif_free(tty_reader_init); + return (void*)0; +} + +#endif + +static ERL_NIF_TERM tty_select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + TTYResource *tty; +#ifdef THREADED_READER + struct tty_reader_init *tty_reader_init; +#endif +#ifndef __WIN32__ + extern int using_oldshell; /* set this to let the rest of erts know */ +#endif + if (!enif_get_resource(env, argv[0], tty_rt, (void **)&tty)) + return enif_make_badarg(env); + +#ifndef __WIN32__ + if (pipe(tty->signal) == -1) { + return make_errno_error(env, "pipe"); + } + SET_NONBLOCKING(tty->signal[0]); + enif_select(env, tty->signal[0], ERL_NIF_SELECT_READ, tty, NULL, argv[1]); + tty_signal_fd = tty->signal[1]; + + sys_signal(SIGCONT, tty_cont); + sys_signal(SIGWINCH, tty_winch); + + using_oldshell = 0; + +#endif + + enif_select(env, tty->ifd, ERL_NIF_SELECT_READ, tty, NULL, argv[2]); + + enif_self(env, &tty->reader); + enif_monitor_process(env, tty, &tty->reader, NULL); + +#ifdef THREADED_READER + + tty_reader_init = enif_alloc(sizeof(struct tty_reader_init)); + tty_reader_init->env = enif_alloc_env(); + tty_reader_init->tty = enif_make_copy(tty_reader_init->env, argv[0]); + + if (enif_thread_create( + "stdin_reader", + &tty->reader_tid, + tty_reader_thread, tty_reader_init, NULL)) { + enif_free(tty_reader_init); + return make_errno_error(env, "enif_thread_create"); + } +#endif + return atom_ok; +} + +static void tty_monitor_down(ErlNifEnv* caller_env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon) { + TTYResource *tty = obj; +#ifdef HAVE_TERMCAP + if (enif_compare_pids(pid, &tty->self) == 0) { + tcsetattr(tty->ifd, TCSANOW, &tty->tty_rmode); + } +#endif + if (enif_compare_pids(pid, &tty->reader) == 0) { + enif_select(caller_env, tty->ifd, ERL_NIF_SELECT_STOP, tty, NULL, atom_undefined); +#ifndef __WIN32__ + enif_select(caller_env, tty->signal[0], ERL_NIF_SELECT_STOP, tty, NULL, atom_undefined); + close(tty->signal[1]); + sys_signal(SIGCONT, SIG_DFL); + sys_signal(SIGWINCH, SIG_DFL); +#endif + } +} + +static void tty_select_stop(ErlNifEnv* caller_env, void* obj, ErlNifEvent event, int is_direct_call) { +/* Only used to close the signal pipe on unix */ +#ifndef __WIN32__ + if (event != 0) + close(event); +#endif +} + +static void load_resources(ErlNifEnv* env, ErlNifResourceFlags rt_flags) { + ErlNifResourceTypeInit rt = { + NULL /* dtor */, + tty_select_stop, + tty_monitor_down}; + +#define ATOM_DECL(A) atom_##A = enif_make_atom(env, #A) +ATOMS +#undef ATOM_DECL + + tty_rt = enif_open_resource_type_x(env, "tty", &rt, rt_flags, NULL); +} + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + *priv_data = NULL; + load_resources(env, ERL_NIF_RT_CREATE); + return 0; +} + +static void unload(ErlNifEnv* env, void* priv_data) +{ + +} + +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, + ERL_NIF_TERM load_info) +{ + if (*old_priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if (*priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + *priv_data = NULL; + load_resources(env, ERL_NIF_RT_TAKEOVER); + return 0; +} diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 1b12ed3293..fe35dbd211 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1918,6 +1918,7 @@ erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time, int poll_only /* fallthrough */ case ERTS_EV_TYPE_NONE: /* Deselected ... */ case_ERTS_EV_TYPE_NONE: + state->flags &= ~ERTS_EV_FLAG_FALLBACK; ASSERT(!state->events && !state->active_events && !state->flags); check_fd_cleanup(state, &free_select, &free_nif); break; diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 0a2ac9abf7..ae269292e5 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -58,6 +58,7 @@ #include <sys/wait.h> #include <sys/types.h> #include <sys/socket.h> +#include <termios.h> #define WANT_NONBLOCKING @@ -432,6 +433,17 @@ static int system_properties_fd(void) } #endif /* __ANDROID__ */ +/* + If beam is terminated using kill -9 or Ctrl-C when +B is set it may not + cleanup the terminal properly. So to clean it up we save the initial state in + erl_child_setup and then reset the terminal if we detect that beam terminated. + + Not all shells and OSs have this issue, but we do it on all unixes anyway as + it is hard for us to know where the bug exists or not and there is no hard in + doing it. + */ +static struct termios initial_tty_mode; + int main(int argc, char *argv[]) { @@ -447,6 +459,10 @@ main(int argc, char *argv[]) ABORT("Invalid arguments to child_setup"); } + if (isatty(0)) { + tcgetattr(0,&initial_tty_mode); + } + /* We close all fds except the uds from beam. All other fds from now on will have the CLOEXEC flags set on them. This means that we @@ -541,12 +557,18 @@ main(int argc, char *argv[]) pipes, 3, MSG_DONTWAIT)) < 0) { if (errno == EINTR) continue; + if (isatty(0)) { + tcsetattr(0,TCSANOW,&initial_tty_mode); + } DEBUG_PRINT("erl_child_setup failed to read from uds: %d, %d", res, errno); _exit(0); } if (res == 0) { DEBUG_PRINT("uds was closed!"); + if (isatty(0)) { + tcsetattr(0,TCSANOW,&initial_tty_mode); + } _exit(0); } /* Since we use unix domain sockets and send the entire data in diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index c3505eddc5..5f269ea938 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -32,7 +32,6 @@ #include "erl_sys_driver.h" #include "global.h" #include "erl_threads.h" -#include "../../drivers/win32/win_con.h" #include "erl_cpu_topology.h" #include <malloc.h> @@ -125,8 +124,6 @@ BOOL WINAPI ctrl_handler(DWORD dwCtrlType); static int max_files = 1024; static BOOL use_named_pipes; -static BOOL win_console = FALSE; - static OSVERSIONINFO int_os_version; /* Version information for Win32. */ @@ -205,10 +202,6 @@ erts_sys_misc_mem_sz(void) */ void sys_tty_reset(int exit_code) { - if (exit_code == ERTS_ERROR_EXIT) - ConWaitForExit(); - else - ConNormalExit(); } void erl_sys_args(int* argc, char** argv) @@ -304,25 +297,16 @@ int erts_set_signal(Eterm signal, Eterm type) { return 0; } +static DWORD dwOriginalOutMode = 0; +static DWORD dwOriginalInMode = 0; + static void init_console(void) { - char* mode = erts_read_env("ERL_CONSOLE_MODE"); - - if (!mode || strcmp(mode, "window") == 0) { - win_console = TRUE; - ConInit(); - /*nohup = 0;*/ - } else if (strncmp(mode, "tty:", 4) == 0) { - if (mode[5] == 'c') { - setvbuf(stdout, NULL, _IONBF, 0); - } - if (mode[6] == 'c') { - setvbuf(stderr, NULL, _IONBF, 0); - } - } - - erts_free_read_env(mode); + GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dwOriginalOutMode); + GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &dwOriginalInMode); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); } int sys_max_files(void) @@ -2194,7 +2178,6 @@ fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) return ERL_DRV_ERROR_GENERAL; } - fd_driver_input = &(dp->in); dp->in.flags = DF_XLAT_CR; if (is_std_error) { dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror @@ -2202,6 +2185,7 @@ fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) } if ( in == 0 && out == 1) { + fd_driver_input = &(dp->in); save_01_port = dp; } else if (in == 2 && out == 2) { save_22_port = dp; @@ -2945,10 +2929,6 @@ sys_get_key(int fd) { ASSERT(fd == 0); - if (win_console) { - return ConGetKey(); - } - /* * Black magic follows. (Code stolen from get_overlapped_result()) */ @@ -2974,6 +2954,32 @@ sys_get_key(int fd) } } } + else { + char c[64]; + DWORD dwBytesRead, dwCurrentOutMode = 0, dwCurrentInMode = 0; + + /* Get current console information */ + GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dwCurrentOutMode); + GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &dwCurrentInMode); + + /* Set the a "oldstyle" terminal with line input that we can use ReadFile on */ + SetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), dwOriginalOutMode); + SetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), + ENABLE_PROCESSED_INPUT | + ENABLE_LINE_INPUT | + ENABLE_ECHO_INPUT | + ENABLE_INSERT_MODE | + ENABLE_QUICK_EDIT_MODE | + ENABLE_AUTO_POSITION + ); + + if (ReadFile(GetStdHandle(STD_INPUT_HANDLE), &c, sizeof(c), &dwBytesRead, NULL) && dwBytesRead > 0) { + /* Restore original console information */ + SetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), dwCurrentOutMode); + SetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), dwCurrentInMode); + return c[0]; + } + } return '*'; /* Error! */ } diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index cee269eed4..b8821e47aa 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -23,11 +23,13 @@ #ifdef HAVE_CONFIG_H # include "config.h" #endif + +#define ERTS_WANT_BREAK_HANDLING + #include "sys.h" #include "erl_alloc.h" #include "erl_thr_progress.h" #include "erl_driver.h" -#include "../../drivers/win32/win_con.h" #if defined(__GNUC__) # define WIN_SYS_INLINE __inline__ @@ -82,7 +84,6 @@ BOOL WINAPI ctrl_handler_ignore_break(DWORD dwCtrlType) } void erts_set_ignore_break(void) { - ConSetCtrlHandler(ctrl_handler_ignore_break); SetConsoleCtrlHandler(ctrl_handler_ignore_break, TRUE); } @@ -92,6 +93,9 @@ BOOL WINAPI ctrl_handler_replace_intr(DWORD dwCtrlType) case CTRL_C_EVENT: return FALSE; case CTRL_BREAK_EVENT: + if (ERTS_BREAK_REQUESTED) { + erts_exit(ERTS_INTR_EXIT, ""); + } SetEvent(erts_sys_break_event); break; case CTRL_LOGOFF_EVENT: @@ -110,7 +114,11 @@ BOOL WINAPI ctrl_handler_replace_intr(DWORD dwCtrlType) /* Don't use ctrl-c for break handler but let it be used by the shell instead (see user_drv.erl) */ void erts_replace_intr(void) { - ConSetCtrlHandler(ctrl_handler_replace_intr); + HANDLE hIn = GetStdHandle(STD_INPUT_HANDLE); + DWORD dwOriginalInMode = 0; + if (GetConsoleMode(hIn, &dwOriginalInMode)) { + SetConsoleMode(hIn, dwOriginalInMode & ~ENABLE_PROCESSED_INPUT); + } SetConsoleCtrlHandler(ctrl_handler_replace_intr, TRUE); } @@ -119,6 +127,9 @@ BOOL WINAPI ctrl_handler(DWORD dwCtrlType) switch (dwCtrlType) { case CTRL_C_EVENT: case CTRL_BREAK_EVENT: + if (ERTS_BREAK_REQUESTED) { + erts_exit(ERTS_INTR_EXIT, ""); + } SetEvent(erts_sys_break_event); break; case CTRL_LOGOFF_EVENT: @@ -135,7 +146,6 @@ BOOL WINAPI ctrl_handler(DWORD dwCtrlType) void init_break_handler() { - ConSetCtrlHandler(ctrl_handler); SetConsoleCtrlHandler(ctrl_handler, TRUE); } diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 33a80f2e53..40e699e8ab 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -149,6 +149,16 @@ NO_OPT= bs_bincomp \ guard \ map +R25= \ + bs_bincomp \ + bs_construct \ + bs_match_bin \ + bs_match_int \ + bs_match_tail \ + bs_match_misc \ + bs_utf + + NATIVE= hibernate NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE) @@ -157,6 +167,9 @@ NO_OPT_ERL_FILES= $(NO_OPT_MODULES:%=%.erl) NATIVE_MODULES= $(NATIVE:%=%_native_SUITE) NATIVE_ERL_FILES= $(NATIVE_MODULES:%=%.erl) +R25_MODULES= $(R25:%=%_r25_SUITE) +R25_ERL_FILES= $(R25_MODULES:%=%.erl) + ERL_FILES= $(MODULES:%=%.erl) TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) @@ -188,13 +201,17 @@ ERL_COMPILE_FLAGS := $(filter-out +deterministic,$($(ERL_COMPILE_FLAGS))) # Targets # ---------------------------------------------------- -make_emakefile: $(NO_OPT_ERL_FILES) $(NATIVE_ERL_FILES) $(KERNEL_ERL_FILES) +make_emakefile: $(NO_OPT_ERL_FILES) $(NATIVE_ERL_FILES) \ + $(KERNEL_ERL_FILES) $(R25_ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) +compressed -o$(EBIN) \ $(MODULES) $(KERNEL_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt +no_postopt +no_ssa_opt +no_bsm_opt \ $(ERL_COMPILE_FLAGS) -o$(EBIN) $(NO_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(NATIVE_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +r25 \ + $(ERL_COMPILE_FLAGS) -o$(EBIN) $(R25_MODULES) >> $(EMAKEFILE) + tests debug opt: make_emakefile erl $(ERL_MAKE_FLAGS) -make @@ -218,6 +235,9 @@ targets: $(TARGET_FILES) %_native_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_r25_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- @@ -232,6 +252,7 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) $(NO_OPT_ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(NATIVE_ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(KERNEL_ERL_FILES) "$(RELSYSDIR)" + $(INSTALL_DATA) $(R25_ERL_FILES) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 7bc8c0b565..14559e4c64 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -28,7 +28,7 @@ huge_float_field/1, system_limit/1, badarg/1, copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1, otp_7422/1, zero_width/1, bad_append/1, bs_append_overflow/1, - reductions/1, fp16/1, error_info/1]). + reductions/1, fp16/1, zero_init/1, error_info/1, little/1]). -include_lib("common_test/include/ct.hrl"). @@ -41,7 +41,8 @@ all() -> in_guard, mem_leak, coerce_to_float, bjorn, append_empty_is_same, huge_float_field, system_limit, badarg, copy_writable_binary, kostis, dynamic, bs_add, otp_7422, zero_width, - bad_append, bs_append_overflow, reductions, fp16, error_info]. + bad_append, bs_append_overflow, reductions, fp16, zero_init, + error_info, little]. init_per_suite(Config) -> Config. @@ -49,14 +50,6 @@ init_per_suite(Config) -> end_per_suite(_Config) -> application:stop(os_mon). -big(1) -> - 57285702734876389752897683. - -i(X) -> X. - -r(L) -> - lists:reverse(L). - -define(T(B, L), {B, ??B, L}). -define(N(B), {B, ??B, unknown}). @@ -89,7 +82,7 @@ l(I_13, I_big1) -> ?T(<<57285702734876389752897684:32>>, [138, 99, 0, 148]), ?T(<<I_big1:32/little>>, - r([138, 99, 0, 147])), + lists:reverse([138, 99, 0, 147])), ?T(<<-1:17/unit:8>>, lists:duplicate(17, 255)), @@ -111,6 +104,9 @@ l(I_13, I_big1) -> ?T(<<4,3,<<1,2>>:1/binary>>, [4,3,1]), + ?T(<< <<153,27:5>>:I_13/bits, 1:3 >>, + [153,217]), + ?T(<<(256*45+47)>>, [47]), @@ -142,9 +138,74 @@ l(I_13, I_big1) -> ?T(<<<<5:3>>/bitstring>>, <<5:3>>), ?T(<<42,<<7:4>>/binary-unit:4>>, <<42,7:4>>), ?T(<<<<344:17>>/binary-unit:17>>, <<344:17>>), - ?T(<<<<42,3,7656:16>>/binary-unit:16>>, <<42,3,7656:16>>) - - ]. + ?T(<<<<42,3,7656:16>>/binary-unit:16>>, <<42,3,7656:16>>), + + %% Different sizes and types. First without types. + ?T(<<I_big1:8>>, [147]), + ?T(<<I_big1:16>>, [0, 147]), + ?T(<<I_big1:24>>, [99, 0, 147]), + ?T(<<I_big1:32>>, [138, 99, 0, 147]), + ?T(<<I_big1:40>>, [5, 138, 99, 0, 147]), + ?T(<<I_big1:48>>, [229, 5, 138, 99, 0, 147]), + ?T(<<I_big1:56>>, [249, 229, 5, 138, 99, 0, 147]), + ?T(<<I_big1:64>>, [42, 249, 229, 5, 138, 99, 0, 147]), + + %% Known integer with range. + ?T(<<(I_big1 band ((1 bsl 56) - 1)):8>>, [147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):16>>, [0, 147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):24>>, [99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):32>>, [138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):40>>, [5, 138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):48>>, [229, 5, 138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 56) - 1)):56>>, [249, 229, 5, 138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 64) - 1)):64>>, [42, 249, 229, 5, 138, 99, 0, 147]), + + %% Known integer with exact range. + ?T(<<(I_big1 band ((1 bsl 8) - 1)):8>>, [147]), + ?T(<<(I_big1 band ((1 bsl 16) - 1)):16>>, [0, 147]), + ?T(<<(I_big1 band ((1 bsl 24) - 1)):24>>, [99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 32) - 1)):32>>, [138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 40) - 1)):40>>, [5, 138, 99, 0, 147]), + ?T(<<(I_big1 band ((1 bsl 48) - 1)):48>>, [229, 5, 138, 99, 0, 147]), + + %% Known integer without range. + ?T(<<(I_big1 + 0):8>>, [147]), + ?T(<<(I_big1 + 0):16>>, [0, 147]), + ?T(<<(I_big1 + 0):24>>, [99, 0, 147]), + ?T(<<(I_big1 + 0):32>>, [138, 99, 0, 147]), + ?T(<<(I_big1 + 0):40>>, [5, 138, 99, 0, 147]), + ?T(<<(I_big1 + 0):48>>, [229, 5, 138, 99, 0, 147]), + ?T(<<(I_big1 + 0):56>>, [249, 229, 5, 138, 99, 0, 147]), + ?T(<<(I_big1 + 0):64>>, [42, 249, 229, 5, 138, 99, 0, 147]), + + %% Known integer. Verify that the value does not bleed into the + %% previous segment. + ?T(<<1, (I_big1 + 0):8>>, [1, 147]), + ?T(<<2, (I_big1 + 0):16>>, [2, 0, 147]), + ?T(<<3, (I_big1 + 0):24>>, [3, 99, 0, 147]), + ?T(<<4, (I_big1 + 0):32>>, [4, 138, 99, 0, 147]), + ?T(<<5, (I_big1 + 0):40>>, [5, 5, 138, 99, 0, 147]), + ?T(<<6, (I_big1 + 0):48>>, [6, 229, 5, 138, 99, 0, 147]), + ?T(<<7, (I_big1 + 0):56>>, [7, 249, 229, 5, 138, 99, 0, 147]), + ?T(<<8, (I_big1 + 0):64>>, [8, 42, 249, 229, 5, 138, 99, 0, 147]), + + %% Test non-byte sizes. + ?T(<<I_big1:33>>, <<197,49,128,73,1:1>>), + ?T(<<I_big1:39>>, <<11,20,198,1,19:7>>), + + ?T(<<I_big1:57>>, <<124,242,130,197,49,128,73,1:1>>), + ?T(<<I_big1:58>>, <<190,121,65,98,152,192,36,3:2>>), + ?T(<<I_big1:59>>, <<95,60,160,177,76,96,18,3:3>>), + ?T(<<I_big1:60>>, <<175,158,80,88,166,48,9,3:4>>), + ?T(<<I_big1:61>>, <<87,207,40,44,83,24,4,19:5>>), + ?T(<<I_big1:62>>, <<171,231,148,22,41,140,2,19:6>>), + ?T(<<I_big1:63>>, <<85,243,202,11,20,198,1,19:7>>), + + %% Test non-byte sizes and also that the value does not bleed + %% into the previous segment. + ?T(<<17, I_big1:33>>, <<17, 197,49,128,73,1:1>>), + ?T(<<19, I_big1:39>>, <<19, 11,20,198,1,19:7>>) + ]. native_3798() -> case <<1:16/native>> of @@ -274,10 +335,10 @@ fail_check(Res, _, _) -> %%% Simple working cases test1(Config) when is_list(Config) -> - I_13 = i(13), - I_big1 = big(1), + I_13 = id(13), + I_big1 = id(57285702734876389752897683), Vars = [{'I_13', I_13}, - {'I_big1', I_big1}], + {'I_big1', I_big1}], lists:foreach(fun one_test/1, eval_list(l(I_13, I_big1), Vars)). %%% Misc @@ -717,6 +778,21 @@ dynamic_big(Bef, N, Int, Lpad, Rpad) -> Bin = id(<<Lpad:Bef,NumBin/bitstring,Rpad:(128-Bef-N)>>), Bin = <<Lpad:Bef,Int:N,Rpad:(128-Bef-N)>>, + %% Units are seldom used with integer segments even in our test + %% suites, and I have never seen non-power-of-two units in + %% production code. + if + Bef rem 8 =:= 0 -> + Bin = <<Lpad:(Bef div 8)/unit:8,Int:N,Rpad:(128-Bef-N)>>; + Bef rem 7 =:= 0 -> + Bin = <<Lpad:(Bef div 7)/unit:7,Int:N,Rpad:(128-Bef-N)>>; + (128-Bef-N) rem 5 =:= 0 -> + Aft = (128 - Bef - N) div 5, + Bin = <<Lpad:Bef,Int:N,Rpad:Aft/unit:5>>; + true -> + ok + end, + %% Further verify the result by matching. LpadMasked = Lpad band ((1 bsl Bef) - 1), RpadMasked = Rpad band ((1 bsl (128-Bef-N)) - 1), @@ -733,6 +809,20 @@ dynamic_little(Bef, N, Int, Lpad, Rpad) -> Bin = id(<<Lpad:Bef/little,NumBin/bitstring,Rpad:(128-Bef-N)/little>>), Bin = <<Lpad:Bef/little,Int:N/little,Rpad:(128-Bef-N)/little>>, + if + Bef rem 8 =:= 0 -> + Bin = <<Lpad:(Bef div 8)/little-unit:8, + Int:N/little,Rpad:(128-Bef-N)/little>>; + Bef rem 9 =:= 0 -> + Bin = <<Lpad:(Bef div 9)/little-unit:9, + Int:N/little,Rpad:(128-Bef-N)/little>>; + (128-Bef-N) rem 17 =:= 0 -> + Aft = (128 - Bef - N) div 17, + Bin = <<Lpad:Bef/little,Int:N/little,Rpad:Aft/little-unit:17>>; + true -> + ok + end, + %% Further verify the result by matching. LpadMasked = Lpad band ((1 bsl Bef) - 1), RpadMasked = Rpad band ((1 bsl (128-Bef-N)) - 1), @@ -742,7 +832,8 @@ dynamic_little(Bef, N, Int, Lpad, Rpad) -> %% Test that the bs_add/5 instruction handles big numbers correctly. bs_add(Config) when is_list(Config) -> - Mod = bs_construct_bs_add, + Mod = list_to_atom(atom_to_list(?MODULE) ++ "_" ++ + atom_to_list(?FUNCTION_NAME)), N = 2000, Code = [{module, Mod}, {exports, [{bs_add,2}]}, @@ -793,9 +884,12 @@ bs_add(Config) when is_list(Config) -> %% Clean up. ok = file:delete(AsmFile), ok = file:delete(code:which(Mod)), + _ = code:delete(Mod), + _ = code:purge(Mod), + ok. - + smallest_big() -> smallest_big_1(1 bsl 24). @@ -1005,6 +1099,233 @@ fp16(_Config) -> ?FP16(16#4000, 2.0), ok. +zero_init(_Config) -> + <<LPad:64/bits,RPad:64/bits>> = id(erlang:md5([42])), + Sizes = [511,512,513, 767,768,769, 1023,1024,1025, + 16#7fff,16#ffff,16#10000] ++ lists:seq(0, 257), + _ = [do_zero_init(Size, LPad, RPad) || Size <- Sizes], + ok. + +do_zero_init(Size, LPad, RPad) -> + try + do_zero_init_1(Size, LPad, RPad) + catch + C:R:Stk -> + io:format("Size = ~p, LPad = ~p, RPad = ~p\n", [Size, LPad, RPad]), + erlang:raise(C, R, Stk) + end. + +do_zero_init_1(Size, LPad, RPad) -> + Zeroes = id(<<0:Size>>), + <<0:Size>> = Zeroes, + Bin = id(<<LPad:64/bits, Zeroes/bits, RPad:64/bits>>), + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>), + Bin = id(<<LPad:(id(64))/bits, 0:Size, RPad:64/bits>>), + + if + Size rem 11 =:= 0 -> + Bin = id(<<LPad:64/bits, 0:(Size div 11)/unit:11, RPad:64/bits>>); + true -> + ok + end, + + case Size of + 0 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 1 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 2 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 3 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 4 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 5 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 6 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 7 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 8 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 9 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 10 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 11 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 12 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 13 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 14 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 15 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 16 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 31 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 32 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 33 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 47 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 48 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 49 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 63 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 64 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 65 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 79 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 80 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 81 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 90 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 91 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 92 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 93 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 94 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 95 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 96 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 97 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 98 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 99 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 100 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 101 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 102 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 103 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 104 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 105 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 106 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 107 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 108 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 109 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 127 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 128 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 129 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 130 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 131 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 132 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 133 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 134 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 135 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 136 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 137 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 138 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 139 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 140 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 141 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 142 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 143 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 144 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 145 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 159 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 160 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 161 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 191 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 192 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 193 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 255 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 256 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 257 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 511 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 512 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 513 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 767 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 768 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 769 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 1023 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 1024 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 1025 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + + 16#7fff -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 16#ffff -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + 16#10000 -> + Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>); + _ -> + ok + end. + + -define(ERROR_INFO(Expr), fun() -> try Expr of @@ -1144,6 +1465,161 @@ error_info_verify(Reason, Stk0, Expr, Overrides) -> io:format("~ts: ~ts\n", [Expr,String]), {Reason, Cause, String}. +little(_Config) -> + _ = rand:uniform(), %Seed generator + io:format("Seed: ~p", [rand:export_seed()]), + + RandBytes = rand:bytes(10), + _ = [do_little(RandBytes, N) || N <- lists:seq(0, 10*8)], + + ok. + +do_little(Bin0, N) -> + <<Bin1:N/bits,Bin2/bits>> = Bin0, + Bin2Size = bit_size(Bin2), + + <<I1:N/little-integer>> = Bin1, + <<I2:Bin2Size/little-integer>> = Bin2, + + Bin1 = <<I1:N/little-integer>>, + Bin1 = do_little_1(N, id(I1)), + + Bin2 = <<I2:Bin2Size/little-integer>>, + Bin2 = do_little_1(Bin2Size, id(I2)), + + ok. + +do_little_1(0, I) -> <<I:0/little-integer>>; +do_little_1(1, I) -> <<I:1/little-integer>>; +do_little_1(2, I) -> <<I:2/little-integer>>; +do_little_1(3, I) -> <<I:3/little-integer>>; +do_little_1(4, I) -> <<I:4/little-integer>>; +do_little_1(5, I) -> <<I:5/little-integer>>; +do_little_1(6, I) -> <<I:6/little-integer>>; +do_little_1(7, I) -> <<I:7/little-integer>>; +do_little_1(8, I) -> <<I:8/little-integer>>; +do_little_1(9, I) -> <<I:9/little-integer>>; +do_little_1(10, I) -> <<I:10/little-integer>>; +do_little_1(11, I) -> <<I:11/little-integer>>; +do_little_1(12, I) -> <<I:12/little-integer>>; +do_little_1(13, I) -> <<I:13/little-integer>>; +do_little_1(14, I) -> <<I:14/little-integer>>; +do_little_1(15, I) -> <<I:15/little-integer>>; +do_little_1(16, I) -> <<I:16/little-integer>>; +do_little_1(17, I) -> <<I:17/little-integer>>; +do_little_1(18, I) -> <<I:18/little-integer>>; +do_little_1(19, I) -> <<I:19/little-integer>>; +do_little_1(20, I) -> <<I:20/little-integer>>; +do_little_1(21, I) -> <<I:21/little-integer>>; +do_little_1(22, I) -> <<I:22/little-integer>>; +do_little_1(23, I) -> <<I:23/little-integer>>; +do_little_1(24, I) -> <<I:24/little-integer>>; +do_little_1(25, I) -> <<I:25/little-integer>>; +do_little_1(26, I) -> <<I:26/little-integer>>; +do_little_1(27, I) -> <<I:27/little-integer>>; +do_little_1(28, I) -> <<I:28/little-integer>>; +do_little_1(29, I) -> <<I:29/little-integer>>; +do_little_1(30, I) -> <<I:30/little-integer>>; +do_little_1(31, I) -> <<I:31/little-integer>>; +do_little_1(32, I) -> <<I:32/little-integer>>; +do_little_1(33, I) -> <<I:33/little-integer>>; +do_little_1(34, I) -> <<I:34/little-integer>>; +do_little_1(35, I) -> <<I:35/little-integer>>; +do_little_1(36, I) -> <<I:36/little-integer>>; +do_little_1(37, I) -> <<I:37/little-integer>>; +do_little_1(38, I) -> <<I:38/little-integer>>; +do_little_1(39, I) -> <<I:39/little-integer>>; +do_little_1(40, I) -> <<I:40/little-integer>>; +do_little_1(41, I) -> <<I:41/little-integer>>; +do_little_1(42, I) -> <<I:42/little-integer>>; +do_little_1(43, I) -> <<I:43/little-integer>>; +do_little_1(44, I) -> <<I:44/little-integer>>; +do_little_1(45, I) -> <<I:45/little-integer>>; +do_little_1(46, I) -> <<I:46/little-integer>>; +do_little_1(47, I) -> <<I:47/little-integer>>; +do_little_1(48, I) -> <<I:48/little-integer>>; +do_little_1(49, I) -> <<I:49/little-integer>>; +do_little_1(50, I) -> <<I:50/little-integer>>; +do_little_1(51, I) -> <<I:51/little-integer>>; +do_little_1(52, I) -> <<I:52/little-integer>>; +do_little_1(53, I) -> <<I:53/little-integer>>; +do_little_1(54, I) -> <<I:54/little-integer>>; +do_little_1(55, I) -> <<I:55/little-integer>>; +do_little_1(56, I) -> <<I:56/little-integer>>; +do_little_1(57, I) -> <<I:57/little-integer>>; +do_little_1(58, I) -> <<I:58/little-integer>>; +do_little_1(59, I) -> <<I:59/little-integer>>; +do_little_1(60, I) -> <<I:60/little-integer>>; +do_little_1(61, I) -> <<I:61/little-integer>>; +do_little_1(62, I) -> <<I:62/little-integer>>; +do_little_1(63, I) -> <<I:63/little-integer>>; +do_little_1(64, I) -> <<I:64/little-integer>>; +do_little_1(65, I) -> <<I:65/little-integer>>; +do_little_1(66, I) -> <<I:66/little-integer>>; +do_little_1(67, I) -> <<I:67/little-integer>>; +do_little_1(68, I) -> <<I:68/little-integer>>; +do_little_1(69, I) -> <<I:69/little-integer>>; +do_little_1(70, I) -> <<I:70/little-integer>>; +do_little_1(71, I) -> <<I:71/little-integer>>; +do_little_1(72, I) -> <<I:72/little-integer>>; +do_little_1(73, I) -> <<I:73/little-integer>>; +do_little_1(74, I) -> <<I:74/little-integer>>; +do_little_1(75, I) -> <<I:75/little-integer>>; +do_little_1(76, I) -> <<I:76/little-integer>>; +do_little_1(77, I) -> <<I:77/little-integer>>; +do_little_1(78, I) -> <<I:78/little-integer>>; +do_little_1(79, I) -> <<I:79/little-integer>>; +do_little_1(80, I) -> <<I:80/little-integer>>; +do_little_1(81, I) -> <<I:81/little-integer>>; +do_little_1(82, I) -> <<I:82/little-integer>>; +do_little_1(83, I) -> <<I:83/little-integer>>; +do_little_1(84, I) -> <<I:84/little-integer>>; +do_little_1(85, I) -> <<I:85/little-integer>>; +do_little_1(86, I) -> <<I:86/little-integer>>; +do_little_1(87, I) -> <<I:87/little-integer>>; +do_little_1(88, I) -> <<I:88/little-integer>>; +do_little_1(89, I) -> <<I:89/little-integer>>; +do_little_1(90, I) -> <<I:90/little-integer>>; +do_little_1(91, I) -> <<I:91/little-integer>>; +do_little_1(92, I) -> <<I:92/little-integer>>; +do_little_1(93, I) -> <<I:93/little-integer>>; +do_little_1(94, I) -> <<I:94/little-integer>>; +do_little_1(95, I) -> <<I:95/little-integer>>; +do_little_1(96, I) -> <<I:96/little-integer>>; +do_little_1(97, I) -> <<I:97/little-integer>>; +do_little_1(98, I) -> <<I:98/little-integer>>; +do_little_1(99, I) -> <<I:99/little-integer>>; +do_little_1(100, I) -> <<I:100/little-integer>>; +do_little_1(101, I) -> <<I:101/little-integer>>; +do_little_1(102, I) -> <<I:102/little-integer>>; +do_little_1(103, I) -> <<I:103/little-integer>>; +do_little_1(104, I) -> <<I:104/little-integer>>; +do_little_1(105, I) -> <<I:105/little-integer>>; +do_little_1(106, I) -> <<I:106/little-integer>>; +do_little_1(107, I) -> <<I:107/little-integer>>; +do_little_1(108, I) -> <<I:108/little-integer>>; +do_little_1(109, I) -> <<I:109/little-integer>>; +do_little_1(110, I) -> <<I:110/little-integer>>; +do_little_1(111, I) -> <<I:111/little-integer>>; +do_little_1(112, I) -> <<I:112/little-integer>>; +do_little_1(113, I) -> <<I:113/little-integer>>; +do_little_1(114, I) -> <<I:114/little-integer>>; +do_little_1(115, I) -> <<I:115/little-integer>>; +do_little_1(116, I) -> <<I:116/little-integer>>; +do_little_1(117, I) -> <<I:117/little-integer>>; +do_little_1(118, I) -> <<I:118/little-integer>>; +do_little_1(119, I) -> <<I:119/little-integer>>; +do_little_1(120, I) -> <<I:120/little-integer>>; +do_little_1(121, I) -> <<I:121/little-integer>>; +do_little_1(122, I) -> <<I:122/little-integer>>; +do_little_1(123, I) -> <<I:123/little-integer>>; +do_little_1(124, I) -> <<I:124/little-integer>>; +do_little_1(125, I) -> <<I:125/little-integer>>; +do_little_1(126, I) -> <<I:126/little-integer>>; +do_little_1(127, I) -> <<I:127/little-integer>>; +do_little_1(128, I) -> <<I:128/little-integer>>. + + %%% %%% Common utilities. %%% diff --git a/erts/emulator/test/bs_match_bin_SUITE.erl b/erts/emulator/test/bs_match_bin_SUITE.erl index a7f5ad2d6b..68119b138c 100644 --- a/erts/emulator/test/bs_match_bin_SUITE.erl +++ b/erts/emulator/test/bs_match_bin_SUITE.erl @@ -23,17 +23,17 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, byte_split_binary/1,bit_split_binary/1,match_huge_bin/1, - bs_match_string_edge_case/1]). + bs_match_string_edge_case/1,contexts/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [byte_split_binary, bit_split_binary, match_huge_bin, - bs_match_string_edge_case]. + bs_match_string_edge_case, contexts]. -groups() -> +groups() -> []. init_per_suite(Config) -> @@ -235,3 +235,38 @@ bs_match_string_edge_case(_Config) -> <<?MATCH512, " ", Tail1/binary>> = Bin, <<" ", Tail1/binary>> = id(Tail0), ok. + +contexts(_Config) -> + Bytes = rand:bytes(12), + _ = [begin + <<B:N/binary,_/binary>> = Bytes, + B = id(get_binary(B)) + end || N <- lists:seq(0, 12)], + ok. + +get_binary(Bin) -> + [A,B,C,D,E,F] = id([1,2,3,4,5,6]), + {Res,_} = get_binary_memory_ctx(A, B, C, D, E, F, Bin), + Res. + + +get_binary_memory_ctx(A, B, C, D, E, F, Bin) -> + %% The match context will be in {x,6}, which is not + %% a X register backed by a CPU register on any platform. + Res = case Bin of + <<Res0:0/binary>> -> Res0; + <<Res0:1/binary>> -> Res0; + <<Res0:2/binary>> -> Res0; + <<Res0:3/binary>> -> Res0; + <<Res0:4/binary>> -> Res0; + <<Res0:5/binary>> -> Res0; + <<Res0:6/binary>> -> Res0; + <<Res0:7/binary>> -> Res0; + <<Res0:8/binary>> -> Res0; + <<Res0:9/binary>> -> Res0; + <<Res0:10/binary>> -> Res0; + <<Res0:11/binary>> -> Res0; + <<Res0:12/binary>> -> Res0 + end, + {Res,{A,B,C,D,E,F}}. + diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl index 0268ba18c8..48775135dc 100644 --- a/erts/emulator/test/bs_match_int_SUITE.erl +++ b/erts/emulator/test/bs_match_int_SUITE.erl @@ -19,10 +19,10 @@ -module(bs_match_int_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1, - match_huge_int/1,bignum/1,unaligned_32_bit/1]). + integer/1,mixed_sizes/1,signed_integer/1,dynamic/1,more_dynamic/1, + mml/1,match_huge_int/1,bignum/1,unaligned_32_bit/1]). -include_lib("common_test/include/ct.hrl"). @@ -30,11 +30,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [integer, signed_integer, dynamic, more_dynamic, mml, +all() -> + [integer, mixed_sizes, signed_integer, dynamic, more_dynamic, mml, match_huge_int, bignum, unaligned_32_bit]. -groups() -> +groups() -> []. init_per_suite(Config) -> @@ -51,6 +51,9 @@ end_per_group(_GroupName, Config) -> integer(Config) when is_list(Config) -> + _ = rand:uniform(), %Seed generator + io:format("Seed: ~p", [rand:export_seed()]), + 0 = get_int(mkbin([])), 0 = get_int(mkbin([0])), 42 = get_int(mkbin([42])), @@ -58,36 +61,628 @@ integer(Config) when is_list(Config) -> 256 = get_int(mkbin([1,0])), 257 = get_int(mkbin([1,1])), 258 = get_int(mkbin([1,2])), - 258 = get_int(mkbin([1,2])), 65534 = get_int(mkbin([255,254])), 16776455 = get_int(mkbin([255,253,7])), 4245492555 = get_int(mkbin([253,13,19,75])), 4294967294 = get_int(mkbin([255,255,255,254])), 4294967295 = get_int(mkbin([255,255,255,255])), + + 16#cafebeef = get_int(<<16#cafebeef:32>>), + 16#cafebeef42 = get_int(<<16#cafebeef42:40>>), + 16#cafebeeffeed = get_int(<<16#cafebeeffeed:48>>), + 16#cafebeeffeed42 = get_int(<<16#cafebeeffeed42:56>>), + 16#1cafebeeffeed42 = get_int(<<16#1cafebeeffeed42:57>>), + 16#2cafebeeffeed42 = get_int(<<16#2cafebeeffeed42:58>>), + 16#7cafebeeffeed42 = get_int(<<16#7cafebeeffeed42:59>>), + + 16#beefcafefeed = get_int(<<16#beefcafefeed:60>>), + 16#beefcafefeed = get_int(<<16#beefcafefeed:61>>), + 16#beefcafefeed = get_int(<<16#beefcafefeed:62>>), + 16#beefcafefeed = get_int(<<16#beefcafefeed:63>>), + + 16#cafebeeffeed42 = get_int(<<16#cafebeeffeed42:60>>), + 16#cafebeeffeed42 = get_int(<<16#cafebeeffeed42:61>>), + 16#cafebeeffeed42 = get_int(<<16#cafebeeffeed42:62>>), + 16#cafebeeffeed42 = get_int(<<16#cafebeeffeed42:63>>), + + 16#acafebeeffeed42 = get_int(<<16#acafebeeffeed42:60>>), + 16#acafebeeffeed42 = get_int(<<16#acafebeeffeed42:61>>), + 16#acafebeeffeed42 = get_int(<<16#acafebeeffeed42:62>>), + 16#acafebeeffeed42 = get_int(<<16#acafebeeffeed42:63>>), + + 16#cafebeeffeed = get_int(<<16#cafebeeffeed:64>>), + 16#cafebeeffeedface = get_int(<<16#cafebeeffeedface:64>>), + + get_int_roundtrip(rand:bytes(12), 0), + get_int_roundtrip(rand:bytes(12), 0), + Eight = [200,1,19,128,222,42,97,111], cmp128(Eight, uint(Eight)), - fun_clause(catch get_int(mkbin(seq(1,5)))), + fun_clause(catch get_int(mkbin(seq(1, 20)))), + ok. + +get_int_roundtrip(Bin0, Size) when Size =< 8*byte_size(Bin0) -> + <<Bin:Size/bits,_/bits>> = Bin0, + <<I:Size>> = Bin, + I = get_int(Bin), + get_int_roundtrip(Bin0, Size+1); +get_int_roundtrip(_, _) -> ok. + +get_int(Bin0) -> + %% Note that it has become impossible to create a byte-sized sub + %% binary (see erts_extract_sub_binary() in erl_bits.c) of size 64 + %% or less. Therefore, to be able to create an unaligned binary, + %% we'll need to base it on on a binary with more than 64 bytes. + Size = bit_size(Bin0), + Filler = rand:bytes(65), + UnsignedBigBin = id(<<Filler/binary,Bin0/bits>>), + I = get_unsigned_big(UnsignedBigBin), + + %% io:format("~p ~p\n", [Size,I]), + if + Size =< 10*8 -> + OversizedUnsignedBig = id(<<Filler/binary,0:16,Bin0/bits>>), + I = get_unsigned_big(OversizedUnsignedBig); + true -> + ok + end, + + test_unaligned(UnsignedBigBin, I, fun get_unsigned_big/1), + + %% Test unsigned little-endian integers. + + UnsignedLittleBin = id(<<Filler/binary,I:Size/little>>), + I = get_unsigned_little(UnsignedLittleBin), + + test_unaligned(UnsignedLittleBin, I, fun get_unsigned_little/1), + + %% Test signed big-endian integers. + + SignedBigBin1 = id(<<Filler/binary,(-I):(Size+1)/big>>), + I = -get_signed_big(SignedBigBin1), + + SignedBigBin2 = id(<<Filler/binary,I:(Size+1)/big>>), + I = get_signed_big(SignedBigBin2), + + %% test_unaligned(SignedBigBin1, -I, fun get_signed_big/1), + test_unaligned(SignedBigBin2, I, fun get_signed_big/1), + + %% Test signed little-endian integers. + + SignedLittleBin1 = id(<<Filler/binary,(-I):(Size+1)/little>>), + I = -get_signed_little(SignedLittleBin1), + + SignedLittleBin2 = id(<<Filler/binary,I:(Size+1)/little>>), + I = get_signed_little(SignedLittleBin2), + + test_unaligned(SignedLittleBin1, -I, fun get_signed_little/1), + test_unaligned(SignedLittleBin2, I, fun get_signed_little/1), + + I. + +test_unaligned(Bin, I, Matcher) -> + <<RandBytes1:8/binary,RandBytes2:8/binary>> = rand:bytes(16), + Size = bit_size(Bin), + _ = [begin + <<_:(Offset+32),UnalignedBin:Size/bits,_/bits>> = + id(<<RandBytes1:(Offset+32)/bits, + Bin:Size/bits, + RandBytes2/binary>>), + I = Matcher(UnalignedBin) + end || Offset <- [1,2,3,4,5,6,7]], ok. -get_int(Bin) -> - I = get_int1(Bin), - get_int(Bin, I). -get_int(Bin0, I) when size(Bin0) < 4 -> - Bin = <<0,Bin0/binary>>, - I = get_int1(Bin), - get_int(Bin, I); -get_int(_, I) -> I. -get_int1(<<I:0>>) -> I; -get_int1(<<I:8>>) -> I; -get_int1(<<I:16>>) -> I; -get_int1(<<I:24>>) -> I; -get_int1(<<I:32>>) -> I. +get_unsigned_big(Bin) -> + Res = get_unsigned_big_plain(Bin), + [A,B,C,D,E] = id([1,2,3,4,5]), + {Res,_} = get_unsigned_big_memory_ctx(A, B, C, D, E, Res, Bin), + Res. + +get_unsigned_big_memory_ctx(A, B, C, D, E, Res0, Bin) -> + %% The match context will be in {x,6}, which is not + %% a X register backed by a CPU register on any platform. + Res = case Bin of + <<_:65/unit:8,I:7>> -> I; + <<_:65/unit:8,I:8>> -> I; + <<_:65/unit:8,I:36>> -> I; + <<_:65/unit:8,I:59>> -> I; + <<_:65/unit:8,I:60>> -> I; + <<_:65/unit:8,I:61>> -> I; + <<_:65/unit:8,I:62>> -> I; + <<_:65/unit:8,I:63>> -> I; + <<_:65/unit:8,I:64>> -> I; + <<_:65/unit:8,I:65>> -> I; + <<_:65/unit:8,I:70>> -> I; + <<_:65/unit:8,I:80>> -> I; + <<_:65/unit:8,I:95>> -> I; + _ -> Res0 + end, + {Res,{A,B,C,D,E}}. + +get_unsigned_big_plain(<<_:65/unit:8,I:0>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:1>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:2>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:3>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:4>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:5>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:6>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:7>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:8>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:9>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:10>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:11>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:12>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:13>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:14>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:15>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:16>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:17>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:18>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:19>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:20>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:21>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:22>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:23>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:24>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:25>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:26>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:27>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:28>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:29>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:30>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:31>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:32>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:33>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:34>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:35>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:36>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:37>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:38>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:39>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:40>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:41>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:42>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:43>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:44>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:45>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:46>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:47>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:48>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:49>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:50>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:51>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:52>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:53>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:54>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:55>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:56>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:57>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:58>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:59>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:60>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:61>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:62>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:63>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:64>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:65>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:66>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:67>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:68>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:69>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:70>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:71>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:72>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:73>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:74>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:75>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:76>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:77>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:78>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:79>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:80>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:81>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:82>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:83>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:84>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:85>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:86>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:87>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:88>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:89>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:90>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:91>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:92>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:93>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:94>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:95>>) -> I; +get_unsigned_big_plain(<<_:65/unit:8,I:96>>) -> I. + +get_unsigned_little(<<_:65/unit:8,I:0/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:1/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:2/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:3/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:4/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:5/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:6/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:7/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:8/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:9/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:10/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:11/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:12/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:13/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:14/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:15/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:16/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:17/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:18/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:19/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:20/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:21/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:22/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:23/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:24/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:25/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:26/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:27/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:28/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:29/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:30/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:31/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:32/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:33/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:34/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:35/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:36/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:37/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:38/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:39/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:40/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:41/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:42/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:43/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:44/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:45/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:46/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:47/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:48/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:49/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:50/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:51/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:52/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:53/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:54/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:55/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:56/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:57/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:58/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:59/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:60/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:61/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:62/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:63/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:64/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:65/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:66/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:67/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:68/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:69/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:70/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:71/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:72/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:73/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:74/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:75/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:76/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:77/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:78/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:79/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:80/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:81/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:82/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:83/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:84/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:85/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:86/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:87/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:88/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:89/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:90/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:91/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:92/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:93/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:94/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:95/little>>) -> I; +get_unsigned_little(<<_:65/unit:8,I:96/little>>) -> I. + +get_signed_big(<<_:65/unit:8,I:0/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:1/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:2/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:3/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:4/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:5/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:6/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:7/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:8/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:9/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:10/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:11/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:12/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:13/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:14/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:15/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:16/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:17/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:18/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:19/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:20/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:21/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:22/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:23/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:24/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:25/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:26/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:27/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:28/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:29/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:30/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:31/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:32/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:33/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:34/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:35/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:36/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:37/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:38/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:39/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:40/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:41/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:42/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:43/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:44/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:45/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:46/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:47/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:48/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:49/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:50/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:51/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:52/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:53/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:54/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:55/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:56/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:57/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:58/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:59/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:60/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:61/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:62/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:63/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:64/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:65/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:66/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:67/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:68/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:69/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:70/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:71/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:72/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:73/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:74/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:75/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:76/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:77/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:78/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:79/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:80/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:81/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:82/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:83/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:84/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:85/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:86/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:87/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:88/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:89/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:90/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:91/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:92/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:93/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:94/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:95/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:96/signed-big>>) -> I; +get_signed_big(<<_:65/unit:8,I:97/signed-big>>) -> I. + +get_signed_little(<<_:65/unit:8,I:0/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:1/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:2/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:3/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:4/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:5/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:6/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:7/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:8/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:9/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:10/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:11/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:12/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:13/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:14/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:15/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:16/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:17/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:18/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:19/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:20/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:21/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:22/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:23/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:24/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:25/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:26/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:27/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:28/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:29/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:30/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:31/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:32/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:33/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:34/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:35/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:36/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:37/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:38/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:39/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:40/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:41/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:42/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:43/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:44/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:45/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:46/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:47/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:48/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:49/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:50/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:51/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:52/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:53/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:54/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:55/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:56/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:57/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:58/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:59/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:60/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:61/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:62/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:63/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:64/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:65/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:66/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:67/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:68/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:69/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:70/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:71/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:72/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:73/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:74/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:75/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:76/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:77/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:78/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:79/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:80/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:81/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:82/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:83/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:84/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:85/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:86/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:87/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:88/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:89/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:90/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:91/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:92/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:93/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:94/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:95/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:96/signed-little>>) -> I; +get_signed_little(<<_:65/unit:8,I:97/signed-little>>) -> I. cmp128(<<I:128>>, I) -> equal; cmp128(_, _) -> not_equal. - + +mixed_sizes(_Config) -> + mixed({345,42}, + fun({A,B}) -> + <<A:9,1:1,B:6>>; + (<<A:9,_:1,B:6>>) -> + {A,B} + end), + + mixed({27033,59991,16#c001cafe,12345,2}, + fun({A,B,C,D,E}) -> + <<A:16,B:16,C:32,D:22,E:2>>; + (<<A:16,B:16,C:32,D:22,E:2>>) -> + {A,B,C,D,E} + end), + + mixed({79,153,17555,50_000,777_000,36#hugebignumber,2222}, + fun({A,B,C,D,E,F,G}) -> + <<A:7,B:8,C:15,0:3,D:17,E:23,F:88,G:13>>; + (<<A:7,B:8,C:15,_:3,D:17,E:23,F:88,G:13>>) -> + {A,B,C,D,E,F,G} + end), + + mixed({16#123456789ABCDEF,13,36#hugenum,979}, + fun({A,B,C,D}) -> + <<A:60,B:4,C:50,D:10>>; + (<<A:60,B:4,C:50,D:10>>) -> + {A,B,C,D} + end), + + mixed({16#123456789ABCDEF,13,36#hugenum,979}, + fun({A,B,C,D}) -> + <<A:60/little,B:4/little,C:50/little,D:10/little>>; + (<<A:60/little,B:4/little,C:50/little,D:10/little>>) -> + {A,B,C,D} + end), + + mixed({15692284513449131826, 17798, 33798}, + fun({A,B,C}) -> + <<A:64,B:15/little,C:16/little>>; + (<<A:64,B:15/little,C:16/little>>) -> + {A,B,C} + end), + + mixed({15692344284519131826, 1779863, 13556268}, + fun({A,B,C}) -> + <<A:64,B:23/little,C:24/little>>; + (<<A:64,B:23/little,C:24/little>>) -> + {A,B,C} + end), + + mixed({15519169234428431825, 194086885, 2813274043}, + fun({A,B,C}) -> + <<A:64,B:29/little,C:32/little>>; + (<<A:64,B:29/little,C:32/little>>) -> + {A,B,C} + end), + + mixed({5,9,38759385,93}, + fun({A,B,C,D}) -> + <<1:3,A:4,B:5,C:47,D:7>>; + (<<1:3,A:4,B:5,C:47,D:7>>) -> + {A,B,C,D} + end), + + mixed({2022,8,22}, + fun({A,B,C}) -> + <<A:16/little,B,C>>; + (<<A:16/little,B,C>>) -> + {A,B,C} + end), + + mixed({2022,8,22}, + fun({A,B,C}) -> + <<A:16/little,B,C>>; + (<<A:16/little,B,C>>) -> + _ = id(0), + {A,B,C} + end), + ok. + +mixed(Data, F) -> + Bin = F(Data), + Data = F(Bin), + true = is_bitstring(Bin). + signed_integer(Config) when is_list(Config) -> {no_match,_} = sint(mkbin([])), {no_match,_} = sint(mkbin([1,2,3])), @@ -133,7 +728,7 @@ dynamic(Bin, S1, S2, A, B) -> %% Extract integers at different alignments and of different sizes. more_dynamic(Config) when is_list(Config) -> - % Unsigned big-endian numbers. + %% Unsigned big-endian numbers. Unsigned = fun(Bin, List, SkipBef, N) -> SkipAft = 8*size(Bin) - N - SkipBef, <<_:SkipBef,Int:N,_:SkipAft>> = Bin, @@ -249,35 +844,40 @@ match_huge_int(Config) when is_list(Config) -> %% because of insufficient memory. {skip, "unoptimized code would use too much memory"}; bs_match_int_SUITE -> - Sz = 1 bsl 27, - Bin = <<0:Sz,13:8>>, - skip_huge_int_1(Sz, Bin), - 0 = match_huge_int_1(Sz, Bin), - - %% Test overflowing the size of an integer field. - nomatch = overflow_huge_int_skip_32(Bin), - case erlang:system_info(wordsize) of - 4 -> - nomatch = overflow_huge_int_32(Bin); - 8 -> - %% An attempt will be made to allocate heap space for - %% the bignum (which will probably fail); only if the - %% allocation succeeds will the matching fail because - %% the binary is too small. - ok - end, - nomatch = overflow_huge_int_skip_64(Bin), - nomatch = overflow_huge_int_64(Bin), - - %% Test overflowing the size of an integer field using - %% variables as sizes. - Sizes = case erlang:system_info(wordsize) of - 4 -> lists:seq(25, 32); - 8 -> [] - end ++ lists:seq(50, 64), - ok = overflow_huge_int_unit128(Bin, Sizes) + do_match_huge_int(); + bs_match_int_r25_SUITE -> + do_match_huge_int() end. +do_match_huge_int() -> + Sz = 1 bsl 27, + Bin = <<0:Sz,13:8>>, + skip_huge_int_1(Sz, Bin), + 0 = match_huge_int_1(Sz, Bin), + + %% Test overflowing the size of an integer field. + nomatch = overflow_huge_int_skip_32(Bin), + case erlang:system_info(wordsize) of + 4 -> + nomatch = overflow_huge_int_32(Bin); + 8 -> + %% An attempt will be made to allocate heap space for + %% the bignum (which will probably fail); only if the + %% allocation succeeds will the matching fail because + %% the binary is too small. + ok + end, + nomatch = overflow_huge_int_skip_64(Bin), + nomatch = overflow_huge_int_64(Bin), + + %% Test overflowing the size of an integer field using + %% variables as sizes. + Sizes = case erlang:system_info(wordsize) of + 4 -> lists:seq(25, 32); + 8 -> [] + end ++ lists:seq(50, 64), + ok = overflow_huge_int_unit128(Bin, Sizes). + overflow_huge_int_unit128(Bin, [Sz0|Sizes]) -> Sz = id(1 bsl Sz0), case Bin of @@ -375,5 +975,9 @@ unaligned_32_bit_1(_) -> unaligned_32_bit_verify([], 0) -> ok; unaligned_32_bit_verify([4294967295|T], N) when N > 0 -> unaligned_32_bit_verify(T, N-1). - + +%%% +%%% Common utilities. +%%% + id(I) -> I. diff --git a/erts/emulator/test/bs_utf_SUITE.erl b/erts/emulator/test/bs_utf_SUITE.erl index a344f5c456..921e2ee860 100644 --- a/erts/emulator/test/bs_utf_SUITE.erl +++ b/erts/emulator/test/bs_utf_SUITE.erl @@ -20,7 +20,7 @@ -module(bs_utf_SUITE). --export([all/0, suite/0, +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1, utf8_roundtrip/1,utf16_roundtrip/1,utf32_roundtrip/1, utf8_illegal_sequences/1,utf16_illegal_sequences/1, utf32_illegal_sequences/1, @@ -34,26 +34,61 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 6}}]. -all() -> +all() -> [utf8_roundtrip, utf16_roundtrip, utf32_roundtrip, utf8_illegal_sequences, utf16_illegal_sequences, utf32_illegal_sequences, bad_construction]. +init_per_suite(Config) -> + %% Make sure that calls to id/1 will hide types. + id(Config), + Config. + +end_per_suite(Config) -> + Config. + utf8_roundtrip(Config) when is_list(Config) -> utf8_roundtrip(0, 16#D7FF), utf8_roundtrip(16#E000, 16#10FFFF), ok. -utf8_roundtrip(First, Last) when First =< Last -> - Bin = int_to_utf8(First), +utf8_roundtrip(First, Last) -> + %% Hide types. + do_utf8_roundtrip(id(First), id(Last)). + +do_utf8_roundtrip(First, Last) when First =< Last -> + Bin = int_to_utf8(id(First)), Bin = id(<<First/utf8>>), Bin = id(<<(id(<<>>))/binary,First/utf8>>), - Unaligned = id(<<3:2,First/utf8>>), - <<_:2,Bin/binary>> = Unaligned, + + <<0:7/unit:8,Bin/binary>> = id(<<0:7/unit:8,First/utf8>>), + + %% Here a heap binary and a sub binary will be allocated. If the + %% write in the utf8 segment extends beyond the end of heap binary, + %% it will will overwrite the header for the sub binary. + <<-1:(64-9)/signed,Bin/binary>> = id(<<-1:(64-9),First/utf8>>), + <<-1:63/signed,Bin/binary>> = id(<<-1:63,First/utf8>>), + + if + is_integer(First) -> + Bin = id(<<First/utf8>>) + end, + + <<1:1,Bin/binary>> = id(<<1:1,First/utf8>>), + <<0:1,Bin/binary>> = id(<<0:1,First/utf8>>), + <<3:2,Bin/binary>> = id(<<3:2,First/utf8>>), + <<5:3,Bin/binary>> = id(<<5:3,First/utf8>>), + <<13:4,Bin/binary>> = id(<<13:4,First/utf8>>), + <<21:5,Bin/binary>> = id(<<21:5,First/utf8>>), + <<51:6,Bin/binary>> = id(<<51:6,First/utf8>>), + <<107:7,Bin/binary>> = id(<<107:7,First/utf8>>), + <<First/utf8>> = Bin, <<First/utf8>> = make_unaligned(Bin), - utf8_roundtrip(First+1, Last); -utf8_roundtrip(_, _) -> ok. + + Bin = id(<<First/utf8>>), + do_utf8_roundtrip(First+1, Last); +do_utf8_roundtrip(_, _) -> ok. utf16_roundtrip(Config) when is_list(Config) -> Big = fun utf16_big_roundtrip/1, @@ -291,6 +326,9 @@ bad_construction(Config) when is_list(Config) -> ?FAIL(<<3.14/utf8>>), ?FAIL(<<3.1415/utf16>>), ?FAIL(<<3.1415/utf32>>), + {'EXIT',_} = (catch <<(id(3.14))/utf8>>), + {'EXIT',_} = (catch <<(id(3.1415))/utf16>>), + {'EXIT',_} = (catch <<(id(3.1415))/utf32>>), ?FAIL(<<(-1)/utf8>>), ?FAIL(<<(-1)/utf16>>), @@ -301,6 +339,9 @@ bad_construction(Config) when is_list(Config) -> ?FAIL(<<16#D800/utf8>>), ?FAIL(<<16#D800/utf16>>), ?FAIL(<<16#D800/utf32>>), + {'EXIT',_} = (catch <<(id(16#D800))/utf8>>), + {'EXIT',_} = (catch <<(id(16#D800))/utf16>>), + {'EXIT',_} = (catch <<(id(16#D800))/utf32>>), ok. diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index 59d791eb2b..2bba329234 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -29,13 +29,49 @@ -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, groups/0, dirty_nif/1, dirty_nif_send/1, dirty_nif_exception/1, call_dirty_nif_exception/1, dirty_scheduler_exit/1, dirty_call_while_terminated/1, dirty_heap_access/1, dirty_process_info/1, dirty_process_register/1, dirty_process_trace/1, code_purge/1, literal_area/1, dirty_nif_send_traced/1, - nif_whereis/1, nif_whereis_parallel/1, nif_whereis_proxy/1]). + nif_whereis/1, nif_whereis_parallel/1, nif_whereis_proxy/1, + set_halt_options_from_nif/1, + delay_halt/1, + delay_halt_old_code/1, + delay_halt_old_and_new_code/1, + flush_false/1, + on_halt/1, + on_halt_old_code/1, + on_halt_old_and_new_code/1, + sync_halt/1, + many_delay_halt/1, + many_on_halt/1]). + +-export([load_nif/2]). + +-nifs([lib_loaded/0, + call_dirty_nif/3, + send_from_dirty_nif/1, + send_wait_from_dirty_nif/1, + call_dirty_nif_exception/1, + call_dirty_nif_zero_args/0, + dirty_call_while_terminated_nif/1, + dirty_sleeper/0, + dirty_sleeper/1, + dirty_heap_access_nif/1, + whereis_term/2, + whereis_send/3, + dirty_terminating_literal_access/2, + delay_halt_normal/3, + delay_halt_io_bound/3, + delay_halt_cpu_bound/3, + sync_halt_io_bound/2, + sync_halt_cpu_bound/2, + set_halt_option_from_nif_normal/1, + set_halt_option_from_nif_io_bound/1, + set_halt_option_from_nif_cpu_bound/1]). -define(nif_stub,nif_stub_error(?LINE)). @@ -55,23 +91,38 @@ all() -> literal_area, dirty_nif_send_traced, nif_whereis, - nif_whereis_parallel]. + nif_whereis_parallel, + {group, halt_normal}, + {group, halt_dirty_cpu}, + {group, halt_dirty_io}, + {group, halt_misc}]. + +halt_sched_tests() -> + [set_halt_options_from_nif, delay_halt, delay_halt_old_code, delay_halt_old_and_new_code]. +halt_dirty_sched_tests() -> + [sync_halt, flush_false]. + +groups() -> + [{halt_normal, [parallel], halt_sched_tests()}, + {halt_dirty_cpu, [parallel], halt_sched_tests()++halt_dirty_sched_tests()}, + {halt_dirty_io, [parallel], halt_sched_tests()++halt_dirty_sched_tests()}, + {halt_misc, [parallel], [on_halt, on_halt_old_code, on_halt_old_and_new_code, many_on_halt, many_delay_halt]}]. + +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(_, Config) -> + proplists:delete(group, Config). init_per_suite(Config) -> - case erlang:system_info(dirty_cpu_schedulers) of - N when N > 0 -> - case lib_loaded() of - false -> - ok = erlang:load_nif( - filename:join(?config(data_dir, Config), - "dirty_nif_SUITE"), []); - true -> - ok - end, - Config; - _ -> - {skipped, "No dirty scheduler support"} - end. + case lib_loaded() of + false -> + ok = erlang:load_nif(filename:join(?config(data_dir, Config), + "dirty_nif_SUITE"), []); + true -> + ok + end, + Config. end_per_suite(_Config) -> ok. @@ -82,6 +133,9 @@ init_per_testcase(Case, Config) -> end_per_testcase(_Case, _Config) -> ok. +load_nif(NifLib, LibInfo) -> + erlang:load_nif(NifLib, LibInfo). + dirty_nif(Config) when is_list(Config) -> Val1 = 42, Val2 = "Erlang", @@ -536,6 +590,356 @@ literal_area(Config) when is_list(Config) -> literal_area_collector_test:check_idle(5000), {comment, "Waited "++integer_to_list(TMO)++" milliseconds after purge"}. +set_halt_options_from_nif(Config) when is_list(Config) -> + case ?config(group, Config) of + halt_normal -> + error = set_halt_option_from_nif_normal(set_on_halt_handler), + error = set_halt_option_from_nif_normal(delay_halt); + halt_dirty_cpu -> + error = set_halt_option_from_nif_cpu_bound(set_on_halt_handler), + error = set_halt_option_from_nif_cpu_bound(delay_halt); + halt_dirty_io -> + error = set_halt_option_from_nif_io_bound(set_on_halt_handler), + error = set_halt_option_from_nif_io_bound(delay_halt) + end, + ok. + +delay_halt(Config) when is_list(Config) -> + delay_halt(Config, new_code). + +delay_halt_old_code(Config) when is_list(Config) -> + delay_halt(Config, old_code). + +delay_halt_old_and_new_code(Config) when is_list(Config) -> + delay_halt(Config, old_and_new_code). + +delay_halt(Config, Type) -> + Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING), + Priv = proplists:get_value(priv_dir, Config), + TypeSuffix = "_"++atom_to_list(Type), + {Fun, FileName} = case ?config(group, Config) of + halt_normal -> + {fun delay_halt_normal/3, "delay_halt_normal"++TypeSuffix}; + halt_dirty_io -> + {fun delay_halt_io_bound/3, "delay_halt_io_bound"++TypeSuffix}; + halt_dirty_cpu -> + {fun delay_halt_cpu_bound/3, "delay_halt_cpu_bound"++TypeSuffix} + end, + Tester = self(), + NifFileName = filename:join(Priv, FileName), + {ok, Peer, Node} = ?CT_PEER(), + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]), + ok = erpc:call(Node, file, set_cwd, [Priv]), + Proxy = spawn_link(Node, + fun () -> + receive + {delay_halt, _} = Msg -> + unlink(Tester), + Tester ! Msg + end + end), + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName, 2) end), + receive {delay_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end, + case Type of + new_code -> + ok; + old_code -> + true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]); + old_and_new_code -> + true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]), + Proxy2 = spawn_link(Node, + fun () -> + receive + {delay_halt, _} = Msg -> + unlink(Tester), + Tester ! Msg + end + end), + ok = erpc:cast(Node, fun () -> Fun(Proxy2, FileName++"_new_code", 2) end), + receive {delay_halt, Pid2} when is_pid(Pid2), Node == node(Pid2) -> ok end + end, + ok = erpc:cast(Node, erlang, halt, []), + ok = wait_until(fun () -> + {ok, <<"ok">>} == file:read_file(NifFileName) + andalso + (Type /= old_and_new_code + orelse {ok, <<"ok">>} == file:read_file(NifFileName++"_new_code")) + end, + 6000), + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("~s time=~pms", [FileName, Time]), + true = Time >= 2000, + receive {'DOWN', Mon, process, Peer, _} -> ok end, + ok. + +flush_false(Config) when is_list(Config) -> + Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING), + Priv = proplists:get_value(priv_dir, Config), + {Fun, FileName} = case ?config(group, Config) of + halt_dirty_io -> + {fun delay_halt_io_bound/3, "flush_false_io_bound"}; + halt_dirty_cpu -> + {fun delay_halt_cpu_bound/3, "flush_false_cpu_bound"} + end, + Tester = self(), + NifFileName = filename:join(Priv, FileName), + OnHaltBaseName = FileName++"_on_halt", + OnHaltFileName = filename:join(Priv, OnHaltBaseName), + {ok, Peer, Node} = ?CT_PEER(), + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {sync_halt, OnHaltBaseName, 1}]), + ok = erpc:call(Node, file, set_cwd, [Priv]), + Proxy = spawn_link(Node, + fun () -> + receive + {delay_halt, _} = Msg -> + unlink(Tester), + Tester ! Msg + end + end), + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName, 1) end), + receive {delay_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end, + ok = erpc:cast(Node, erlang, halt, [0, [{flush,false}]]), + receive {'DOWN', Mon, process, Peer, _} -> ok end, + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("~s time=~pms", [FileName, Time]), + Wait = 3000-Time, + if Wait > 0 -> receive after Wait -> ok end; + true -> ok + end, + {error,enoent} = file:read_file(NifFileName), + {error,enoent} = file:read_file(OnHaltFileName), + ok. + +many_delay_halt(Config) when is_list(Config) -> + try + many_delay_halt_test(Config) + catch + throw:{skip, _} = Skip -> + Skip + end. + +many_delay_halt_test(Config) -> + Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING), + Priv = proplists:get_value(priv_dir, Config), + Tester = self(), + {ok, Peer, Node} = ?CT_PEER(), + Chk = fun () -> + case erlang:system_info(schedulers_online) of + 1 -> throw({skip, "Too few schedulers online"}); + _ -> ok + end, + case erlang:system_info(dirty_cpu_schedulers_online) of + 1 -> throw({skip, "Too few dirty cpu schedulers online"}); + _ -> ok + end, + case erlang:system_info(dirty_io_schedulers) of + 1 -> throw({skip, "Too few dirty io schedulers online"}); + _ -> ok + end + end, + try + erpc:call(Node, Chk) + catch + throw:{skip, _} = Skip -> + peer:stop(Peer), + throw(Skip) + end, + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]), + ok = erpc:call(Node, file, set_cwd, [Priv]), + ProxyFun = fun (Tag) -> + fun () -> + receive + {delay_halt, _} = Msg -> + unlink(Tester), + Tester ! {Tag, Msg} + end + end + end, + [P1, P2, P3, P4, P5] = [spawn_link(Node, ProxyFun(X)) || X <- lists:seq(1, 5)], + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, fun () -> delay_halt_io_bound(P1, "many_delay_halt_io2", 2) end), + ok = erpc:cast(Node, fun () -> delay_halt_io_bound(P2, "many_delay_halt_io1", 1) end), + ok = erpc:cast(Node, fun () -> delay_halt_cpu_bound(P3, "many_delay_halt_cpu1", 1) end), + ok = erpc:cast(Node, fun () -> delay_halt_cpu_bound(P4, "many_delay_halt_cpu2", 2) end), + _ = [receive + {X, {delay_halt, Pid}} when is_pid(Pid), Node == node(Pid) -> + ok + end || X <- lists:seq(1, 4)], + ok = erpc:cast(Node, fun () -> delay_halt_normal(P5, "many_delay_halt_normal", 1) end), + receive + {5, {delay_halt, Pid}} when is_pid(Pid), Node == node(Pid) -> + ok + end, + ok = erpc:cast(Node, erlang, halt, []), + ok = wait_until(fun () -> + {ok, <<"ok">>} == file:read_file(filename:join(Priv, "many_delay_halt_io2")) + andalso {ok, <<"ok">>} == file:read_file(filename:join(Priv, "many_delay_halt_cpu2")) + end, + 3000), + {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_cpu1")), + {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_io1")), + {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_normal")), + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("many_delay_halt time=~pms", [Time]), + true = Time >= 2000, + receive {'DOWN', Mon, process, Peer, _} -> ok end, + ok. + +on_halt(Config) when is_list(Config) -> + on_halt(Config, new_code). + +on_halt_old_code(Config) when is_list(Config) -> + on_halt(Config, old_code). + +on_halt_old_and_new_code(Config) when is_list(Config) -> + on_halt(Config, old_and_new_code). + +on_halt(Config, Type) -> + Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING), + Priv = proplists:get_value(priv_dir, Config), + FileName = "on_halt_"++atom_to_list(Type), + OnHaltFileName = filename:join(Priv, FileName), + {ok, Peer, Node} = ?CT_PEER(), + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {on_halt, FileName, 1}]), + case Type of + new_code -> + ok; + old_code -> + true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]); + old_and_new_code -> + true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {on_halt, FileName++"_new", 1}]) + end, + ok = erpc:call(Node, file, set_cwd, [Priv]), + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, erlang, halt, []), + ok = wait_until(fun () -> + {ok, <<"ok">>} == file:read_file(OnHaltFileName) + andalso (Type /= old_and_new_code + orelse {ok, <<"ok">>} == file:read_file(OnHaltFileName++"_new")) + end, + 3000), + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("~s time=~pms", [FileName, Time]), + if Type == old_and_new_code -> true = Time >= 2000; + true -> true = Time >= 1000 + end, + receive {'DOWN', Mon, process, Peer, _} -> ok end, + ok. + +on_halt_module_code_format() -> + lists:flatten(["-module(~s).~n", + "-export([load/1, lib_loaded/0]).~n", + "-nifs([lib_loaded/0]).~n", + "load(SoFile) -> erlang:load_nif(SoFile, ?MODULE_STRING).~n", + "lib_loaded() -> false.~n"]). + +many_on_halt(Config) when is_list(Config) -> + DDir = ?config(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + OnHaltModules = ["on_halt_a","on_halt_b","on_halt_c","on_halt_d","on_halt_e","on_halt_f"], + DeleteOnHaltModules = ["on_halt_a","on_halt_c","on_halt_d","on_halt_f"], + PurgeOnHaltModules = DeleteOnHaltModules -- ["on_halt_d"], + ActiveOnHaltModules = OnHaltModules -- PurgeOnHaltModules, + lists:foreach(fun (ModStr) -> + Code = io_lib:format(on_halt_module_code_format(), [ModStr]), + ok = file:write_file(filename:join(DDir, ModStr++".erl"), Code) + end, + OnHaltModules), + {ok, Peer, Node} = ?CT_PEER(), + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, + fun () -> + ok = file:set_cwd(Priv), + lists:foreach(fun (ModStr) -> + AbsModStr = filename:join(DDir, ModStr), + {ok,Mod,Bin} = compile:file(AbsModStr, [binary]), + {module, Mod} = erlang:load_module(Mod, Bin), + ok = Mod:load(AbsModStr), + true = Mod:lib_loaded() + end, + OnHaltModules), + lists:foreach(fun (ModStr) -> + Mod = list_to_atom(ModStr), + true = erlang:delete_module(Mod) + end, DeleteOnHaltModules), + lists:foreach(fun (ModStr) -> + Mod = list_to_atom(ModStr), + true = erlang:purge_module(Mod) + end, PurgeOnHaltModules), + ok + end), + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, erlang, halt, []), + ok = wait_until(fun () -> + try + lists:foreach(fun (ModStr) -> + FileName = filename:join(Priv, ModStr), + {ok, <<"ok">>} = file:read_file(FileName) + end, ActiveOnHaltModules), + true + catch + _:_ -> + false + end + end, + 1000*(length(ActiveOnHaltModules)+1)), + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("many_on_halt time=~pms", [Time]), + true = Time >= length(ActiveOnHaltModules)*1000, + receive {'DOWN', Mon, process, Peer, _} -> ok end, + ok. + +sync_halt(Config) when is_list(Config) -> + Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING), + Priv = proplists:get_value(priv_dir, Config), + {Fun, FileName} = case ?config(group, Config) of + halt_dirty_io -> + {fun sync_halt_io_bound/2, "sync_halt_io_bound"}; + halt_dirty_cpu -> + {fun sync_halt_cpu_bound/2, "sync_halt_cpu_bound"} + end, + Tester = self(), + NifFileName = filename:join(Priv, FileName), + OnHaltBaseFileName = FileName++".onhalt", + OnHaltFileName = filename:join(Priv, OnHaltBaseFileName), + {ok, Peer, Node} = ?CT_PEER(), + Mon = erlang:monitor(process, Peer), + ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {sync_halt, OnHaltBaseFileName, 1}]), + ok = erpc:call(Node, file, set_cwd, [Priv]), + Proxy = spawn_link(Node, + fun () -> + receive + {sync_halt, _} = Msg -> + unlink(Tester), + Tester ! Msg + end + end), + ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName) end), + receive {sync_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end, + Start = erlang:monotonic_time(millisecond), + ok = erpc:cast(Node, erlang, halt, []), + ok = wait_until(fun () -> + {ok, <<"ok">>} == file:read_file(OnHaltFileName) + end, + 2000), + ok = wait_until(fun () -> + {ok, <<"ok">>} == file:read_file(NifFileName) + end, + 4000), + Time = erlang:monotonic_time(millisecond) - Start, + ct:log("~s time=~pms", [FileName, Time]), + true = Time >= 1000, + receive {'DOWN', Mon, process, Peer, _} -> ok end, + ok. + %% %% Internal... %% @@ -774,6 +1178,14 @@ dirty_heap_access_nif(_) -> ?nif_stub. whereis_term(_Type,_Name) -> ?nif_stub. whereis_send(_Type,_Name,_Msg) -> ?nif_stub. dirty_terminating_literal_access(_Me, _Literal) -> ?nif_stub. +delay_halt_normal(_Pid, _FileName, _Delay) -> ?nif_stub. +delay_halt_io_bound(_Pid, _FileName, _Delay) -> ?nif_stub. +delay_halt_cpu_bound(_Pid, _FileName, _Delay) -> ?nif_stub. +sync_halt_io_bound(_Pid, _FileName) -> ?nif_stub. +sync_halt_cpu_bound(_Pid, _FileName) -> ?nif_stub. +set_halt_option_from_nif_normal(_Op) -> ?nif_stub. +set_halt_option_from_nif_io_bound(_Op) -> ?nif_stub. +set_halt_option_from_nif_cpu_bound(_Op) -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src index 4462afd815..55ca552cb2 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src +++ b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src @@ -1,5 +1,5 @@ -NIF_LIBS = dirty_nif_SUITE@dll@ +NIF_LIBS = dirty_nif_SUITE@dll@ on_halt_a@dll@ on_halt_b@dll@ on_halt_c@dll@ on_halt_d@dll@ on_halt_e@dll@ on_halt_f@dll@ all: $(NIF_LIBS) echo_drv@dll@ diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index fb5146278b..7f0fc9ea46 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -19,11 +19,14 @@ */ #include <erl_nif.h> #include <assert.h> +#include <errno.h> #ifdef __WIN32__ #include <windows.h> #else #include <unistd.h> #endif +#include <stdio.h> +#include <string.h> /* * Hack to get around this function missing from the NIF API. @@ -43,8 +46,128 @@ static ERL_NIF_TERM atom_pid; static ERL_NIF_TERM atom_port; static ERL_NIF_TERM atom_send; +typedef struct { + int halting; + int on_halt_wait; + ErlNifMutex *mtx; + ErlNifCond *cnd; + char *filename; +} PrivData; + +static PrivData *make_priv_data(void) +{ + PrivData *pdata = enif_alloc(sizeof(PrivData)); + if (!pdata) + return NULL; + pdata->halting = 0; + pdata->on_halt_wait = 0; + pdata->mtx = NULL; + pdata->cnd = NULL; + pdata->filename = NULL; + return pdata; +} + +static void unload(ErlNifEnv *env, void *priv_data) +{ + if (priv_data) { + PrivData *pdata = priv_data; + if (pdata->mtx) + enif_mutex_destroy(pdata->mtx); + if (pdata->cnd) + enif_cond_destroy(pdata->cnd); + if (pdata->filename) + enif_free(pdata->filename); + enif_free(pdata); + } +} + +static void on_halt(void *priv_data); + static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { + int arity; + const ERL_NIF_TERM *array; + if (enif_get_tuple(env, load_info, &arity, &array)) { + char atom_text[32]; + int err; + unsigned filename_len; + PrivData *pdata = NULL; + + if (arity < 1 || 3 < arity) + return __LINE__; + if (!enif_get_atom(env, array[0], &atom_text[0], + sizeof(atom_text), ERL_NIF_LATIN1)) { + return __LINE__; + } + pdata = make_priv_data(); + if (!pdata) + return __LINE__; + if (strcmp(atom_text, "on_halt") == 0) { + if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) { + unload(env, pdata); + return __LINE__; + } + } + else if (strcmp(atom_text, "delay_halt") == 0) { + if (0 != enif_set_option(env, ERL_NIF_OPT_DELAY_HALT)) { + unload(env, pdata); + return __LINE__; + } + } + else if (strcmp(atom_text, "sync_halt") == 0) { + if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) { + unload(env, pdata); + return __LINE__; + } + if (0 != enif_set_option(env, ERL_NIF_OPT_DELAY_HALT)) { + unload(env, pdata); + return __LINE__; + } + pdata->mtx = enif_mutex_create("sync_halt_dirty_nif_SUITE"); + pdata->cnd = enif_cond_create("sync_halt_dirty_nif_SUITE"); + if (!pdata->mtx || !pdata->cnd) { + unload(env, pdata); + return __LINE__; + } + } + else { + unload(env, pdata); + return __LINE__; + } + + if (arity >= 2) { + if (!enif_get_list_length(env, array[1], &filename_len)) { + unload(env, pdata); + return __LINE__; + } + if (filename_len > 0) { + filename_len++; + pdata->filename = enif_alloc(filename_len); + if (!pdata->filename) { + unload(env, pdata); + return __LINE__; + } + if (filename_len != enif_get_string(env, + array[1], + pdata->filename, + filename_len, + ERL_NIF_LATIN1)) { + unload(env, pdata); + return __LINE__; + } + } + } + if (arity == 3) { + if (!enif_get_int(env, array[2], &pdata->on_halt_wait) + || pdata->on_halt_wait < 0 + || pdata->on_halt_wait*1000 < 0) { + unload(env, pdata); + return __LINE__; + } + } + *priv_data = (void *) pdata; + } + atom_badarg = enif_make_atom(env, "badarg"); atom_error = enif_make_atom(env, "error"); atom_false = enif_make_atom(env,"false"); @@ -57,6 +180,11 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + return load(env, priv_data, load_info); +} + static ERL_NIF_TERM lib_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { return enif_make_atom(env, "true"); @@ -468,6 +596,149 @@ static ERL_NIF_TERM dirty_terminating_literal_access(ErlNifEnv* env, int argc, c return self_term; } +static int fn_write_ok(char *filename) +{ + FILE *file = fopen(filename, "w"); + if (!file) + return EINVAL; + if (1 != fwrite("ok", 2, 1, file)) + return EINVAL; + fclose(file); + return 0; +} + +static int efn_write_ok(ErlNifEnv *env, const ERL_NIF_TERM arg) +{ + int res; + unsigned filename_len; + char *filename; + + if (!enif_get_list_length(env, arg, &filename_len) || filename_len < 2) { + res = EINVAL; + goto done; + } + filename_len++; + filename = enif_alloc(filename_len); + if (!filename) { + res = ENOMEM; + goto done; + } + if (filename_len != enif_get_string(env, + arg, + filename, + filename_len, + ERL_NIF_LATIN1)) { + res = EINVAL; + goto done; + } + res = fn_write_ok(filename); +done: + if (filename) + enif_free(filename); + switch (res) { + case 0: + return atom_ok; + case ENOMEM: + return enif_raise_exception(env, enif_make_atom(env, "enomem")); + default: + return enif_make_badarg(env); + } +} + +static ERL_NIF_TERM delay_halt(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM msg; + ErlNifPid receiver, self; + int res, secs; + + if (argc != 3) + return enif_make_badarg(env); + if (!enif_get_int(env, argv[2], &secs)) + return enif_make_badarg(env); + if (secs < 0 || secs*1000 < 0) + return enif_make_badarg(env); + if (!enif_self(env, &self)) + return enif_make_badarg(env); + if (!enif_get_local_pid(env, argv[0], &receiver)) + return enif_make_badarg(env); + msg = enif_make_tuple2(env, enif_make_atom(env, "delay_halt"), enif_make_pid(env, &self)); + res = enif_send(env, &receiver, NULL, msg); + if (!res) + return enif_make_badarg(env); + +#ifdef __WIN32__ + Sleep(secs*1000); +#else + sleep(secs); +#endif + return efn_write_ok(env, argv[1]); +} + +static ERL_NIF_TERM sync_halt(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM msg; + ErlNifPid receiver, self; + int res; + PrivData *pdata = enif_priv_data(env); + if (!pdata) + return enif_raise_exception(env, enif_make_atom(env, "missing_priv_data")); + if (argc != 2) + return enif_make_badarg(env); + if (!enif_self(env, &self)) + return enif_make_badarg(env); + if (!enif_get_local_pid(env, argv[0], &receiver)) + return enif_make_badarg(env); + msg = enif_make_tuple2(env, enif_make_atom(env, "sync_halt"), enif_make_pid(env, &self)); + res = enif_send(env, &receiver, NULL, msg); + if (!res) + return enif_make_badarg(env); + enif_mutex_lock(pdata->mtx); + while (!pdata->halting) + enif_cond_wait(pdata->cnd, pdata->mtx); + enif_mutex_unlock(pdata->mtx); + return efn_write_ok(env, argv[1]); +} + +static ERL_NIF_TERM set_halt_option_from_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned len; + char atom_text[32]; + if (argc != 1) + return enif_make_badarg(env); + if (!enif_get_atom(env, argv[0], &atom_text[0], sizeof(atom_text), ERL_NIF_LATIN1)) + return enif_make_badarg(env); + if (strcmp(atom_text, "set_on_halt_handler") == 0) { + if (0 == enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) + return atom_ok; + return atom_error; + } + else if (strcmp(atom_text, "delay_halt") == 0) { + if (0 == enif_set_option(env, ERL_NIF_OPT_DELAY_HALT)) + return atom_ok; + return atom_error; + } + return enif_make_badarg(env); +} + +static void on_halt(void *priv_data) +{ + PrivData *pdata = (PrivData *)priv_data; + int res; +#ifdef __WIN32__ + Sleep(pdata->on_halt_wait*1000); +#else + sleep(pdata->on_halt_wait); +#endif + if (pdata->mtx) { + enif_mutex_lock(pdata->mtx); + assert(!pdata->halting); + pdata->halting = !0; + enif_cond_broadcast(pdata->cnd); + enif_mutex_unlock(pdata->mtx); + } + res = fn_write_ok(pdata->filename); + assert(res == 0); +} static ErlNifFunc nif_funcs[] = { @@ -484,6 +755,14 @@ static ErlNifFunc nif_funcs[] = {"whereis_send", 3, whereis_send, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"whereis_term", 2, whereis_term, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_terminating_literal_access", 2, dirty_terminating_literal_access, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"delay_halt_normal", 3, delay_halt, 0}, + {"delay_halt_io_bound", 3, delay_halt, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"delay_halt_cpu_bound", 3, delay_halt, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"sync_halt_io_bound", 2, sync_halt, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"sync_halt_cpu_bound", 2, sync_halt, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"set_halt_option_from_nif_normal", 1, set_halt_option_from_nif, 0}, + {"set_halt_option_from_nif_io_bound", 1, set_halt_option_from_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"set_halt_option_from_nif_cpu_bound", 1, set_halt_option_from_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} }; -ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,NULL,NULL) +ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c new file mode 100644 index 0000000000..73d50a2bf1 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_a,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c new file mode 100644 index 0000000000..b9e13a17fa --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_b,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c new file mode 100644 index 0000000000..db875e7f18 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_c,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c new file mode 100644 index 0000000000..e3b64245ce --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_d,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c new file mode 100644 index 0000000000..73357c9b9d --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_e,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c new file mode 100644 index 0000000000..58b4955d4f --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c @@ -0,0 +1,23 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "on_halt_nif.c" + +ERL_NIF_INIT(on_halt_f,nif_funcs,load,NULL,NULL,unload) diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c new file mode 100644 index 0000000000..610b80d3f7 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c @@ -0,0 +1,96 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2022. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +#include "erl_nif.h" +#include <errno.h> +#include <assert.h> +#ifdef __WIN32__ +#include <windows.h> +#else +#include <unistd.h> +#endif +#include <stdio.h> +#include <string.h> + + +static int fn_write_ok(char *filename) +{ + FILE *file = fopen(filename, "w"); + if (!file) + return EINVAL; + if (1 != fwrite("ok", 2, 1, file)) + return EINVAL; + fclose(file); + return 0; +} + +static void on_halt(void *priv_data) +{ + int res; +#ifdef __WIN32__ + Sleep(1000); +#else + sleep(1); +#endif + assert(priv_data); + res = fn_write_ok((char *) priv_data); + assert(res == 0); +} + +static void unload(ErlNifEnv *env, void *priv_data) +{ + if (priv_data) + enif_free(priv_data); +} + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + unsigned filename_len; + char *filename; + if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) + return __LINE__; + if (!enif_get_list_length(env, load_info, &filename_len)) + return __LINE__; + if (filename_len == 0) + return __LINE__; + filename_len++; + filename = enif_alloc(filename_len); + if (!filename) + return __LINE__; + if (filename_len != enif_get_string(env, + load_info, + filename, + filename_len, + ERL_NIF_LATIN1)) { + enif_free(filename); + return __LINE__; + } + *priv_data = (void *) filename; + return 0; +} + +static ERL_NIF_TERM lib_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return enif_make_atom(env, "true"); +} + +static ErlNifFunc nif_funcs[] = +{ + {"lib_loaded", 0, lib_loaded} +}; diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index ea76a473f2..9d4d0972c2 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -846,6 +846,8 @@ error_info(_Config) -> {display, ["test erlang:display/1"], [no_fail]}, {display_string, [{a,b,c}]}, + {display_string, [standard_out,"test erlang:display/2"]}, + {display_string, [stdout,{a,b,c}]}, %% Internal undcoumented BIFs. {dist_ctrl_get_data, 1}, diff --git a/erts/emulator/test/jit_SUITE.erl b/erts/emulator/test/jit_SUITE.erl index 67de1b23dc..e1bb42e152 100644 --- a/erts/emulator/test/jit_SUITE.erl +++ b/erts/emulator/test/jit_SUITE.erl @@ -24,7 +24,7 @@ init_per_suite/1, end_per_suite/1, init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2]). --export([annotate/1, named_labels/1, symbols/1]). +-export([annotate/1, jmsingle/1, named_labels/1, symbols/1]). suite() -> [{timetrap, {minutes, 4}}]. @@ -33,7 +33,7 @@ groups() -> [{perf, [symbols, annotate]}]. all() -> - [{group, perf}, named_labels]. + [{group, perf}, jmsingle, named_labels]. init_per_suite(Config) -> case erlang:system_info(emu_flavor) of @@ -180,6 +180,70 @@ annotate(Config) -> [Symbol, Anno]) end. +run_jmsingle_test(Param, ExpectSuccess, ErrorMsg) -> + Cmd = "erl +JMsingle " ++ Param ++ " -noshell " ++ + "-eval \"erlang:display(all_is_well),erlang:halt(0).\"", + Result = os:cmd(Cmd), + SuccessfulEmulatorStart = + case Result of + "all_is_well" ++ _ -> + true; + _ -> + Error = "Failed to allocate executable+writable memory", + case string:find(Result, Error) of + nomatch -> false; + _ -> internal_error + end + end, + case SuccessfulEmulatorStart of + ExpectSuccess -> + ok; + _ -> + ct:fail(ErrorMsg) + end. + +jmsingle(Config) -> + %% Smoke test to check that the emulator starts with the +JMsingle + %% true/false option and fails with a non-boolean, that is, we + %% parse the command line correctly. + IsAarch64Apple = case erlang:system_info(system_architecture) of + "aarch64-apple" ++ _ -> true; + _ -> false + end, + IsNetBSD = case os:type() of + {_,netbsd} -> true; + _ -> false + end, + + if + IsAarch64Apple or IsNetBSD -> + %% +JMsingle true does not work on macOS running + %% on Apple Silicon computers nor on NetBSD. + + %% The emulator will dump a core here, so we set the cwd + %% to a temporary directory that we delete when the test is done. + {ok, Cwd} = file:get_cwd(), + TestDir = filename:join(proplists:get_value(priv_dir, Config), + "jmsingle"), + ok = file:make_dir(TestDir), + try + ok = file:set_cwd(TestDir), + run_jmsingle_test("true", internal_error, + "Emulator did not print the correct diagnostic " + "(crashed?) with +JMsingle true") + after + file:set_cwd(Cwd), + file:del_dir_r(TestDir) + end; + true -> + run_jmsingle_test("true", true, + "Emulator did not start with +JMsingle true") + end, + run_jmsingle_test("false", true, + "Emulator did not start with +JMsingle false"), + run_jmsingle_test("broken", false, + "Emulator started with bad +JMsingle parameter"). + get_tmp_asm_files() -> {ok, Cwd} = file:get_cwd(), filelib:wildcard(filename:join(Cwd, "*.asm")). diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index 23c072ce86..8769dea81f 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -796,6 +796,9 @@ t_is_map_key(Config) when is_list(Config) -> true = is_map_key({1,1.0}, M1), true = is_map_key(<<"k2">>, M1), + true = is_map_key(id(a), M1), + true = is_map_key(id("hello"), M1), + %% error cases do_badmap(fun(T) -> {'EXIT',{{badmap,T},[{erlang,is_map_key,_,_}|_]}} = diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl index 1100f6cf46..7bf24cf63d 100644 --- a/erts/emulator/test/mtx_SUITE.erl +++ b/erts/emulator/test/mtx_SUITE.erl @@ -275,7 +275,7 @@ hammer_sched_rwlock_test(FreqRead, LockCheck, Blocking, WaitLocked, WaitUnlocked _ -> {_, RunTime} = statistics(runtime), io:format("RunTime=~p~n", [RunTime]), - true = RunTime < 700, + true = RunTime < max(700,8*Onln), {comment, "Run-time during test was " ++ integer_to_list(RunTime) diff --git a/erts/emulator/test/op_SUITE.erl b/erts/emulator/test/op_SUITE.erl index 1f1f34d35f..3f343a4100 100644 --- a/erts/emulator/test/op_SUITE.erl +++ b/erts/emulator/test/op_SUITE.erl @@ -25,7 +25,7 @@ -export([all/0, suite/0, bsl_bsr/1,logical/1,t_not/1,relop_simple/1,relop/1, complex_relop/1,unsafe_fusing/1, - range_tests/1]). + range_tests/1, combined_relops/1]). -import(lists, [foldl/3,flatmap/2]). @@ -35,7 +35,8 @@ suite() -> all() -> [bsl_bsr, logical, t_not, relop_simple, relop, - complex_relop, unsafe_fusing, range_tests]. + complex_relop, unsafe_fusing, range_tests, + combined_relops]. %% Test the bsl and bsr operators. bsl_bsr(Config) when is_list(Config) -> @@ -523,7 +524,7 @@ range_tests(_Config) -> inside = range_big(MinSmall), inside = range_big(-1 bsl 58), inside = range_big(0), - inside = range_barely_small(17.75), + inside = range_big(17.75), inside = range_big(1 bsl 58), inside = range_big(MaxSmall), @@ -531,6 +532,39 @@ range_tests(_Config) -> greater = range_big(1 bsl 64), greater = range_big(float(1 bsl 64)), + inside = int_range_1(id(-100_000)), + inside = int_range_1(id(-10)), + inside = int_range_1(id(100)), + inside = int_range_1(id(100_000)), + + outside = int_range_1(id(atom)), + outside = int_range_1(id(-1 bsl 60)), + outside = int_range_1(id(-100_001)), + outside = int_range_1(id(100_001)), + outside = int_range_1(id(1 bsl 60)), + + inside = int_range_2(id(1)), + inside = int_range_2(id(42)), + inside = int_range_2(id(16#f000_0000)), + + outside = int_range_2(id([a,list])), + outside = int_range_2(id(0)), + outside = int_range_1(id(-1 bsl 60)), + outside = int_range_1(id(1 bsl 60)), + + inside = int_range_3(id(1 bsl 28)), + inside = int_range_3(id((1 bsl 28) + 1)), + inside = int_range_3(id((1 bsl 33) + 555)), + inside = int_range_3(id((1 bsl 58) - 1)), + inside = int_range_3(id(1 bsl 58)), + + outside = int_range_3(id({a,tuple})), + outside = int_range_3(id(-1 bsl 60)), + outside = int_range_3(id(-1000)), + outside = int_range_3(id(100)), + outside = int_range_3(id((1 bsl 58) + 1)), + outside = int_range_3(id(1 bsl 60)), + ok. range(X) -> @@ -683,6 +717,221 @@ range_big_2(X) when (-1 bsl 59) - 1 =< X, X =< 1 bsl 59 -> range_big_2(_) -> outside. +int_range_1(X) when is_integer(X), -100_000 =< X, X =< 100_000 -> + inside; +int_range_1(_) -> + outside. + +int_range_2(X) when is_integer(X), 1 =< X, X =< 16#f000_0000 -> + inside; +int_range_2(_) -> + outside. + +int_range_3(X) when is_integer(X), 1 bsl 28 =< X, X =< 1 bsl 58 -> + inside; +int_range_3(_) -> + outside. + +combined_relops(_Config) -> + other = test_tok_char(-1 bsl 64), + other = test_tok_char($A - 1), + + var = test_tok_char($A), + var = test_tok_char($B), + var = test_tok_char($P), + var = test_tok_char($Y), + var = test_tok_char($Z), + + other = test_tok_char($Z + 1), + + var = tok_char($_), + other = tok_char(float($_)), + + other = test_tok_char(1 bsl 64), + + other = test_tok_char(atom), + other = test_tok_char(self()), + + %% + b = ge_ge_int_range_1(-200), + b = ge_ge_int_range_1(-101), + + a = ge_ge_int_range_1(-100), + a = ge_ge_int_range_1(-50), + a = ge_ge_int_range_1(-10), + + b = ge_ge_int_range_1(-9), + b = ge_ge_int_range_1(-6), + + a = ge_ge_int_range_1(-5), + + b = ge_ge_int_range_1(-4), + b = ge_ge_int_range_1(0), + b = ge_ge_int_range_1(42), + + %% + b = ge_ge_int_range_2(-1 bsl 59), + + a = ge_ge_int_range_2((-1 bsl 59) + 1), + a = ge_ge_int_range_2(-1 bsl 58), + a = ge_ge_int_range_2(-1000), + a = ge_ge_int_range_2(1 bsl 58), + a = ge_ge_int_range_2((1 bsl 59) - 10), + + b = ge_ge_int_range_2((1 bsl 59) - 9), + + a = ge_ge_int_range_2((1 bsl 59) - 5), + + b = ge_ge_int_range_2((1 bsl 59) - 4), + b = ge_ge_int_range_2((1 bsl 59) - 1), + + %% + b = ge_ge_int_range_3(-1 bsl 59), + + b = ge_ge_int_range_3((-1 bsl 59) + 1), + b = ge_ge_int_range_3(-1 bsl 58), + b = ge_ge_int_range_3(-1000), + b = ge_ge_int_range_3(1 bsl 58), + + a = ge_ge_int_range_3((1 bsl 59) - 20), + a = ge_ge_int_range_3((1 bsl 59) - 15), + a = ge_ge_int_range_3((1 bsl 59) - 10), + + b = ge_ge_int_range_3((1 bsl 59) - 9), + + a = ge_ge_int_range_3((1 bsl 59) - 5), + + b = ge_ge_int_range_3((1 bsl 59) - 4), + b = ge_ge_int_range_3((1 bsl 59) - 1), + + %% + b = ge_ge_int_range_4(-1 bsl 59), + + a = ge_ge_int_range_4((-1 bsl 59) + 1), + a = ge_ge_int_range_4((-1 bsl 59) + 3), + a = ge_ge_int_range_4((-1 bsl 59) + 5), + + b = ge_ge_int_range_4((-1 bsl 59) + 6), + b = ge_ge_int_range_4((-1 bsl 59) + 9), + + a = ge_ge_int_range_4((-1 bsl 59) + 10), + + b = ge_ge_int_range_4((-1 bsl 59) + 11), + + b = ge_ge_int_range_4(0), + b = ge_ge_int_range_4(1000), + + b = ge_ge_int_range_4((1 bsl 59) - 1), + + %% Test a sequence that can't occur in optimized code: + %% is_ge Fail Src 10 + %% is_ge Fail Src 5 + Module = {?FUNCTION_NAME,[{test,1}],[], + [{function, test, 1, 2, + [{label,1}, + {line,[{location,"t.erl",4}]}, + {func_info,{atom,?FUNCTION_NAME},{atom,test},1}, + {label,2}, + {test,is_ge,{f,4}, + [{tr,{x,0},{t_integer,{0,1000}}}, + {integer,10}]}, + {test,is_ge, + {f,3}, + [{tr,{x,0},{t_integer,{0,1000}}}, + {integer,5}]}, + {label,3}, + {move,{atom,a},{x,0}}, + return, + {label,4}, + {move,{atom,b},{x,0}}, + return]}], + 5}, + + {ok,Mod,Code} = compile:forms(Module, [from_asm,time,report]), + {module,Mod} = code:load_binary(Mod, Mod, Code), + + b = Mod:test(0), + b = Mod:test(5), + b = Mod:test(9), + + a = Mod:test(10), + a = Mod:test(11), + a = Mod:test(1000), + + true = code:delete(Mod), + _ = code:purge(Mod), + + ok. + +test_tok_char(C) -> + Result = tok_char(C), + if + is_integer(C) -> + Result = tok_char(float(C)), + Result = tok_char_int(C), + if + C band 16#FFFF =:= C -> + Result = tok_char_int_range(C); + true -> + Result + end; + true -> + Result + end. + +%% is_ge + is_lt +tok_char(C) when $A =< C, C =< $Z -> + var; +tok_char($_) -> + var; +tok_char(_) -> + other. + +%% is_ge + is_ge +tok_char_int(C) when $A =< C, C =< $Z -> + var; +tok_char_int($_) -> + var; +tok_char_int(_) -> + other. + +%% is_ge + is_ge +tok_char_int_range(C) when $A =< C, C =< $Z -> + var; +tok_char_int_range($_) -> + var; +tok_char_int_range(_) -> + other. + +%% is_ge + is_ge +ge_ge_int_range_1(X) when -100 =< X, X =< -10 -> + a; +ge_ge_int_range_1(-5) -> + a; +ge_ge_int_range_1(_) -> + b. + +ge_ge_int_range_2(X) when (-1 bsl 59) + 1 =< X, X =< (1 bsl 59) - 10 -> + a; +ge_ge_int_range_2((1 bsl 59) - 5) -> + a; +ge_ge_int_range_2(_) -> + b. + +ge_ge_int_range_3(X) when (1 bsl 59) - 20 =< X, X =< (1 bsl 59) - 10 -> + a; +ge_ge_int_range_3((1 bsl 59) - 5) -> + a; +ge_ge_int_range_3(_) -> + b. + +ge_ge_int_range_4(X) when (-1 bsl 59) + 1 =< X, X =< (-1 bsl 59) + 5 -> + a; +ge_ge_int_range_4((-1 bsl 59) + 10) -> + a; +ge_ge_int_range_4(_) -> + b. + %%% %%% Utilities. %%% diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl index 35216aa78d..39dd2d3428 100644 --- a/erts/emulator/test/persistent_term_SUITE.erl +++ b/erts/emulator/test/persistent_term_SUITE.erl @@ -604,27 +604,54 @@ collisions_delete([], _) -> colliding_keys() -> %% Collisions found by find_colliding_keys() below - L = [[77674392,148027], - [103370644,950908], - [106444046,870178], - [22217246,735880], - [18088843,694607], - [63426007,612179], - [117354942,906431], - [121434305,94282311,816072], - [118441466,93873772,783366], - [124338174,1414801,123089], - [20240282,17113486,923647], - [126495528,61463488,164994], - [125341723,5729072,445539], - [127450932,80442669,348245], - [123354692,85724182,14241288,180793], - [99159367,65959274,61680971,289939], - [107637580,104512101,62639807,181644], - [139547511,51654420,2062545,151944], - [88078274,73031465,53388204,428872], - [141314238,75761379,55699508,861797], - [88045216,59272943,21030492,180903]], + %% ct:timetrap({minutes, 60}), + %% ct:pal("Colliding keys = ~p", [find_colliding_keys()]), + Collisions = + #{ + %% Collisions for Jenkins96 hashing. + 1268203079 => [[77674392,148027], + [103370644,950908], + [106444046,870178], + [22217246,735880], + [18088843,694607], + [63426007,612179], + [117354942,906431], + [121434305,94282311,816072], + [118441466,93873772,783366], + [124338174,1414801,123089], + [20240282,17113486,923647], + [126495528,61463488,164994], + [125341723,5729072,445539], + [127450932,80442669,348245], + [123354692,85724182,14241288,180793], + [99159367,65959274,61680971,289939], + [107637580,104512101,62639807,181644], + [139547511,51654420,2062545,151944], + [88078274,73031465,53388204,428872], + [141314238,75761379,55699508,861797], + [88045216,59272943,21030492,180903]], + %% Collisions for CRC32-C hashing. + 1982459178 => [[-4294967296,654663773], + [-3758096384,117792861], + [-3221225472,1728405597], + [-2684354560,1191534685], + [-2147483648,2706162303], + [-1610612736,2169291391], + [-1073741824,3779904127], + [-536870912,3243033215], + [-3640303523,0], + [-4177174435,536870912], + [-2566561699,1073741824], + [-3103432611,1610612736], + [-1588804993,2147483648], + [-2125675905,2684354560], + [-515063169,3221225472], + [-1051934081,3758096384]] + }, + + Key = internal_hash(2), + ct:pal("internal_hash(2) = ~p", [Key]), + #{ Key := L } = Collisions, %% Verify that the keys still collide (this will fail if the %% internal hash function has been changed). @@ -649,57 +676,47 @@ internal_hash(Term) -> erts_debug:get_internal_state({internal_hash,Term}). %% Use this function to (re)generate the list in colliding_keys/0 +%% +%% Grab a coffee, it will take a while. find_colliding_keys() -> - MaxCollSz = 4, - OfEachSz = 7, erts_debug:set_internal_state(available_internal_state, true), - MaxInserts = 1 bsl 20, - T = ets:new(x, [set, private]), - ok = fck_loop_1(T, 1, MaxInserts, MaxCollSz, OfEachSz), - fck_collect(T, MaxCollSz, OfEachSz, []). - -fck_collect(_T, 1, _OfEachSz, Acc) -> - Acc; -fck_collect(T, CollSz, OfEachSz, Acc) -> - {List, _} = ets:select(T, - [{{'$1','$2'}, [{'==',{length,'$2'},CollSz}], ['$2']}], - OfEachSz), - fck_collect(T, CollSz-1, OfEachSz, List ++ Acc). - - -fck_loop_1(T, Key, 0, MaxCollSz, MaxSzLeft) -> - fck_loop_2(T, Key, MaxCollSz, MaxSzLeft); -fck_loop_1(T, Key, Inserts, MaxCollSz, MaxSzLeft) -> - Hash = internal_hash(Key), - case ets:insert_new(T, {Hash, [Key]}) of - true -> - fck_loop_1(T, Key+1, Inserts-1, MaxCollSz, MaxSzLeft); - false -> - [{Hash, KeyList}] = ets:lookup(T, Hash), - true = ets:insert(T, {Hash, [Key | KeyList]}), - fck_loop_1(T, Key+1, Inserts, MaxCollSz, MaxSzLeft) - end. - -fck_loop_2(_T, _Key, _MaxCollSz, 0) -> - ok; -fck_loop_2(T, Key, MaxCollSz, MaxSzLeft0) -> - Hash = internal_hash(Key), - case ets:lookup(T, Hash) of - [] -> - fck_loop_2(T, Key+1, MaxCollSz, MaxSzLeft0); - [{Hash, KeyList}] -> - true = ets:insert(T, {Hash, [Key | KeyList]}), - MaxSzLeft1 = case length(KeyList)+1 of - MaxCollSz -> - MaxSzLeft0 - 1; - _ -> - MaxSzLeft0 - end, - fck_loop_2(T, Key+1, MaxCollSz, MaxSzLeft1) + NumScheds = erlang:system_info(schedulers_online), + Start = -(1 bsl 32), + End = -Start, + Range = End - Start, + Step = Range div NumScheds, + timer:tc(fun() -> fck_spawn(NumScheds, NumScheds, Start, End, Step, []) end). + +fck_spawn(0, _NumScheds, _Start, _End, _Step, Refs) -> + fck_await(Refs); +fck_spawn(N, NumScheds, Start, End, Step, Refs) -> + Key = Start + (N - 1) * Step, + {_, Ref} = spawn_monitor(fun() -> exit(fck_finder(Start, End, Key)) end), + fck_spawn(N - 1, NumScheds, Start, End, Step, [Ref | Refs]). + +fck_await([Ref | Refs]) -> + receive + {'DOWN', Ref, _, _, [_Initial]} -> + %% Ignore slices where the initial value only collided with itself. + fck_await(Refs); + {'DOWN', Ref, _, _, Collisions} -> + [Collisions | fck_await(Refs)] + end; +fck_await([]) -> + []. + +fck_finder(Start, End, Key) -> + true = Key >= Start, true = Key < End, %Assertion. + fck_finder_1(Start, End, internal_hash(Key)). + +fck_finder_1(Same, Same, _Target) -> + []; +fck_finder_1(Next, End, Target) -> + case internal_hash(Next) =:= Target of + true -> [Next | fck_finder_1(Next + 1, End, Target)]; + false -> fck_finder_1(Next + 1, End, Target) end. - - %% OTP-17700 Bug skipped refc++ of shared magic reference shared_magic_ref(_Config) -> Ref = atomics:new(10, []), diff --git a/erts/emulator/test/small_SUITE.erl b/erts/emulator/test/small_SUITE.erl index 3a0c0c5ede..b76a838590 100644 --- a/erts/emulator/test/small_SUITE.erl +++ b/erts/emulator/test/small_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0, suite/0, groups/0]). -export([edge_cases/1, - addition/1, subtraction/1, multiplication/1, division/1, + addition/1, subtraction/1, negation/1, multiplication/1, division/1, test_bitwise/1, test_bsl/1, element/1, range_optimization/1]). @@ -40,7 +40,7 @@ all() -> groups() -> [{p, [parallel], [edge_cases, - addition, subtraction, multiplication, division, + addition, subtraction, negation, multiplication, division, test_bitwise, test_bsl, element, range_optimization]}]. @@ -158,45 +158,88 @@ add_gen_pairs() -> gen_add_function({Name,{A,B}}) -> APlusOne = abs(A) + 1, BPlusOne = abs(B) + 1, - ?Q("'@Name@'(X0, Y0) when is_integer(X0), is_integer(Y0)-> + ?Q("'@Name@'(integer, X0, Y0) when is_integer(X0), is_integer(Y0)-> X1 = X0 rem _@APlusOne@, Y1 = Y0 rem _@BPlusOne@, Res = X0 + Y0, Res = X1 + Y1, Res = Y1 + X1, Res = X0 + Y1, - Res = X1 + Y0. "). + Res = X1 + Y0; + '@Name@'(number0, X, Y) when is_number(X), is_number(Y), + X < _@APlusOne@, 0 =< Y, Y < _@BPlusOne@ -> + Res = X + Y, + Res = Y + X; + '@Name@'(number0, X, Y) when is_number(X), is_number(Y), + X < _@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + Res = X + Y, + Res = Y + X; + '@Name@'(number1, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, 0 =< Y, Y < _@BPlusOne@ -> + Res = X + Y, + Res = Y + X; + '@Name@'(number1, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + Res = X + Y, + Res = Y + X; + '@Name@'(number1, X, Y) when X > -_@APlusOne@, Y > -_@BPlusOne@ -> + Res = X + Y, + Res = Y + X. "). test_addition([{Name,{A,B}}|T], Mod) -> + F = fun Mod:Name/3, try Res0 = A + B, - Res0 = Mod:Name(A, B), + Res0 = F(integer, A, B), + Res0 = F(number0, A, B), + Res0 = F(number1, A, B), Res1 = -A + B, - Res1 = Mod:Name(-A, B), + Res1 = F(integer, -A, B), + Res1 = F(number0, -A, B), + Res1 = F(number1, -A, B), Res2 = A + (-B), - Res2 = Mod:Name(A, -B), + Res2 = F(integer, A, -B), + Res2 = F(number0, A, -B), + Res2 = F(number1, A, -B), Res3 = -A + (-B), - Res3 = Mod:Name(-A, -B) + Res3 = F(integer, -A, -B), + Res3 = F(number0, -A, -B), + Res3 = F(number1, -A, -B), + + AbsB = abs(B), + Res4 = A + AbsB, + Res4 = F(number0, A, AbsB), + Res4 = F(number1, A, AbsB) catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,A,B]), erlang:raise(C, R, Stk) end, + bad_arith(F, [a], B), + bad_arith(F, aa, B), + bad_arith(F, A, [b]), + bad_arith(F, A, bb), + bad_arith(F, {a,b}, {c,d}), + test_addition(T, Mod); test_addition([], _) -> ok. +bad_arith(F, A, B) -> + {'EXIT',{badarith,_}} = catch F(number1, A, B), + ok. + %% Test that the JIT only omits the overflow check when it's safe. subtraction(_Config) -> _ = rand:uniform(), %Seed generator io:format("Seed: ~p", [rand:export_seed()]), Mod = list_to_atom(lists:concat([?MODULE,"_",?FUNCTION_NAME])), Pairs = sub_gen_pairs(), - io:format("~p\n", [Pairs]), + %% io:format("~p\n", [Pairs]), Fs0 = gen_func_names(Pairs, 0), Fs = [gen_sub_function(F) || F <- Fs0], Tree = ?Q(["-module('@Mod@').", @@ -222,38 +265,134 @@ sub_gen_pairs() -> gen_sub_function({Name,{A,B}}) -> APlusOne = abs(A) + 1, BPlusOne = abs(B) + 1, - ?Q("'@Name@'(X0, Y0) when is_integer(X0), is_integer(Y0)-> + ?Q("'@Name@'(integer, X0, Y0) when is_integer(X0), is_integer(Y0)-> X1 = X0 rem _@APlusOne@, Y1 = Y0 rem _@BPlusOne@, Res = X0 - Y0, Res = X1 - Y1, Res = X0 - Y1, - Res = X1 - Y0. "). + Res = X1 - Y0; + '@Name@'(number0, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, 0 =< Y, Y < _@BPlusOne@ -> + X - Y; + '@Name@'(number0, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + X - Y; + '@Name@'(number1, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, 0 =< Y, Y < _@BPlusOne@ -> + X - Y; + '@Name@'(number1, X, Y) when is_number(X), is_number(Y), + X > -_@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + X - Y; + '@Name@'(number1, X, Y) when X > -_@APlusOne@, Y > -_@BPlusOne@ -> + X - Y. "). test_subtraction([{Name,{A,B}}|T], Mod) -> + F = fun Mod:Name/3, try Res0 = A - B, - Res0 = Mod:Name(A, B), + Res0 = F(integer, A, B), + Res0 = F(number0, A, B), Res1 = -A - B, - Res1 = Mod:Name(-A, B), + Res1 = F(integer, -A, B), + Res1 = F(number0, -A, B), Res2 = A - (-B), - Res2 = Mod:Name(A, -B), + Res2 = F(integer, A, -B), + Res2 = F(number0, A, -B), Res3 = -A - (-B), - Res3 = Mod:Name(-A, -B) + Res3 = F(integer, -A, -B), + Res3 = F(number0, -A, -B), + + AbsB = abs(B), + Res4 = A - AbsB, + Res4 = F(integer, A, AbsB), + Res4 = F(number0, A, AbsB) catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,A,B]), erlang:raise(C, R, Stk) end, + bad_arith(F, [a], B), + bad_arith(F, aa, B), + bad_arith(F, A, [b]), + bad_arith(F, A, bb), + bad_arith(F, {a,b}, {c,d}), + test_subtraction(T, Mod); test_subtraction([], _) -> ok. %% Test that the JIT only omits the overflow check when it's safe. +negation(_Config) -> + _ = rand:uniform(), %Seed generator + io:format("Seed: ~p", [rand:export_seed()]), + Mod = list_to_atom(lists:concat([?MODULE,"_",?FUNCTION_NAME])), + Integers = neg_gen_integers(), + %% io:format("~p\n", [Pairs]), + Fs0 = gen_func_names(Integers, 0), + Fs = [gen_neg_function(F) || F <- Fs0], + Tree = ?Q(["-module('@Mod@').", + "-compile([export_all,nowarn_export_all])."]) ++ Fs, + merl:print(Tree), + {ok,_Bin} = merl:compile_and_load(Tree, []), + test_negation(Fs0, Mod), + ok. + +neg_gen_integers() -> + {MinSmall, MaxSmall} = determine_small_limits(0), + + N = 1000, + M = MaxSmall + N div 2, + Ns = [M - rand:uniform(N) || _ <- lists:seq(1, 75)], + + lists:seq(MinSmall-2, MinSmall+2) ++ lists:seq(MaxSmall-2, MaxSmall+2) ++ Ns. + +gen_neg_function({Name,A}) -> + APlusOne = abs(A) + 1, + ?Q("'@Name@'(integer0, X0) when is_integer(X0) -> + X1 = X0 rem _@APlusOne@, + Res = -X0, + Res = -X1; + '@Name@'(integer1, X) when is_integer(X), X > -_@APlusOne@ -> + -X; + '@Name@'(integer2, X) when is_integer(X), X > -_@APlusOne@ -> + -X; + '@Name@'(number, X) when is_number(X), X > -_@APlusOne@ -> + -X; + '@Name@'(number, X) when is_number(X), X > -_@APlusOne@ -> + -X; + '@Name@'(number, X) when is_number(X) -> + -X. "). + +test_negation([{Name,A}|T], Mod) -> + F = fun Mod:Name/2, + try + Res0 = -A, + Res0 = F(integer0, A), + Res0 = F(integer1, A), + Res0 = F(integer2, A), + Res0 = F(number, A), + + Res1 = A, + Res1 = F(integer0, -A), + Res1 = F(integer1, -A), + Res1 = F(integer2, -A), + Res1 = F(number, -A) + catch + C:R:Stk -> + io:format("~p failed. numbers: ~p\n", [Name,A]), + erlang:raise(C, R, Stk) + end, + + test_negation(T, Mod); +test_negation([], _) -> + ok. + +%% Test that the JIT only omits the overflow check when it's safe. multiplication(_Config) -> _ = rand:uniform(), %Seed generator io:format("Seed: ~p", [rand:export_seed()]), @@ -295,7 +434,7 @@ gen_mul_function({Name,{A,B}}) -> BPlusOne = B + 1, NumBitsA = num_bits(A), NumBitsB = num_bits(B), - ?Q("'@Name@'(X0, Y0, More) when is_integer(X0), is_integer(Y0)-> + ?Q("'@Name@'(X0, Y0, More) when is_integer(X0), is_integer(Y0), is_boolean(More) -> X1 = X0 rem _@APlusOne@, Y1 = Y0 rem _@BPlusOne@, Res = X0 * Y0, @@ -311,20 +450,36 @@ gen_mul_function({Name,{A,B}}) -> Res = X2 * Y1; true -> Res - end. "). + end; + '@Name@'(X, Y, number) when -_@APlusOne@ < X, X < _@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + Res = X * Y, + Res = Y * X; + '@Name@'(X, fixed, number) when -_@APlusOne@ < X, X < _@APlusOne@ -> + X * _@B@; + '@Name@'(fixed, Y, number) when -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> + _@A@ * Y. "). test_multiplication([{Name,{A,B}}|T], Mod) -> + F = fun Mod:Name/3, try Res0 = A * B, %% io:format("~p * ~p = ~p; size = ~p\n", %% [A,B,Res0,erts_debug:flat_size(Res0)]), - Res0 = Mod:Name(A, B, true), - Res0 = Mod:Name(-A, -B, false), + Res0 = F(A, B, true), + Res0 = F(-A, -B, false), + Res0 = F(A, B, number), + Res0 = F(fixed, B, number), + Res0 = F(A, fixed, number), + Res0 = F(-A, -B, number), Res1 = -(A * B), - Res1 = Mod:Name(-A, B, false), - Res1 = Mod:Name(A, -B, false) + Res1 = F(-A, B, false), + Res1 = F(A, -B, false), + Res1 = F(-A, B, number), + Res1 = F(A, -B, number), + Res1 = F(-A, fixed, number), + Res1 = F(fixed, -B, number) catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,A,B]), @@ -367,16 +522,22 @@ div_gen_pairs() -> NumBitsMaxSmall = num_bits(MaxSmall), %% Generate random pairs of smalls. - Pairs0 = [{rand:uniform(MaxSmall),rand:uniform(MaxSmall)} || + Pairs0 = [{rand:uniform(MaxSmall) * rand_sign(), + rand:uniform(MaxSmall) * rand_sign()} || _ <- lists:seq(1, 75)], Pairs1 = [{rand:uniform(MaxSmall), N} || - N <- [-3,-2,-1,1,2,3,5,17,63,64,1111,22222]] ++ Pairs0, + N <- [-4,-3,-2,-1,1,2,3,5,17,63,64,1111,22222]] ++ Pairs0, %% Generate pairs of numbers whose product are bignums. [{rand:uniform(MaxSmall),1 bsl Pow} || Pow <- lists:seq(NumBitsMaxSmall - 4, NumBitsMaxSmall - 1)] ++ Pairs1. +rand_sign() -> + case rand:uniform() < 0.2 of + true -> -1; + false -> 1 + end. gen_div_function({Name,{A,B}}) -> APlusOne = abs(A) + 1, @@ -405,6 +566,12 @@ gen_div_function({Name,{A,B}}) -> R = X rem Y, Q = X div Y, {Q, R}; + '@Name@'(integer3, X, fixed) when is_integer(X), -_@APlusOne@ < X, X < _@APlusOne@ -> + Y = _@B@, + Q = X div Y, + put(prevent_div_rem_fusion, Q), + R = X rem Y, + {Q, R}; '@Name@'(number0, X, Y) when -_@APlusOne@ < X, X < _@APlusOne@, -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> Q = X div Y, @@ -414,6 +581,22 @@ gen_div_function({Name,{A,B}}) -> -_@BPlusOne@ < Y, Y < _@BPlusOne@ -> R = X rem Y, Q = X div Y, + {Q, R}; + '@Name@'(number2, X, fixed) when -_@APlusOne@ < X, X < _@APlusOne@ -> + Y = _@B@, + Q = X div Y, + R = X rem Y, + {Q, R}; + '@Name@'(number3, X, fixed) when -_@APlusOne@ < X, X < _@APlusOne@ -> + Y = _@B@, + R = X rem Y, + Q = X div Y, + {Q, R}; + '@Name@'(number4, X, fixed) when -_@APlusOne@ < X, X < _@APlusOne@ -> + Y = _@B@, + Q = X div Y, + put(prevent_div_rem_fusion, Q), + R = X rem Y, {Q, R}. "). test_division([{Name,{A,B}}|T], Mod) -> @@ -423,8 +606,12 @@ test_division([{Name,{A,B}}|T], Mod) -> Res0 = F(integer0, A, B), Res0 = F(integer1, A, fixed), Res0 = F(integer2, A, fixed), + Res0 = F(integer3, A, fixed), Res0 = F(number0, A, B), - Res0 = F(number1, A, B) + Res0 = F(number1, A, B), + Res0 = F(number2, A, fixed), + Res0 = F(number3, A, fixed), + Res0 = F(number4, A, fixed) catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,A,B]), @@ -473,31 +660,59 @@ gen_bitwise_function({Name,{A,B}}) -> Y1 = Y0 rem _@BPlusOne@, AndRes = X0 band Y0, + AndRes = Y0 band X0, AndRes = X1 band Y1, AndRes = Y1 band X1, AndRes = X0 band Y1, AndRes = X1 band Y0, OrRes = X0 bor Y0, + OrRes = Y0 bor X0, OrRes = X1 bor Y1, OrRes = Y1 bor X1, OrRes = X0 bor Y1, OrRes = X1 bor Y0, XorRes = X0 bxor Y0, + XorRes = Y0 bxor X0, XorRes = X1 bxor Y1, XorRes = Y1 bxor X1, XorRes = X0 bxor Y1, XorRes = X1 bxor Y0, - {AndRes, OrRes, XorRes}. "). + {AndRes, OrRes, XorRes}; + '@Name@'(X0, fixed) when is_integer(X0) -> + X1 = X0 rem _@APlusOne@, + + AndRes = X0 band _@B@, + AndRes = _@B@ band X0, + AndRes = X1 band _@B@, + AndRes = _@B@ band X1, + + OrRes = X0 bor _@B@, + OrRes = _@B@ bor X0, + OrRes = X1 bor _@B@, + OrRes = _@B@ bor X1, + + XorRes = X0 bxor _@B@, + XorRes = _@B@ bxor X0, + XorRes = X1 bxor _@B@, + XorRes = _@B@ bxor X1, + + {AndRes, OrRes, XorRes}. + "). test_bitwise([{Name,{A,B}}|T], Mod) -> try test_bitwise_1(A, B, Mod, Name), test_bitwise_1(-A, B, Mod, Name), test_bitwise_1(A, -B, Mod, Name), - test_bitwise_1(-A, -B, Mod, Name) + test_bitwise_1(-A, -B, Mod, Name), + + AndRes = A band B, + OrRes = A bor B, + XorRes = A bxor B, + {AndRes, OrRes, XorRes} = Mod:Name(A, fixed) catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,A,B]), @@ -541,15 +756,42 @@ bsl_gen_pairs() -> gen_bsl_function({Name,{N,S}}) -> Mask = (1 bsl num_bits(N)) - 1, - ?Q("'@Name@'(N0) -> + ?Q("'@Name@'(N0, fixed) -> + Res = N0 bsl _@S@, N = N0 band _@Mask@, - N bsl _@S@. "). + Res = N0 bsl _@S@, + Res = N bsl _@S@; + '@Name@'(N0, S) when is_integer(S), 0 =< S, S =< _@S@ -> + Res = N0 bsl S, + N = N0 band _@Mask@, + Res = N0 bsl S, + Res = N bsl S. "). test_bsl([{Name,{N,S}}|T], Mod) -> - Res = N bsl S, - try Mod:Name(N) of - Res -> - ok + try + Res0 = N bsl S, + Res0 = Mod:Name(N, fixed), + + if + S >= 0 -> + Res1 = N bsl S, + Res1 = Mod:Name(N, S), + + N = Mod:Name(N, 0), + + if + S >= 2 -> + Res2 = N bsl (S - 1), + Res2 = Mod:Name(N, S - 1), + + Res3 = N bsl (S - 2), + Res3 = Mod:Name(N, S - 2); + true -> + ok + end; + S < 0 -> + {'EXIT', {function_clause,_}} = catch Mod:Name(N, S) + end catch C:R:Stk -> io:format("~p failed. numbers: ~p ~p\n", [Name,N,S]), diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index c230aa9f19..014c0a114e 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -382,7 +382,7 @@ run_scheduler_wall_time_test(Type) -> Pid end, StartDirtyHog = fun(Func) -> - F = fun () -> + F = fun() -> erts_debug:Func(alive_waitexiting, MeMySelfAndI) end, @@ -470,7 +470,7 @@ online_statistics(Stats) -> DirtyCPUSchedulersOnline = erlang:system_info(dirty_cpu_schedulers_online), DirtyIOSchedulersOnline = erlang:system_info(dirty_io_schedulers), SortedStats = lists:sort(Stats), - ct:pal("Stats: ~p~n", [SortedStats]), + ct:log("Stats: ~p~n", [SortedStats]), SchedulersStats = lists:sublist(SortedStats, 1, SchedulersOnline), DirtyCPUSchedulersStats = diff --git a/erts/emulator/test/tuple_SUITE_data/get_two_tuple_elements.S b/erts/emulator/test/tuple_SUITE_data/get_two_tuple_elements.S index dc873fe73f..5774424284 100644 --- a/erts/emulator/test/tuple_SUITE_data/get_two_tuple_elements.S +++ b/erts/emulator/test/tuple_SUITE_data/get_two_tuple_elements.S @@ -63,7 +63,7 @@ {trim,2,0}. {line,[{location,"get_two_tuple_elements.erl",5}]}. {call,2,{f,5}}. - {'%',{var_info,{x,0},[{type,number}]}}. + {'%',{var_info,{x,0},[{type,{t_number,any}}]}}. {test,is_eq_exact,{f,3},[{x,0},{integer,3}]}. {move,{atom,ok},{x,0}}. {deallocate,0}. diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index da850ed7cb..c7ea032026 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -1141,6 +1141,7 @@ sub emulator_output { print '#include "erl_map.h"', "\n"; print '#include "big.h"', "\n"; print '#include "erl_bits.h"', "\n"; + print '#include "erl_binary.h"', "\n"; print '#include "beam_transform_helpers.h"', "\n"; print '#include "erl_global_literals.h"', "\n"; print "\n"; diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index 42d4395eb2..2a569d3513 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -168,7 +168,6 @@ INSTALL_PROGS = \ $(BINDIR)/erlsrv.exe \ $(BINDIR)/erl.exe \ $(BINDIR)/erl_log.exe\ - $(BINDIR)/werl.exe \ $(BINDIR)/$(ERLEXEC) \ $(INSTALL_EMBEDDED_PROGS) @@ -216,6 +215,12 @@ INSTALL_PROGS = \ $(INSTALL_EMBEDDED_PROGS) endif +CREATE_DIRS=$(OBJDIR) $(BINDIR) + +ifneq ($(strip $(CREATE_DIRS)),) +_create_dirs := $(shell mkdir -p $(CREATE_DIRS)) +endif + .PHONY: etc etc: $(ENTRY_OBJ) $(INSTALL_PROGS) $(EXTRA_LIBS) $(INSTALL_LIBS) $(TEXTFILES) $(INSTALL_TOP_BIN) @@ -267,13 +272,12 @@ endif rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/vxcall.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl_log.o - rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/werl.o rm -f $(TEXTFILES) rm -f *~ core #------------------------------------------------------------------------ # Windows specific targets -# The windows platform is quite different from the others. erl/werl are small C programs +# The windows platform is quite different from the others. erl are small C programs # loading a DLL. INI files are used instead of environment variables and the Install # script is actually a program, also Install has an INI file which tells of emulator # versions etc. @@ -287,9 +291,6 @@ $(BINDIR)/$(ERLEXEC): $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init $(BINDIR)/erl@EXEEXT@: $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) $(V_LD) $(LDFLAGS) -o $@ $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) -$(BINDIR)/werl@EXEEXT@: $(OBJDIR)/werl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) - $(V_LD) $(LDFLAGS) -o $@ $(OBJDIR)/werl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) - $(BINDIR)/erl_log@EXEEXT@: $(OBJDIR)/erl_log.o $(V_LD) $(LDFLAGS) -o $@ $(OBJDIR)/erl_log.o @@ -367,10 +368,6 @@ $(OBJDIR)/erlsrv_util.o: $(WINETC)/erlsrv/erlsrv_util.c $(ERLSRV_HEADERS) \ $(OBJDIR)/erlsrv_logmess.h $(RC_GENERATED) $(V_CC) $(CFLAGS) -I$(OBJDIR) $(MT_FLAG) -o $@ -c $< -$(OBJDIR)/werl.o: $(WINETC)/erl.c $(WINETC)/init_file.h $(RC_GENERATED) - $(V_CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \ - -DWIN32_WERL -o $@ -c $(WINETC)/erl.c - $(OBJDIR)/erl_log.o: $(WINETC)/erl_log.c $(RC_GENERATED) $(V_CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \ -o $@ -c $(WINETC)/erl_log.c diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c index 0e9c2bea83..522c427691 100644 --- a/erts/etc/common/ct_run.c +++ b/erts/etc/common/ct_run.c @@ -77,7 +77,6 @@ static void* emalloc(size_t size); static void efree(void *p); #endif static char* strsave(char* string); -static void push_words(char* src); static int run_erlang(char* name, char** argv); static char* get_default_emulator(char* progname); #ifdef __WIN32__ @@ -187,8 +186,7 @@ int main(int argc, char** argv) eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); eargv = eargv_base; eargc = 0; - push_words(emulator); - free(emulator); + PUSH(emulator); eargc_base = eargc; eargv = eargv + eargv_size/2; eargc = 0; @@ -319,26 +317,6 @@ int main(int argc, char** argv) return run_erlang(eargv[0], eargv); } -static void -push_words(char* src) -{ - char sbuf[MAXPATHLEN]; - char* dst; - - dst = sbuf; - while ((*dst++ = *src++) != '\0') { - if (isspace((int)*src)) { - *dst = '\0'; - PUSH(strsave(sbuf)); - dst = sbuf; - do { - src++; - } while (isspace((int)*src)); - } - } - if (sbuf[0]) - PUSH(strsave(sbuf)); -} #ifdef __WIN32__ wchar_t *make_commandline(char **argv) { diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c index 6cded37733..da378a8654 100644 --- a/erts/etc/common/erlc.c +++ b/erts/etc/common/erlc.c @@ -742,14 +742,14 @@ call_compile_server(char** argv) ei_x_encode_atom(&args, "encoding"); ei_x_encode_atom(&args, get_encoding()); ei_x_encode_atom(&args, "cwd"); - ei_x_encode_string(&args, cwd); + ei_x_encode_binary(&args, cwd, strlen(cwd)); ei_x_encode_atom(&args, "env"); encode_env(&args); ei_x_encode_atom(&args, "command_line"); argc = 0; while (argv[argc]) { ei_x_encode_list_header(&args, 1); - ei_x_encode_string(&args, possibly_unquote(argv[argc])); + ei_x_encode_binary(&args, possibly_unquote(argv[argc]), strlen(argv[argc])); argc++; } ei_x_encode_empty_list(&args); /* End of command_line */ @@ -773,7 +773,6 @@ call_compile_server(char** argv) /* * Decode the answer. */ - dec_index = 0; if (ei_decode_atom(reply.buff, &dec_index, atom) == 0 && strcmp(atom, "wrong_config") == 0) { diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 3e44ad2b77..f8e7fb09d7 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -39,14 +39,12 @@ #define DIRSEP "\\" #define PATHSEP ";" #define NULL_DEVICE "nul" -#define BINARY_EXT "" #define DLL_EXT ".dll" #define EMULATOR_EXECUTABLE "beam.dll" #else #define PATHSEP ":" #define DIRSEP "/" #define NULL_DEVICE "/dev/null" -#define BINARY_EXT "" #define EMULATOR_EXECUTABLE "beam" #endif @@ -219,7 +217,6 @@ static char* possibly_quote(char* arg); /* * Functions from win_erlexec.c */ -int start_win_emulator(char* emu, char *startprog,char** argv, int start_detached); int start_emulator(char* emu, char*start_prog, char** argv, int start_detached); #endif @@ -247,7 +244,7 @@ static const char* emu_flavor = DEFAULT_SUFFIX; /* Flavor of emulator (smp, jit #ifdef __WIN32__ static char *start_emulator_program = NULL; /* For detached mode - - erl.exe/werl.exe */ + erl.exe */ static char* key_val_name = ERLANG_VERSION; /* Used by the registry * access functions. */ @@ -257,7 +254,6 @@ static int config_script_cnt = 0; static int got_start_erl = 0; static HANDLE this_module_handle; -static int run_werl; static WCHAR *utf8_to_utf16(unsigned char *bytes); static char *utf16_to_utf8(WCHAR *wstr); static WCHAR *latin1_to_utf16(char *str); @@ -415,7 +411,7 @@ static void add_boot_config(void) #define NEXT_ARG_CHECK() NEXT_ARG_CHECK_NAMED(argv[i]) #ifdef __WIN32__ -__declspec(dllexport) int win_erlexec(int argc, char **argv, HANDLE module, int windowed) +__declspec(dllexport) int win_erlexec(int argc, char **argv, HANDLE module) #else int main(int argc, char **argv) #endif @@ -436,7 +432,6 @@ int main(int argc, char **argv) #ifdef __WIN32__ this_module_handle = module; - run_werl = windowed; /* if we started this erl just to get a detached emulator, * the arguments are already prepared for beam, so we skip * directly to start_emulator */ @@ -454,6 +449,18 @@ int main(int argc, char **argv) Eargsp[argc] = NULL; emu = argv[0]; start_emulator_program = strsave(argv[0]); + /* We set the stdandard handles to nul in order for prim_tty_nif + and erlang:display_string to work without returning ebadf for + detached emulators */ + SetStdHandle(STD_INPUT_HANDLE, + CreateFile("nul", GENERIC_READ, 0, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL)); + SetStdHandle(STD_OUTPUT_HANDLE, + CreateFile("nul", GENERIC_WRITE, 0, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL)); + SetStdHandle(STD_ERROR_HANDLE, + CreateFile("nul", GENERIC_WRITE, 0, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL)); goto skip_arg_massage; } free_env_val(s); @@ -523,7 +530,7 @@ int main(int argc, char **argv) emu = add_extra_suffixes(emu); emu_name = strsave(emu); - erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu); + erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s", bindir, emu); emu = strsave(tmpStr); s = get_env("ESCRIPT_NAME"); @@ -651,13 +658,12 @@ int main(int argc, char **argv) break; case 'd': - if (strcmp(argv[i], "-detached") != 0) { - add_arg(argv[i]); - } else { - start_detached = 1; - add_args("-noshell", "-noinput", NULL); - } - break; + add_arg(argv[i]); + if (strcmp(argv[i], "-detached") == 0) { + start_detached = 1; + add_args("-noshell", "-noinput", NULL); + } + break; case 'e': if (strcmp(argv[i], "-extra") == 0) { @@ -1117,24 +1123,7 @@ int main(int argc, char **argv) skip_arg_massage: /*DebugBreak();*/ - if (run_werl) { - if (start_detached) { - char *p; - /* transform werl to erl */ - p = start_emulator_program+strlen(start_emulator_program); - while (--p >= start_emulator_program && *p != '/' && *p != '\\' && - *p != 'W' && *p != 'w') - ; - if (p >= start_emulator_program && (*p == 'W' || *p == 'w') && - (p[1] == 'E' || p[1] == 'e') && (p[2] == 'R' || p[2] == 'r') && - (p[3] == 'L' || p[3] == 'l')) { - memmove(p,p+1,strlen(p)); - } - } - return start_win_emulator(emu, start_emulator_program, Eargsp, start_detached); - } else { - return start_emulator(emu, start_emulator_program, Eargsp, start_detached); - } + return start_emulator(emu, start_emulator_program, Eargsp, start_detached); #else @@ -1232,7 +1221,7 @@ usage_aux(void) "[-emu_type TYPE] [-emu_flavor FLAVOR] " "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c [BOOLEAN]] " "[+C MODE] [+dcg DECENTRALIZED_COUNTER_GROUPS_LIMIT] [+h HEAP_SIZE_OPTION] " - "[+J[Pperf] JIT_OPTION] " + "[+J[Pperf|Msingle] JIT_OPTION] " "[+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+Q MAX_PORTS] " "[+R COMPAT_REL] " "[+r] [+rg READER_GROUPS_LIMIT] [+s<SUBSWITCH> SCHEDULER_OPTION] " @@ -1600,6 +1589,14 @@ static void get_parameters(int argc, char** argv) emu = EMULATOR_EXECUTABLE; start_emulator_program = strsave(argv[0]); + /* in wsl argv[0] is given as "erl.exe", but start_emulator_program should be + an absolute path, so we prepend BINDIR to it */ + if (strcmp(start_emulator_program, "erl.exe") == 0) { + erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s", bindir, + start_emulator_program); + start_emulator_program = strsave(tmpStr); + } + free(ini_filename); } diff --git a/erts/etc/common/escript.c b/erts/etc/common/escript.c index 078937e676..e418daf430 100644 --- a/erts/etc/common/escript.c +++ b/erts/etc/common/escript.c @@ -510,8 +510,8 @@ main(int argc, char** argv) PUSH(emulator); PUSH("+B"); - PUSH2("-boot", "no_dot_erlang"); PUSH("-noshell"); + PUSH2("-boot", "no_dot_erlang"); /* * Read options from the %%! row in the script and add them as args diff --git a/erts/etc/common/etc_common.h b/erts/etc/common/etc_common.h index 289a33b42a..865cb6a6c6 100644 --- a/erts/etc/common/etc_common.h +++ b/erts/etc/common/etc_common.h @@ -35,6 +35,7 @@ # include <io.h> # include <winbase.h> # include <process.h> +# include <direct.h> // _getcwd #endif #include <errno.h> diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c index f9ca5f6373..6ebe793edc 100644 --- a/erts/etc/unix/to_erl.c +++ b/erts/etc/unix/to_erl.c @@ -82,13 +82,13 @@ # define STRERROR(x) "" #endif -#define noDEBUG +#define noDEBUG_TOERL #define PIPE_DIR "/tmp/" #define PIPE_STUBNAME "erlang.pipe" #define PIPE_STUBLEN strlen(PIPE_STUBNAME) -#ifdef DEBUG +#ifdef DEBUG_TOERL #define STATUS(s) { fprintf(stderr, (s)); fflush(stderr); } #else #define STATUS(s) @@ -106,7 +106,7 @@ static int protocol_ver = RUN_ERL_LO_VER; /* assume lowest to begin with */ static int write_all(int fd, const char* buf, int len); static int window_size_seq(char* buf, size_t bufsz); static int version_handshake(char* buf, int len, int wfd); -#ifdef DEBUG +#ifdef DEBUG_TOERL static void show_terminal_settings(struct termios *); #endif @@ -155,7 +155,7 @@ int main(int argc, char **argv) pipeIx = 2; } -#ifdef DEBUG +#ifdef DEBUG_TOERL fprintf(stderr, "%s: pid is : %d\n", argv[0], (int)getpid()); #endif @@ -209,25 +209,25 @@ int main(int argc, char **argv) } if ((rfd = open (FIFO1, O_RDONLY|DONT_BLOCK_PLEASE, 0)) < 0) { -#ifdef DEBUG +#ifdef DEBUG_TOERL fprintf(stderr, "Could not open FIFO %s for reading.\n", FIFO1); #endif fprintf(stderr, "No running Erlang on pipe %s: %s\n", pipename, strerror(errno)); exit(1); } -#ifdef DEBUG +#ifdef DEBUG_TOERL fprintf(stderr, "to_erl: %s opened for reading\n", FIFO1); #endif if ((wfd = open (FIFO2, O_WRONLY|DONT_BLOCK_PLEASE, 0)) < 0) { -#ifdef DEBUG +#ifdef DEBUG_TOERL fprintf(stderr, "Could not open FIFO %s for writing.\n", FIFO2); #endif fprintf(stderr, "No running Erlang on pipe %s: %s\n", pipename, strerror(errno)); close(rfd); exit(1); } -#ifdef DEBUG +#ifdef DEBUG_TOERL fprintf(stderr, "to_erl: %s opened for writing\n", FIFO2); #endif @@ -245,7 +245,7 @@ int main(int argc, char **argv) } tty_smode = tty_rmode; tty_eof = '\004'; /* Ctrl+D to exit */ -#ifdef DEBUG +#ifdef DEBUG_TOERL show_terminal_settings(&tty_rmode); #endif tty_smode.c_iflag = @@ -347,7 +347,7 @@ int main(int argc, char **argv) tcsetattr(0, TCSADRAIN, &tty_smode); -#ifdef DEBUG +#ifdef DEBUG_TOERL show_terminal_settings(&tty_smode); #endif /* @@ -420,7 +420,7 @@ int main(int argc, char **argv) } if (len) { -#ifdef DEBUG +#ifdef DEBUG_TOERL write_all(1, buf, len); #endif if (write_all(wfd, buf, len) != len) { @@ -580,7 +580,7 @@ static int version_handshake(char* buf, int len, int wfd) } -#ifdef DEBUG +#ifdef DEBUG_TOERL #define S(x) ((x) > 0 ? 1 : 0) static void show_terminal_settings(struct termios *t) diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index 1b8f894dc9..497dd537fd 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -24,6 +24,7 @@ */ #include <windows.h> +#include <winerror.h> #include <stdio.h> #include <stdlib.h> #include "init_file.h" @@ -47,11 +48,12 @@ int wmain(int argc, wchar_t **argv) InitFile *ini_file; InitSection *ini_section; HANDLE module = GetModuleHandle(NULL); - wchar_t *binaries[] = { L"erl.exe", L"werl.exe", L"erlc.exe", L"erl_call.exe", + wchar_t *binaries[] = { L"erl.exe", L"erlc.exe", L"erl_call.exe", L"dialyzer.exe", L"typer.exe", L"escript.exe", L"ct_run.exe", NULL }; wchar_t *scripts[] = { L"start_clean.boot", L"start_sasl.boot", L"no_dot_erlang.boot", NULL }; + wchar_t *links[][2] = { { L"erl.exe", L"werl.exe" }, NULL }; wchar_t fromname[MAX_PATH]; wchar_t toname[MAX_PATH]; size_t converted; @@ -175,7 +177,32 @@ int wmain(int argc, wchar_t **argv) fprintf(stderr,"Continuing installation anyway...\n"); } } - + + for (i = 0; links[i][0] != NULL; ++i) { + swprintf(toname,MAX_PATH,L"%s\\%s",bin_dir,links[i][1]); + if (!CreateSymbolicLinkW(toname,links[i][0],0)) { + DWORD err = GetLastError(); + if (err == ERROR_PRIVILEGE_NOT_HELD) { + fprintf(stderr,"Must be administrator to create link, copying %S instead.\n", + links[i][0]); + swprintf(fromname,MAX_PATH,L"%s\\%s",bin_dir,links[i][0]); + if (!CopyFileW(fromname,toname,FALSE)) { + fprintf(stderr,"Could not copy file %S to %S\n", + fromname,toname); + fprintf(stderr,"Continuing installation anyway...\n"); + } + } else { + wchar_t buf[256]; + FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buf, (sizeof(buf) / sizeof(wchar_t)), NULL); + fprintf(stderr,"Could not create links from %S to %S %d: %S\n", + fromname, toname, err, buf); + fprintf(stderr,"Continuing installation anyway...\n"); + } + } + } + for (i = 0; scripts[i] != NULL; ++i) { swprintf(fromname,MAX_PATH,L"%s\\%s",release_dir,scripts[i]); swprintf(toname,MAX_PATH,L"%s\\%s",bin_dir,scripts[i]); diff --git a/erts/etc/win32/Makefile b/erts/etc/win32/Makefile index c6376ebe74..f553f83e92 100644 --- a/erts/etc/win32/Makefile +++ b/erts/etc/win32/Makefile @@ -39,7 +39,6 @@ ROOTDIR = $(ERL_TOP)/erts INSTALL_PROGS = \ $(BINDIR)/inet_gethost.exe \ $(BINDIR)/erl.exe \ - $(BINDIR)/werl.exe \ $(BINDIR)/heart.exe \ $(BINDIR)/erlc.exe \ $(BINDIR)/erlsrv.exe \ diff --git a/erts/etc/win32/erl.c b/erts/etc/win32/erl.c index 99a41b99e5..31650de831 100644 --- a/erts/etc/win32/erl.c +++ b/erts/etc/win32/erl.c @@ -23,7 +23,7 @@ #include <stdlib.h> #include "init_file.h" -typedef int ErlexecFunction(int, char **, HANDLE, int); +typedef int ErlexecFunction(int, char **, HANDLE); #define INI_FILENAME L"erl.ini" #define INI_SECTION "erlang" @@ -35,18 +35,8 @@ static void error(char* format, ...); static wchar_t *erlexec_name; static wchar_t *erlexec_dir; -#ifdef WIN32_WERL -#define WERL 1 -int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, - PWSTR szCmdLine, int iCmdShow) -{ - int argc = __argc; - wchar_t **argv = __wargv; -#else -#define WERL 0 int wmain(int argc, wchar_t **argv) { -#endif HANDLE erlexec_handle; /* Instance */ ErlexecFunction *win_erlexec; wchar_t *path = malloc(100*sizeof(wchar_t)); @@ -120,7 +110,7 @@ int wmain(int argc, wchar_t **argv) } #endif - return (*win_erlexec)(argc,utf8argv,erlexec_handle,WERL); + return (*win_erlexec)(argc,utf8argv,erlexec_handle); } @@ -316,7 +306,6 @@ static void get_parameters(void) free(ini_filename); } - static void error(char* format, ...) { char sbuf[2048]; @@ -326,11 +315,6 @@ static void error(char* format, ...) vsprintf(sbuf, format, ap); va_end(ap); -#ifndef WIN32_WERL - fprintf(stderr, "%s\n", sbuf); -#else - MessageBox(NULL, sbuf, "Werl", MB_OK|MB_ICONERROR); -#endif + fprintf(stderr, "%s\n", sbuf); exit(1); } - diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi index ae933f59af..2e317fd362 100644 --- a/erts/etc/win32/nsis/erlang20.nsi +++ b/erts/etc/win32/nsis/erlang20.nsi @@ -190,7 +190,7 @@ cp_files: CreateDirectory "$SMPROGRAMS\$STARTMENU_FOLDER"
continue_create:
CreateShortCut "$SMPROGRAMS\$STARTMENU_FOLDER\Erlang.lnk" \
- "$INSTDIR\bin\werl.exe"
+ "$INSTDIR\bin\erl.exe"
!insertmacro MUI_STARTMENU_WRITE_END
; And once again, the verbosity...
diff --git a/erts/etc/win32/win_erlexec.c b/erts/etc/win32/win_erlexec.c index 7b21ed3785..53b1ac92b0 100644 --- a/erts/etc/win32/win_erlexec.c +++ b/erts/etc/win32/win_erlexec.c @@ -48,7 +48,6 @@ static char* win32_errorstr(int error); static int has_console(void); static char** fnuttify_argv(char **argv); static void free_fnuttified(char **v); -static int windowed = 0; #ifdef LOAD_BEAM_DYNAMICALLY typedef int SysGetKeyFunction(int); @@ -133,103 +132,6 @@ free_env_val(char *value) free(value); } - -int -start_win_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_detached) -{ - int len; - int argc = 0; - - windowed = 1; - while (utf8argv[argc] != NULL) { - ++argc; - } - - if (start_detached) { - wchar_t *start_prog=NULL; - int result; - int i; - wchar_t **argv; - close(0); - close(1); - close(2); - - set_env("ERL_CONSOLE_MODE", "detached"); - set_env(DLL_ENV, utf8emu); - - utf8argv[0] = utf8start_prog; - utf8argv = fnuttify_argv(utf8argv); - - len = MultiByteToWideChar(CP_UTF8, 0, utf8start_prog, -1, NULL, 0); - start_prog = malloc(len*sizeof(wchar_t)); - MultiByteToWideChar(CP_UTF8, 0, utf8start_prog, -1, start_prog, len); - - /* Convert utf8argv to multibyte argv */ - argv = malloc((argc+1) * sizeof(wchar_t*)); - for (i=0; i<argc; i++) { - len = MultiByteToWideChar(CP_UTF8, 0, utf8argv[i], -1, NULL, 0); - argv[i] = malloc(len*sizeof(wchar_t)); - MultiByteToWideChar(CP_UTF8, 0, utf8argv[i], -1, argv[i], len); - } - argv[argc] = NULL; - -#ifdef ARGS_HARDDEBUG - { - wchar_t tempbuf[2048] = L""; - wchar_t *sbuf; - int i; - sbuf=tempbuf; - sbuf += swprintf(sbuf, 2048, L"utf16: %s\n", start_prog); - for (i = 0; i < argc; ++i) { - sbuf += swprintf(sbuf, 2048, L"|%s|", argv[i]); - }; - sbuf += swprintf(sbuf, 2048, L"\nutf8: \n"); - for (i = 0; i < argc; ++i) { - sbuf += swprintf(sbuf, 2048, L"|%S|", utf8argv[i]); - }; - MessageBoxW(NULL, tempbuf, L"respawn args", MB_OK|MB_ICONERROR); - } -#endif - - result = _wspawnv(_P_DETACH, start_prog, argv); - free_fnuttified(utf8argv); - if (result == -1) { - error("Failed to execute %S: %s", start_prog, win32_errorstr(_doserrno)); - } - } else { - wchar_t *emu=NULL; -#ifdef LOAD_BEAM_DYNAMICALLY - HMODULE beam_module = NULL; - len = MultiByteToWideChar(CP_UTF8, 0, utf8emu, -1, NULL, 0); - emu = malloc(len*sizeof(wchar_t)); - MultiByteToWideChar(CP_UTF8, 0, utf8emu, -1, emu, len); -#ifdef ARGS_HARDDEBUG - { - char sbuf[2048] = ""; - int i; - strcat(sbuf,utf8emu); - strcat(sbuf,":"); - for (i = 0; i < argc; ++i) { - strcat(sbuf,"|"); - strcat(sbuf, utf8argv[i]); - strcat(sbuf,"| "); - } - MessageBox(NULL, sbuf, "erl_start args", MB_OK|MB_ICONERROR); - } -#endif - beam_module = load_win_beam_dll(emu); -#endif - set_env("ERL_CONSOLE_MODE", "window"); -#ifdef LOAD_BEAM_DYNAMICALLY - (*sys_primitive_init_p)(beam_module); - (*erl_start_p)(argc,utf8argv); -#else - erl_start(argc,utf8argv); -#endif - } - return 0; -} - void __cdecl do_keep_window(void) { @@ -244,8 +146,6 @@ do_keep_window(void) int start_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_detached) { - static char console_mode[] = "tty:ccc"; - char* fd_type; char* title; int len; int argc = 0; @@ -254,8 +154,6 @@ start_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_d fprintf(stderr,"utf8emu = %s, start_prog = %s\n", utf8emu, utf8start_prog); #endif - fd_type = strchr(console_mode, ':'); - fd_type++; _flushall(); while (utf8argv[argc] != NULL) { @@ -312,7 +210,14 @@ start_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_d if (result == (HANDLE)-1) { #ifdef ARGS_HARDDEBUG - MessageBox(NULL, "_wspawnv failed","Start detached",MB_OK); + wchar_t buf[256], buffer[256], path[1024]; + DWORD err = GetLastError(); + FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buf, (sizeof(buf) / sizeof(wchar_t)), NULL); + GetEnvironmentVariableW(L"PATH",path,sizeof(path) / sizeof(wchar_t)); + wsprintfW(buffer,L"_wspawnv failed %s(%d)\nPATH=%s", buf, err, path); + MessageBoxW(NULL, buffer , L"Start detached",MB_OK); #endif return 1; } @@ -339,7 +244,7 @@ start_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_d } #endif beam_module = load_win_beam_dll(emu); -#endif +#endif /* * Start the emulator. @@ -350,8 +255,8 @@ start_emulator(char* utf8emu, char *utf8start_prog, char** utf8argv, int start_d SetConsoleTitle(title); } free_env_val(title); - - set_env("ERL_CONSOLE_MODE", console_mode); + + set_env("ERL_CONSOLE_MODE", "windowed"); if (keep_window) { atexit(do_keep_window); } @@ -387,10 +292,8 @@ error(char* format, ...) vsprintf(sbuf, format, ap); va_end(ap); - if (!windowed && has_console()) { + if (has_console()) { fprintf(stderr, "%s\n", sbuf); - } else { - MessageBox(NULL, sbuf, "Werl", MB_OK|MB_ICONERROR); } exit(1); } diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 043ef2dbd0..392d10f493 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -94,7 +94,11 @@ CFLAGS += -DERTS_OPCODE_COUNTER_SUPPORT PRE_LD= else override TYPE=opt +ifeq (@JIT_ENABLED@, yes) +OMIT_OMIT_FP=yes +else OMIT_FP=true +endif TYPE_SUFFIX= PRE_LD= endif @@ -311,15 +315,6 @@ include $(YCF_SOURCE_DIR)/main_target.mk $(OBJ_DIR)/MADE: $(YCF_EXECUTABLE) $(ETHREAD_LIB) $(ERTS_INTERNAL_LIBS) $(gen_verbose) -ifeq ($(OMIT_OMIT_FP),yes) - @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' - @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' - @echo '* * * *' - @echo '* * NOTE: Omit frame pointer optimization has been omitted * *' - @echo '* * * *' - @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' - @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' -endif $(V_at)echo $? > $(OBJ_DIR)/MADE # diff --git a/erts/prebuild.keep b/erts/prebuild.keep index a2e6bd485b..8e695ec83a 100644 --- a/erts/prebuild.keep +++ b/erts/prebuild.keep @@ -1 +1 @@ -doc/ +doc diff --git a/erts/preloaded/ebin/atomics.beam b/erts/preloaded/ebin/atomics.beam Binary files differindex 6ab249d8fc..96da98160f 100644 --- a/erts/preloaded/ebin/atomics.beam +++ b/erts/preloaded/ebin/atomics.beam diff --git a/erts/preloaded/ebin/counters.beam b/erts/preloaded/ebin/counters.beam Binary files differindex 2d0580cc89..6a048aed01 100644 --- a/erts/preloaded/ebin/counters.beam +++ b/erts/preloaded/ebin/counters.beam diff --git a/erts/preloaded/ebin/erl_init.beam b/erts/preloaded/ebin/erl_init.beam Binary files differindex b0ee881e2f..66cbf7ee77 100644 --- a/erts/preloaded/ebin/erl_init.beam +++ b/erts/preloaded/ebin/erl_init.beam diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 8612761a9c..93f01c4b5b 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam Binary files differindex 380f5d8ea9..9d45a377ae 100644 --- a/erts/preloaded/ebin/erl_tracer.beam +++ b/erts/preloaded/ebin/erl_tracer.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 759d4de03f..eb5b3a72e7 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex 7918e37d6a..3905dc19de 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam b/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam Binary files differindex b63e4dd6fc..f64885905b 100644 --- a/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam +++ b/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex c2a9d1fbea..e202c2f648 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differindex 165bfa4622..48a0ed10b2 100644 --- a/erts/preloaded/ebin/erts_literal_area_collector.beam +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 93c95568ad..e145c6d4a7 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/persistent_term.beam b/erts/preloaded/ebin/persistent_term.beam Binary files differindex eee6b18bd4..329c8472f9 100644 --- a/erts/preloaded/ebin/persistent_term.beam +++ b/erts/preloaded/ebin/persistent_term.beam diff --git a/erts/preloaded/ebin/prim_buffer.beam b/erts/preloaded/ebin/prim_buffer.beam Binary files differindex 70b331a353..ebd7dd42b3 100644 --- a/erts/preloaded/ebin/prim_buffer.beam +++ b/erts/preloaded/ebin/prim_buffer.beam diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam Binary files differindex 7253202ea4..82728e68b3 100644 --- a/erts/preloaded/ebin/prim_eval.beam +++ b/erts/preloaded/ebin/prim_eval.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex 4bb0a1b9e7..e9ca16f8a6 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 5f61ab8cba..62acb5a0ba 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_net.beam b/erts/preloaded/ebin/prim_net.beam Binary files differindex a39a3d1115..c868a24b5e 100644 --- a/erts/preloaded/ebin/prim_net.beam +++ b/erts/preloaded/ebin/prim_net.beam diff --git a/erts/preloaded/ebin/prim_socket.beam b/erts/preloaded/ebin/prim_socket.beam Binary files differindex 7eb287597f..fe76f4be67 100644 --- a/erts/preloaded/ebin/prim_socket.beam +++ b/erts/preloaded/ebin/prim_socket.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 8bee02d875..25f86f98bf 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/socket_registry.beam b/erts/preloaded/ebin/socket_registry.beam Binary files differindex c8f4e9e983..d10f1649d4 100644 --- a/erts/preloaded/ebin/socket_registry.beam +++ b/erts/preloaded/ebin/socket_registry.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex 53f355e465..7c4e0ce5e3 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 470f861e08..2703e865cd 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -216,7 +216,7 @@ -export([crc32/2, crc32_combine/3, date/0, decode_packet/3]). -export([delete_element/2]). -export([delete_module/1, demonitor/1, demonitor/2, display/1]). --export([display_nl/0, display_string/1, erase/0, erase/1]). +-export([display_string/1, display_string/2, erase/0, erase/1]). -export([error/1, error/2, error/3, exit/1, exit/2, exit_signal/2, external_size/1]). -export([external_size/2, finish_after_on_load/2, finish_loading/1, float/1]). -export([float_to_binary/1, float_to_binary/2, @@ -843,15 +843,21 @@ unalias(_Alias) -> display(_Term) -> erlang:nif_error(undefined). -%% display_nl/0 --spec erlang:display_nl() -> true. -display_nl() -> - erlang:nif_error(undefined). - %% display_string/1 -spec erlang:display_string(P1) -> true when + P1 :: string() | binary(). +display_string(String) -> + try erlang:display_string(stderr, String) + catch error:badarg:ST -> + [{erlang, display_string, _, [ErrorInfo]}|_] = ST, + erlang:error(badarg, [String], [ErrorInfo]) + end. + +%% display_string/2 +-spec erlang:display_string(Device, P1) -> true when + Device :: stdin | stdout | stderr, P1 :: string(). -display_string(_P1) -> +display_string(_Stream,_P1) -> erlang:nif_error(undefined). %% dt_append_vm_tag_data/1 @@ -1204,8 +1210,13 @@ halt() -> %% halt/1 %% Shadowed by erl_bif_types: erlang:halt/1 --spec halt(Status) -> no_return() when - Status :: non_neg_integer() | 'abort' | string(). +-spec halt(Status :: non_neg_integer()) -> + no_return(); + (Abort :: abort) -> + no_return(); + (CrashDumpSlogan :: string()) -> + no_return(). + -dialyzer({no_return, halt/1}). halt(Status) -> try @@ -1216,11 +1227,18 @@ halt(Status) -> %% halt/2 %% Shadowed by erl_bif_types: erlang:halt/2 --spec halt(Status, Options) -> no_return() when - Status :: non_neg_integer() | 'abort' | string(), - Options :: [Option], - Option :: {flush, boolean()}. -halt(_Status, _Options) -> +-type halt_options() :: + [{flush, boolean()}]. + +-spec halt(Status :: non_neg_integer(), Options :: halt_options()) -> + no_return(); + (Abort :: abort, Options :: halt_options()) -> + no_return(); + (CrashDumpSlogan :: string(), Options :: halt_options()) -> + no_return(). + +-dialyzer({no_return, halt/2}). +halt(_, _) -> erlang:nif_error(undefined). %% has_prepared_code_on_load/1 @@ -2197,8 +2215,9 @@ get_module_info(_Module, _Item) -> erlang:nif_error(undefined). %% Shadowed by erl_bif_types: erlang:hd/1 --spec hd(List) -> term() when - List :: [term(), ...]. +-spec hd(List) -> Head when + List :: nonempty_maybe_improper_list(), + Head :: term(). hd(_List) -> erlang:nif_error(undefined). @@ -2774,8 +2793,9 @@ term_to_iovec(_Term, _Options) -> erlang:nif_error(undefined). %% Shadowed by erl_bif_types: erlang:tl/1 --spec tl(List) -> term() when - List :: nonempty_maybe_improper_list(). +-spec tl(List) -> Tail when + List :: nonempty_maybe_improper_list(), + Tail :: term(). tl(_List) -> erlang:nif_error(undefined). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 5ea349511d..9d28d214fa 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -53,7 +53,7 @@ get_argument/1,script_id/0,script_name/0]). %% for the on_load functionality; not for general use --export([run_on_load_handlers/0]). +-export([run_on_load_handlers/0, run_on_load_handlers/1]). %% internal exports -export([fetch_loaded/0,ensure_loaded/1,make_permanent/2, @@ -396,8 +396,8 @@ boot_loop(BootPid, State) -> loop(State#state{status = {started,PS}, subscribed = []}); {'EXIT',BootPid,Reason} -> - erlang:display({"init terminating in do_boot",Reason}), - crash("init terminating in do_boot", [Reason]); + % erlang:display({"init terminating in do_boot",Reason}), + crash("Runtime terminating during boot", [Reason]); {'EXIT',Pid,Reason} -> Kernel = State#state.kernel, terminate(Pid,Kernel,Reason), %% If Pid is a Kernel pid, halt()! @@ -462,9 +462,9 @@ new_kernelpid({_Name,ignore},BootPid,State) -> BootPid ! {self(),ignore}, State; new_kernelpid({Name,What},BootPid,State) -> - erlang:display({"could not start kernel pid",Name,What}), + % erlang:display({"could not start kernel pid",Name,What}), clear_system(false,BootPid,State), - crash("could not start kernel pid", [Name, What]). + crash("Could not start kernel pid", [Name, What]). %% Here is the main loop after the system has booted. @@ -558,7 +558,7 @@ do_handle_msg(Msg,State) -> true -> logger:info("init got unexpected: ~p", [X], #{ error_logger=>#{tag=>info_msg}}); false -> - erlang:display_string("init got unexpected: "), + erlang:display_string(stdout, "init got unexpected: "), erlang:display(X) end end @@ -833,8 +833,9 @@ del(_Item, []) -> []. terminate(Pid,Kernel,Reason) -> case kernel_pid(Pid,Kernel) of {ok,Name} -> + %% If you change this time, also change the time in logger_simple_h.erl sleep(500), %% Flush error printouts! - erlang:display({"Kernel pid terminated",Name,Reason}), + % erlang:display({"Kernel pid terminated",Name,Reason}), crash("Kernel pid terminated", [Name, Reason]); _ -> false @@ -1225,21 +1226,54 @@ start_it([]) -> ok; start_it({eval,Bin}) -> Str = b2s(Bin), - {ok,Ts,_} = erl_scan:string(Str), - Ts1 = case reverse(Ts) of - [{dot,_}|_] -> Ts; - TsR -> reverse([{dot,erl_anno:new(1)} | TsR]) - end, - {ok,Expr} = erl_parse:parse_exprs(Ts1), - {value, _Value, _Bs} = erl_eval:exprs(Expr, erl_eval:new_bindings()), - ok; -start_it([_|_]=MFA) -> - case MFA of - [M] -> M:start(); - [M,F] -> M:F(); - [M,F|Args] -> M:F(Args) % Args is a list + try + {ok,Ts,_} = erl_scan:string(Str), + Ts1 = case reverse(Ts) of + [{dot,_}|_] -> Ts; + TsR -> reverse([{dot,erl_anno:new(1)} | TsR]) + end, + {ok,Expr} = erl_parse:parse_exprs(Ts1), + {value, _Value, _Bs} = erl_eval:exprs(Expr, erl_eval:new_bindings()), + ok + catch E:R:ST -> + Message = [<<"Error! Failed to eval: ">>, Bin, <<"\r\n\r\n">>], + erlang:display_string(binary_to_list(iolist_to_binary(Message))), + erlang:raise(E,R,ST) + end; +start_it([M|FA]) -> + case code:ensure_loaded(M) of + {module, M} -> + case FA of + [] -> M:start(); + [F] -> M:F(); + [F|Args] -> M:F(Args) % Args is a list + end; + + {error, Reason} -> + Message = [explain_ensure_loaded_error(M, Reason), <<"\r\n\r\n">>], + erlang:display_string(binary_to_list(iolist_to_binary(Message))), + erlang:error(undef) end. +explain_ensure_loaded_error(M, badfile) -> + S = [<<"it requires a more recent Erlang/OTP version " + "or its .beam file was corrupted.\r\n" + "(You are running Erlang/OTP ">>, + erlang:system_info(otp_release), <<".)">>], + explain_add_head(M, S); +explain_ensure_loaded_error(M, nofile) -> + S = <<"it cannot be found. Make sure that the module name is correct and\r\n", + "that its .beam file is in the code path.">>, + explain_add_head(M, S); +explain_ensure_loaded_error(M, Other) -> + [<<"Error! Failed to load module '", (atom_to_binary(M))/binary, + "'. Reason: ">>, + atom_to_binary(Other)]. + +explain_add_head(M, S) -> + [<<"Error! Failed to load module '", (atom_to_binary(M))/binary, + "' because ">>, S]. + %% Load a module. do_load_module(Mod, BinCode) -> @@ -1456,8 +1490,11 @@ archive_extension() -> %%% run_on_load_handlers() -> + run_on_load_handlers(all). + +run_on_load_handlers(Mods) when is_list(Mods); Mods =:= all -> Ref = monitor(process, ?ON_LOAD_HANDLER), - catch ?ON_LOAD_HANDLER ! run_on_load, + catch ?ON_LOAD_HANDLER ! {run_on_load, self(), Ref, Mods}, receive {'DOWN',Ref,process,_,noproc} -> %% There is no on_load handler process, @@ -1470,7 +1507,9 @@ run_on_load_handlers() -> {'DOWN',Ref,process,_,Res} -> %% Failure to run an on_load handler. %% This is fatal during start-up. - exit(Res) + exit(Res); + {reply, Ref, on_load_done} -> + ok end. start_on_load_handler_process() -> @@ -1486,9 +1525,14 @@ on_load_loop(Mods, Debug0) -> on_load_loop(Mods, Debug); {loaded,Mod} -> on_load_loop([Mod|Mods], Debug0); - run_on_load -> + {run_on_load, _, _, all} -> run_on_load_handlers(Mods, Debug0), - exit(on_load_done) + exit(on_load_done); + {run_on_load, From, Ref, ModsToRun} -> + [run_on_load_handlers([Mod], Debug0) + || Mod <- ModsToRun, lists:member(Mod, Mods)], + From ! {reply, Ref, on_load_done}, + on_load_loop(Mods -- ModsToRun, Debug0) end. run_on_load_handlers([M|Ms], Debug) -> diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl index 449aedf301..a0417051b7 100644 --- a/erts/test/erlc_SUITE.erl +++ b/erts/test/erlc_SUITE.erl @@ -28,6 +28,7 @@ compile_yecc/1, compile_script/1, compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1, make_dep_options/1, + unicode_paths/1, features_erlc_describe/1, features_erlc_unknown/1, features_directives/1, @@ -56,7 +57,8 @@ groups() -> tests() -> [compile_erl, compile_yecc, compile_script, compile_mib, - good_citizen, deep_cwd, arg_overflow, make_dep_options]. + good_citizen, deep_cwd, arg_overflow, make_dep_options, + unicode_paths]. feature_tests() -> [features_erlc_describe, @@ -487,6 +489,28 @@ make_dep_options(Config) -> false = exists(BeamFileName), ok. +unicode_paths(Config) -> + case {os:type(), file:native_name_encoding()} of + {{win32,_}, _} -> {skip, "Unicode paths not supported on windows"}; + {_,latin1} -> {skip, "Cannot interpret unicode filenames when native_name_encoding is latin1"}; + _ -> + DepRE = ["_OK_"], + {SrcDir,OutDir0,Cmd0} = get_cmd(Config), + OutDir = filename:join(OutDir0,"😀"), + ok = case file:make_dir(OutDir) of + {error, eexist} -> ok; + ok -> ok; + E -> E + end, + Cmd = Cmd0 ++ " +brief -o "++OutDir, + FileName = filename:join([SrcDir, "😀", "erl_test_unicode.erl"]), + BeamFileName = filename:join(OutDir, "erl_test_unicode.beam"), + run(Config, Cmd, FileName, "", DepRE), + true = exists(BeamFileName), + file:delete(BeamFileName), + file:delete(OutDir) + end, + ok. %%% Tests related to the features mechanism %% Support macros and functions @@ -951,9 +975,10 @@ features_runtime(Config) when is_list(Config) -> experimental_ftr_1, experimental_ftr_2], Approved = [approved_ftr_2, - approved_ftr_1], + approved_ftr_1, + maybe_expr], - {Compile, _SrcDir, _OutDir} = compile_fun(Config), + {_Compile, _SrcDir, _OutDir} = compile_fun(Config), {Peer0, Node0} = peer([]), @@ -986,14 +1011,14 @@ features_runtime(Config) when is_list(Config) -> peer:stop(Peer0), {Peer1, Node1} = peer(["-enable-feature", "experimental_ftr_2"]), - [experimental_ftr_2, approved_ftr_2, approved_ftr_1] = + [experimental_ftr_2, approved_ftr_2, approved_ftr_1, maybe_expr] = erpc:call(Node1, erl_features, enabled, []), [while, until, unless] = erpc:call(Node1, erl_features, keywords, []), peer:stop(Peer1), {Peer2, Node2} = peer(["-disable-feature", "all"]), - [] = erpc:call(Node2, erl_features, enabled, []), + [maybe_expr] = erpc:call(Node2, erl_features, enabled, []), [] = erpc:call(Node2, erl_features, keywords, []), peer:stop(Peer2), diff --git a/erts/test/erlc_SUITE_data/src/😀/erl_test_unicode.erl b/erts/test/erlc_SUITE_data/src/😀/erl_test_unicode.erl new file mode 100644 index 0000000000..32a208b7e2 --- /dev/null +++ b/erts/test/erlc_SUITE_data/src/😀/erl_test_unicode.erl @@ -0,0 +1,25 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(erl_test_unicode). +-export(['😀'/1]). + +'😀'(0) -> + '😀'. diff --git a/lib/Makefile b/lib/Makefile index 067f00787b..2a2acfacf8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -56,28 +56,11 @@ ifdef BUILD_STATIC_LIBS SUB_DIRECTORIES = asn1 crypto else ifdef BOOTSTRAP - SUB_DIRECTORIES = \ - kernel stdlib compiler - else - ifdef SECONDARY_BOOTSTRAP - ifdef TINY_BUILD - SUB_DIRECTORIES = parsetools sasl - else - SUB_DIRECTORIES = parsetools asn1/src - endif - else - ifdef TERTIARY_BOOTSTRAP - SUB_DIRECTORIES = snmp sasl erl_interface jinterface syntax_tools wx public_key - else - ifdef DOC_BOOTSTRAP - SUB_DIRECTORIES = xmerl edoc erl_docgen public_key - else # Not bootstrap build - SUB_DIRECTORIES = $(ERTS_APPLICATIONS) \ - $(ERLANG_APPLICATIONS) \ - $(EXTRA_APPLICATIONS) - endif - endif - endif + SUB_DIRECTORIES = $(BOOTSTRAP) + else # Not bootstrap build + SUB_DIRECTORIES = $(ERTS_APPLICATIONS) \ + $(ERLANG_APPLICATIONS) \ + $(EXTRA_APPLICATIONS) endif endif diff --git a/lib/asn1/src/asn1_db.erl b/lib/asn1/src/asn1_db.erl index 7486fa2db9..28c1fc4563 100644 --- a/lib/asn1/src/asn1_db.erl +++ b/lib/asn1/src/asn1_db.erl @@ -82,11 +82,11 @@ loop(#state{parent = Parent, monitor = MRef, table = Table, includes = Includes} = State) -> receive {set, Mod, K2, V} -> - [{_, Modtab}] = ets:lookup(Table, Mod), + Modtab = ets:lookup_element(Table, Mod, 2), ets:insert(Modtab, {K2, V}), loop(State); {set, Mod, Kvs} -> - [{_, Modtab}] = ets:lookup(Table, Mod), + Modtab = ets:lookup_element(Table, Mod, 2), ets:insert(Modtab, Kvs), loop(State); {From, {get, Mod, K2}} -> @@ -105,7 +105,7 @@ loop(#state{parent = Parent, monitor = MRef, table = Table, end, loop(State); {save, OutFile, Mod} -> - [{_,Mtab}] = ets:lookup(Table, Mod), + Mtab = ets:lookup_element(Table, Mod, 2), TempFile = OutFile ++ ".#temp", ok = ets:tab2file(Mtab, TempFile), ok = file:rename(TempFile, OutFile), @@ -146,10 +146,7 @@ get_table(Table, Mod, Includes) -> end. lookup(Tab, K) -> - case ets:lookup(Tab, K) of - [] -> undefined; - [{K,V}] -> V - end. + ets:lookup_element(Tab, K, 2, undefined). info(EruleMaps) -> {asn1ct:vsn(),EruleMaps}. diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index de10339c41..3ab83f6262 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -20,6 +20,7 @@ %% %% -module(asn1ct). +-feature(maybe_expr, enable). %% Compile Time functions for ASN.1 (e.g ASN.1 compiler). @@ -2233,15 +2234,13 @@ maybe_rename_function2(Thing,Name,Suffix) %% generated_functions_member/4 checks on both Name and Pattern if %% the element exists in L generated_functions_member(M,Name,L,Pattern) -> - case generated_functions_member(M,Name,L) of - true -> - L2 = generated_functions_filter(M,Name,L), - case lists:keysearch(Pattern,3,L2) of - {value,_} -> - true; - _ -> false - end; - _ -> false + maybe + true ?= generated_functions_member(M,Name,L), + L2 = generated_functions_filter(M,Name,L), + {value,_} ?= lists:keysearch(Pattern,3,L2), + true + else + false -> false end. generated_functions_member(_M,Name,[{Name,_,_}|_]) -> diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index b94cc15374..b2d4a933b9 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -36,9 +36,6 @@ print_frequency/0 ]). -%%% Mandatory include --include_lib("common_test/include/ct.hrl"). - %%%================================================================ %%% %%% API diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 7b6f03311a..fa3613fdb9 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -51,7 +51,7 @@ %%---------------------------------------------------------------------- -module(cth_conn_log). --include_lib("common_test/include/ct.hrl"). +-include("ct.hrl"). -export([init/2, pre_init_per_testcase/4, diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index feb607b46e..d6a882e315 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -2789,7 +2789,8 @@ peer_name(Module, TestCase) -> peer:random_name(lists:concat([Module, "-", TestCase])). %% Command line arguments passed --spec start_peer([string()] | peer:start_options(), atom() | string(), TestCase :: atom() | string()) -> +-spec start_peer([string()] | peer:start_options() | #{ start_cover => boolean() }, + atom() | string(), TestCase :: atom() | string()) -> {ok, gen_statem:server_ref(), node()} | {error, term()}. start_peer(Args, Module, TestCase) when is_list(Args) -> start_peer(#{args => Args, name => peer_name(Module, TestCase)}, Module); @@ -2801,9 +2802,10 @@ start_peer(Opts, Module, TestCase) -> start_peer(Opts#{name => peer_name(Module, TestCase)}, Module). %% Release compatibility testing --spec start_peer([string()] | peer:start_options(), atom() | string(), TestCase :: atom() | string(), - Release :: string(), OutDir :: file:filename()) -> - {ok, gen_statem:server_ref(), node()} | {error, term()} | not_available. +-spec start_peer([string()] | peer:start_options() | #{ start_cover => boolean() }, + atom() | string(), TestCase :: atom() | string(), + Release :: string(), OutDir :: file:filename()) -> + {ok, gen_statem:server_ref(), node()} | {error, term()} | not_available. start_peer(Args, Module, TestCase, Release, OutDir) when is_list(Args) -> start_peer(#{args => Args}, Module, TestCase, Release, OutDir); start_peer(Opts, Module, TestCase, Release, OutDir) -> @@ -2815,8 +2817,8 @@ start_peer(Opts, Module, TestCase, Release, OutDir) -> %% for old releases. Keep ERL_FLAGS, and ERL_ZFLAGS for sometimes you might need it... Env = maps:get(env, Opts, []) ++ [{"ERL_AFLAGS", false}], NewArgs = ["-pa", peer_compile(Erl, code:which(peer), OutDir) | maps:get(args, Opts, [])], - start_peer(Opts#{exec => Erl, args => NewArgs, - env => Env}, Module, TestCase) + start_peer(Opts#{exec => Erl, args => NewArgs, env => Env, + start_cover => false }, Module, TestCase) end. %% Internal implementation @@ -2844,7 +2846,6 @@ start_peer(#{name := Name} = Opts, Module) -> Shutdown = binary_to_term(term_to_binary({10000, CoverMain})), case peer:start_link(Opts#{args => FullArgs, shutdown => Shutdown}) of {ok, Peer, Node} -> - do_cover_for_node(Node, start), {ok, Peer, Node}; Other -> Other @@ -2861,7 +2862,7 @@ peer_compile(Erl, cover_compiled, OutDir) -> peer_compile(Erl, ModPath, OutDir) -> {ok, ModSrc} = filelib:find_source(ModPath), Erlc = filename:join(filename:dirname(Erl), "erlc"), - cmd(Erlc, ["-o", OutDir, ModSrc]), + cmd(Erlc, ["-o", OutDir, unicode:characters_to_binary(ModSrc)]), OutDir. %% This should really be implemented as os:cmd. diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 538ea0de44..326a824b61 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -116,7 +116,8 @@ create_priv_dir=auto_per_run, finish=false, target_info, cover=false, wait_for_node=[], testcase_callback=undefined, idle_notify=[], - get_totals=false, random_seed=undefined}). + get_totals=false, random_seed=undefined, + old_releases=#{}}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% OPERATOR INTERFACE @@ -917,13 +918,19 @@ handle_call({is_release_available, Release}, _From, State) -> %% %% Find the path of the release's erl file if available -handle_call({find_release, Release}, _From, State) -> - R = - case test_server_node:is_release_available(Release) of - true -> test_server_node:find_release(Release); - _ -> not_available - end, - {reply, R, State}. +handle_call({find_release, Release}, From, State = #state{ old_releases = OldReleases }) -> + case maps:find(Release, OldReleases) of + error -> + R = + case test_server_node:find_release(Release) of + none -> not_available; + PathToRelease -> PathToRelease + end, + handle_call({find_release, Release}, From, + State#state{ old_releases = OldReleases#{ Release => R }}); + {ok, R} -> + {reply, R, State} + end. %%-------------------------------------------------------------------- set_hosts(Hosts) -> diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index 5d9ba008f4..7a6e00f974 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -471,7 +471,7 @@ find_release(Rel) -> none -> case string:take(Rel,"_",true) of {Rel,[]} -> - false; + none; {RelNum,_} -> find_release_path(RelNum) end; @@ -490,19 +490,19 @@ find_release_path([Path|T], Rel) -> false -> find_release_path(T, Rel); ErlExec -> - Pattern = filename:join([Path,"..","releases","*","OTP_VERSION"]), - case filelib:wildcard(Pattern) of - [VersionFile] -> - {ok, VsnBin} = file:read_file(VersionFile), - [MajorVsn|_] = string:lexemes(VsnBin, "."), - case unicode:characters_to_list(MajorVsn) of - Rel -> - ErlExec; - _Else -> - find_release_path(T, Rel) + QuotedExec = "\""++ErlExec++"\"", + Release = os:cmd(QuotedExec ++ " -noinput -eval 'io:format(\"~ts\", [erlang:system_info(otp_release)])' -s init stop"), + case Release =:= Rel of + true -> + %% Check is the release is a source tree release, + %% if so we should not use it. + case os:cmd(QuotedExec ++ " -noinput -eval 'io:format(\"~p\",[filelib:is_file(filename:join([code:root_dir(),\"OTP_VERSION\"]))]).' -s init stop") of + "true" -> + find_release_path(T, Rel); + "false" -> + ErlExec end; - _Else -> - find_release_path(T, Rel) + false -> find_release_path(T, Rel) end end; find_release_path([], _) -> diff --git a/lib/common_test/test_server/.gitignore b/lib/common_test/test_server/.gitignore new file mode 100644 index 0000000000..7c38aa35ba --- /dev/null +++ b/lib/common_test/test_server/.gitignore @@ -0,0 +1,4 @@ +conf_vars +config.log +config.status +variables diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 5828fc6a17..06e3d70fa3 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -826,7 +826,23 @@ module.beam: module.erl \ (or contract) for an exported or unexported function is not given. Use this option to turn on this kind of warning.</p> </item> - </taglist> + + <tag><c>nowarn_redefined_builtin_type</c></tag> + <item> + <p>By default, a warning is emitted when a built-in type + is locally redefined. Use this option to turn off this + kind of warning.</p> + </item> + + <tag><c>{nowarn_redefined_builtin_type, Types}</c></tag> + <item> + <p>By default, a warning is emitted when a built-in type + is locally redefined. Use this option to turn off this + kind of warning for the types in <c>Types</c>, where + <c>Types</c> is a tuple <c>{TypeName,Arity}</c> or a list + of such tuples.</p> + </item> +</taglist> <p>Other kinds of warnings are <em>opportunistic warnings</em>. They are generated when the compiler happens to diff --git a/lib/compiler/scripts/smoke b/lib/compiler/scripts/smoke index ae31c923b8..98e5fd55df 100755 --- a/lib/compiler/scripts/smoke +++ b/lib/compiler/scripts/smoke @@ -29,7 +29,7 @@ clone_elixir() -> true -> GetHeadSHA1 = "cd elixir && git rev-parse --verify HEAD", Before = os:cmd(GetHeadSHA1), - cmd("cd elixir && git pull --ff-only origin master"), + cmd("cd elixir && git pull --ff-only origin main"), case os:cmd(GetHeadSHA1) of Before -> ok; diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index d801d6baa0..2a720777f0 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -199,13 +199,17 @@ release_docs_spec: # Dependencies -- alphabetically, please # ---------------------------------------------------- -$(EBIN)/beam_asm.beam: beam_asm.hrl +$(EBIN)/beam_a.beam: beam_asm.hrl beam_types.hrl +$(EBIN)/beam_asm.beam: beam_asm.hrl beam_opcodes.hrl beam_types.hrl $(EBIN)/beam_call_types.beam: beam_types.hrl $(EBIN)/beam_block.beam: beam_asm.hrl -$(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl beam_asm.hrl +$(EBIN)/beam_dict.beam: beam_types.hrl +$(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl \ + beam_asm.hrl beam_types.hrl $(EBIN)/beam_jump.beam: beam_asm.hrl $(EBIN)/beam_kernel_to_ssa.beam: v3_kernel.hrl beam_ssa.hrl -$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl beam_asm.hrl +$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl \ + beam_asm.hrl beam_types.hrl $(EBIN)/beam_ssa.beam: beam_ssa.hrl $(EBIN)/beam_ssa_bsm.beam: beam_ssa.hrl $(EBIN)/beam_ssa_bool.beam: beam_ssa.hrl @@ -213,11 +217,11 @@ $(EBIN)/beam_ssa_codegen.beam: beam_ssa.hrl beam_asm.hrl $(EBIN)/beam_ssa_dead.beam: beam_ssa.hrl $(EBIN)/beam_ssa_lint.beam: beam_ssa.hrl $(EBIN)/beam_ssa_opt.beam: beam_ssa.hrl -$(EBIN)/beam_ssa_pp.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_pp.beam: beam_ssa.hrl beam_types.hrl $(EBIN)/beam_ssa_pre_codegen.beam: beam_ssa.hrl beam_asm.hrl $(EBIN)/beam_ssa_recv.beam: beam_ssa.hrl $(EBIN)/beam_ssa_share.beam: beam_ssa.hrl -$(EBIN)/beam_ssa_throw.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_throw.beam: beam_ssa.hrl beam_types.hrl $(EBIN)/beam_ssa_type.beam: beam_ssa.hrl beam_types.hrl $(EBIN)/beam_trim.beam: beam_asm.hrl $(EBIN)/beam_types.beam: beam_types.hrl diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 97f4089a1b..60dc20582d 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -25,6 +25,8 @@ -export([module/2]). +-include("beam_asm.hrl"). + -spec module(beam_asm:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -85,6 +87,24 @@ rename_instrs([I|Is]) -> [rename_instr(I)|rename_instrs(Is)]; rename_instrs([]) -> []. +rename_instr({bif,Bif,Fail,[A,B],Dst}=I) -> + case Bif of + '=<' -> + {bif,'>=',Fail,[B,A],Dst}; + '<' -> + case [A,B] of + [{integer,N},{tr,_,#t_integer{}}=Src] -> + {bif,'>=',Fail,[Src,{integer,N+1}],Dst}; + [{tr,_,#t_integer{}}=Src,{integer,N}] -> + {bif,'>=',Fail,[{integer,N-1},Src],Dst}; + [_,_] -> + I + end; + '>' -> + rename_instr({bif,'<',Fail,[B,A],Dst}); + _ -> + I + end; rename_instr({bs_put_binary=I,F,Sz,U,Fl,Src}) -> {bs_put,F,{I,U,Fl},[Sz,Src]}; rename_instr({bs_put_float=I,F,Sz,U,Fl,Src}) -> diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 7317b2409d..d959e21ea1 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -26,7 +26,7 @@ -export_type([fail/0,label/0,src/0,module_code/0,function_name/0]). --import(lists, [map/2,member/2,keymember/3,duplicate/2,splitwith/2]). +-import(lists, [append/1,duplicate/2,map/2,member/2,keymember/3,splitwith/2]). -include("beam_opcodes.hrl"). -include("beam_asm.hrl"). @@ -481,6 +481,13 @@ encode_arg({extfunc, M, F, A}, Dict0) -> encode_arg({list, List}, Dict0) -> {L, Dict} = encode_list(List, Dict0, []), {[encode(?tag_z, 1), encode(?tag_u, length(List))|L], Dict}; +encode_arg({commands, List0}, Dict) -> + List1 = [begin + [H|T] = tuple_to_list(Tuple), + [{atom,H}|T] + end || Tuple <- List0], + List = append(List1), + encode_arg({list, List}, Dict); encode_arg({float, Float}, Dict) when is_float(Float) -> encode_literal(Float, Dict); encode_arg({fr,Fr}, Dict) -> @@ -540,13 +547,13 @@ encode_alloc_list_1([], Dict, Acc) -> -spec encode(non_neg_integer(), integer()) -> iolist() | integer(). -encode(Tag, N) when N < 0 -> +encode(Tag, N) when is_integer(N), N < 0 -> encode1(Tag, negative_to_bytes(N)); -encode(Tag, N) when N < 16 -> +encode(Tag, N) when is_integer(N), N < 16 -> (N bsl 4) bor Tag; -encode(Tag, N) when N < 16#800 -> +encode(Tag, N) when is_integer(N), N < 16#800 -> [((N bsr 3) band 2#11100000) bor Tag bor 2#00001000, N band 16#ff]; -encode(Tag, N) -> +encode(Tag, N) when is_integer(N) -> encode1(Tag, to_bytes(N)). encode1(Tag, Bytes) -> diff --git a/lib/compiler/src/beam_bounds.erl b/lib/compiler/src/beam_bounds.erl index 6c9472379e..ce98f6e813 100644 --- a/lib/compiler/src/beam_bounds.erl +++ b/lib/compiler/src/beam_bounds.erl @@ -26,45 +26,261 @@ %% %% -module(beam_bounds). --export(['+'/2, '-'/2, '*'/2, 'div'/2, 'rem'/2, - 'band'/2, 'bor'/2, 'bxor'/2, 'bsr'/2, 'bsl'/2, - relop/3]). - --type range() :: {integer(), integer()} | 'any'. +-export([bounds/2, bounds/3, relop/3, infer_relop_types/3, + is_masking_redundant/2]). +-export_type([range/0]). + +-type range() :: {integer(), integer()} | + {'-inf', integer()} | + {integer(), '+inf'} | + 'any'. -type range_result() :: range() | 'any'. -type relop() :: '<' | '=<' | '>' | '>='. -type bool_result() :: 'true' | 'false' | 'maybe'. +-type op() :: atom(). --spec '+'(range(), range()) -> range_result(). +%% Maximum size of integers in bits to keep ranges for. +-define(NUM_BITS, 128). -'+'({A,B}, {C,D}) when abs(A) bsr 256 =:= 0, abs(B) bsr 256 =:= 0, - abs(C) bsr 256 =:= 0, abs(D) bsr 256 =:= 0 -> - verify_range({A+C,B+D}); -'+'(_, _) -> - any. +-spec bounds(op(), range()) -> range_result(). --spec '-'(range(), range()) -> range_result(). +bounds('bnot', R0) -> + case R0 of + {A,B} -> + R = {inf_add(inf_neg(B), -1), inf_add(inf_neg(A), -1)}, + normalize(R); + _ -> + any + end; +bounds(abs, R) -> + case R of + {A,B} when is_integer(A), is_integer(B) -> + Min = 0, + Max = max(abs(A), abs(B)), + {Min,Max}; + _ -> + {0,'+inf'} + end. -'-'({A,B}, {C,D}) when abs(A) bsr 256 =:= 0, abs(B) bsr 256 =:= 0, - abs(C) bsr 256 =:= 0, abs(D) bsr 256 =:= 0 -> - verify_range({A-D,B-C}); -'-'(_, _) -> - any. +-spec bounds(op(), range(), range()) -> range_result(). + +bounds('+', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({A+C,B+D}); + {{'-inf',B}, {_C,D}} when abs(B) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({'-inf',B+D}); + {{_A,B}, {'-inf',D}} when abs(B) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({'-inf',B+D}); + {{A,'+inf'}, {C,_D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0 -> + normalize({A+C,'+inf'}); + {{A,_B}, {C,'+inf'}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0 -> + normalize({A+C,'+inf'}); + {_, _} -> + any + end; +bounds('-', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({A-D,B-C}); + {{A,'+inf'}, {_C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({A-D,'+inf'}); + {{_A,B}, {C,'+inf'}} when abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0 -> + normalize({'-inf',B-C}); + {{'-inf',B}, {C,_D}} when abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0 -> + normalize({'-inf',B-C}); + {{A,_B}, {'-inf',D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + normalize({A-D,'+inf'}); + {_, _} -> + any + end; +bounds('*', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0 -> + All = [X * Y || X <- [A,B], Y <- [C,D]], + Min = lists:min(All), + Max = lists:max(All), + normalize({Min,Max}); + {{A,'+inf'}, {C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0, + C >= 0 -> + {min(A*C, A*D),'+inf'}; + {{'-inf',B}, {C,D}} when abs(B) bsr ?NUM_BITS =:= 0, + abs(C) bsr ?NUM_BITS =:= 0, + abs(D) bsr ?NUM_BITS =:= 0, + C >= 0 -> + {'-inf',max(B*C, B*D)}; + {{A,B}, {'-inf',_}} when is_integer(A), is_integer(B) -> + bounds('*', R2, R1); + {{A,B}, {_,'+inf'}} when is_integer(A), is_integer(B) -> + bounds('*', R2, R1); + {_, _} -> + any + end; +bounds('div', R1, R2) -> + div_bounds(R1, R2); +bounds('rem', R1, R2) -> + rem_bounds(R1, R2); +bounds('band', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when A bsr ?NUM_BITS =:= 0, A >= 0, + C bsr ?NUM_BITS =:= 0, C >= 0, + is_integer(B), is_integer(D) -> + Min = min_band(A, B, C, D), + Max = max_band(A, B, C, D), + {Min,Max}; + {_, {C,D}} when is_integer(C), C >= 0 -> + {0,D}; + {{A,B}, _} when is_integer(A), A >= 0 -> + {0,B}; + {_, _} -> + any + end; +bounds('bor', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when A bsr ?NUM_BITS =:= 0, A >= 0, + C bsr ?NUM_BITS =:= 0, C >= 0, + is_integer(B), is_integer(D) -> + Min = min_bor(A, B, C, D), + Max = max_bor(A, B, C, D), + {Min,Max}; + {_, _} -> + any + end; +bounds('bxor', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when A bsr ?NUM_BITS =:= 0, A >= 0, + C bsr ?NUM_BITS =:= 0, C >= 0, + is_integer(B), is_integer(D) -> + Max = max_bxor(A, B, C, D), + {0,Max}; + {_, _} -> + any + end; +bounds('bsr', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when is_integer(C), C >= 0 -> + Min = inf_min(inf_bsr(A, C), inf_bsr(A, D)), + Max = inf_max(inf_bsr(B, C), inf_bsr(B, D)), + normalize({Min,Max}); + {_, _} -> + any + end; +bounds('bsl', R1, R2) -> + case {R1,R2} of + {{A,B}, {C,D}} when abs(A) bsr ?NUM_BITS =:= 0, + abs(B) bsr ?NUM_BITS =:= 0 -> + Min = inf_min(inf_bsl(A, C), inf_bsl(A, D)), + Max = inf_max(inf_bsl(B, C), inf_bsl(B, D)), + normalize({Min,Max}); + {_, _} -> + any + end; +bounds(max, R1, R2) -> + case {R1,R2} of + {{A,B},{C,D}} -> + normalize({inf_max(A, C),inf_max(B, D)}); + {_,_} -> + any + end; +bounds(min, R1, R2) -> + case {R1,R2} of + {{A,B},{C,D}} -> + normalize({inf_min(A, C),inf_min(B, D)}); + {_,_} -> + any + end. --spec '*'(range(), range()) -> range_result(). +-spec relop(relop(), range(), range()) -> bool_result(). -'*'({A,B}, {C,D}) when abs(A) bsr 256 =:= 0, abs(B) bsr 256 =:= 0, - abs(C) bsr 256 =:= 0, abs(D) bsr 256 =:= 0 -> - All = [X * Y || X <- [A,B], Y <- [C,D]], - Min = lists:min(All), - Max = lists:max(All), - verify_range({Min,Max}); -'*'(_, _) -> +relop('<', {A,B}, {C,D}) -> + case {inf_lt(B, C),inf_lt(A, D)} of + {Bool,Bool} -> Bool; + {_,_} -> 'maybe' + end; +relop('=<', {A,B}, {C,D}) -> + case {inf_le(B, C),inf_le(A, D)} of + {Bool,Bool} -> Bool; + {_,_} -> 'maybe' + end; +relop('>=', {A,B}, {C,D}) -> + case {inf_ge(B, C),inf_ge(A, D)} of + {Bool,Bool} -> Bool; + {_,_} -> 'maybe' + end; +relop('>', {A,B}, {C,D}) -> + case {inf_gt(B, C),inf_gt(A, D)} of + {Bool,Bool} -> Bool; + {_,_} -> 'maybe' + end; +relop(_, _, _) -> + 'maybe'. + +-spec infer_relop_types(relop(), range(), range()) -> any(). + +infer_relop_types(Op, {_,_}=Range1, {_,_}=Range2) -> + case relop(Op, Range1, Range2) of + 'maybe' -> + infer_relop_types_1(Op, Range1, Range2); + _ -> + any + end; +infer_relop_types('<', {A,_}=R1, any) -> + {R1, normalize({inf_add(A, 1), '+inf'})}; +infer_relop_types('<', any, {_,D}=R2) -> + {normalize({'-inf', inf_add(D, -1)}), R2}; +infer_relop_types('=<', {A,_}=R1, any) -> + {R1, normalize({A, '+inf'})}; +infer_relop_types('=<', any, {_,D}=R2) -> + {normalize({'-inf', D}), R2}; +infer_relop_types('>=', {_,B}=R1, any) -> + {R1, normalize({'-inf', B})}; +infer_relop_types('>=', any, {C,_}=R2) -> + {normalize({C, '+inf'}), R2}; +infer_relop_types('>', {_,B}=R1, any) -> + {R1, normalize({'-inf', inf_add(B, -1)})}; +infer_relop_types('>', any, {C,_}=R2) -> + {normalize({inf_add(C, 1), '+inf'}), R2}; +infer_relop_types(_Op, _R1, _R2) -> any. --spec 'div'(range(), range()) -> range_result(). +-spec is_masking_redundant(range(), integer()) -> boolean(). + +is_masking_redundant(_, -1) -> + true; +is_masking_redundant({A,B}, M) + when M band (M + 1) =:= 0, %Is M + 1 a power of two? + M > 0, + is_integer(A), A >= 0, + B band M =:= B -> + true; +is_masking_redundant(_, _) -> + false. + +%%% +%%% Internal functions. +%%% -'div'({A,B}, {C,D}) -> +div_bounds({A,B}, {C,D}) when is_integer(A), is_integer(B), + is_integer(C), is_integer(D) -> Denominators = [min(C, D),max(C, D)| %% Handle zero crossing for the denominator. if @@ -78,90 +294,34 @@ Y =/= 0], Min = lists:min(All), Max = lists:max(All), - verify_range({Min,Max}); -'div'(_, _) -> + normalize({Min,Max}); +div_bounds({A,'+inf'}, {C,D}) when is_integer(C), C > 0, is_integer(D) -> + Min = min(A div C, A div D), + Max = '+inf', + normalize({Min,Max}); +div_bounds({'-inf',B}, {C,D}) when is_integer(C), C > 0, is_integer(D) -> + Min = '-inf', + Max = max(B div C, B div D), + normalize({Min,Max}); +div_bounds(_, _) -> any. --spec 'rem'(range(), range()) -> range_result(). - -'rem'({A,_}, {C,D}) when C > 0 -> - Max = D - 1, +rem_bounds({A,_}, {C,D}) when is_integer(C), is_integer(D), C > 0 -> + Max = inf_add(D, -1), Min = if + A =:= '-inf' -> -Max; A >= 0 -> 0; true -> -Max end, - verify_range({Min,Max}); -'rem'(_, {C,D}) when C =/= 0; D =/= 0 -> + normalize({Min,Max}); +rem_bounds(_, {C,D}) when is_integer(C), is_integer(D), + C =/= 0 orelse D =/= 0 -> Max = max(abs(C), abs(D)) - 1, Min = -Max, - verify_range({Min,Max}); -'rem'(_, _) -> - any. - --spec 'band'(range(), range()) -> range_result(). - -'band'({A,B}, {C,D}) when A >= 0, A bsr 256 =:= 0, C >= 0, C bsr 256 =:= 0 -> - Min = min_band(A, B, C, D), - Max = max_band(A, B, C, D), - {Min,Max}; -'band'(_, {C,D}) when C >= 0 -> - {0,D}; -'band'({A,B}, _) when A >= 0 -> - {0,B}; -'band'(_, _) -> - any. - --spec 'bor'(range(), range()) -> range_result(). - -'bor'({A,B}, {C,D}) when A >= 0, A bsr 256 =:= 0, C >= 0, C bsr 256 =:= 0 -> - Min = min_bor(A, B, C, D), - Max = max_bor(A, B, C, D), - {Min,Max}; -'bor'(_, _) -> + normalize({Min,Max}); +rem_bounds(_, _) -> any. --spec 'bxor'(range(), range()) -> range_result(). - -'bxor'({A,B}, {C,D}) when A >= 0, A bsr 256 =:= 0, C >= 0, C bsr 256 =:= 0 -> - Max = max_bxor(A, B, C, D), - {0,Max}; -'bxor'(_, _) -> - any. - --spec 'bsr'(range(), range()) -> range_result(). - -'bsr'({A,B}, {C,D}) when C >= 0 -> - Min = min(A bsr C, A bsr D), - Max = max(B bsr C, B bsr D), - {Min,Max}; -'bsr'(_, _) -> - any. - --spec 'bsl'(range(), range()) -> range_result(). - -'bsl'({A,B}, {C,D}) when abs(B) bsr 128 =:= 0, C >= 0, D < 128 -> - Min = min(A bsl C, A bsl D), - Max = max(B bsl C, B bsl D), - {Min,Max}; -'bsl'(_, _) -> - any. - --spec relop(relop(), range(), range()) -> bool_result(). - -relop(Op, {A,B}, {C,D}) -> - case {erlang:Op(B, C),erlang:Op(A, D)} of - {Bool,Bool} -> Bool; - {_,_} -> 'maybe' - end; -relop(_, _, _) -> - 'maybe'. - -%%% -%%% Internal functions. -%%% - -verify_range({Min,Max}=T) when Min =< Max -> T. - min_band(A, B, C, D) -> M = 1 bsl (upper_bit(A bor C) + 1), min_band(A, B, C, D, M). @@ -295,3 +455,92 @@ upper_bit_1(Val0, N) -> 0 -> N; Val -> upper_bit_1(Val, N + 1) end. + +infer_relop_types_1('<', {A,B}, {C,D}) -> + Left = normalize({A, clamp(inf_add(D, -1), A, B)}), + Right = normalize({clamp(inf_add(A, 1), C, D), D}), + {Left,Right}; +infer_relop_types_1('=<', {A,B}, {C,D}) -> + Left = normalize({A, clamp(D, A, B)}), + Right = normalize({clamp(A, C, D), D}), + {Left,Right}; +infer_relop_types_1('>=', {A,B}, {C,D}) -> + Left = normalize({clamp(C, A, B), B}), + Right = normalize({C, clamp(B, C, D)}), + {Left,Right}; +infer_relop_types_1('>', {A,B}, {C,D}) -> + Left = normalize({clamp(inf_add(C, 1), A, B), B}), + Right = normalize({C,clamp(inf_add(B, -1), C, D)}), + {Left,Right}. + +%%% +%%% Handling of ranges. +%%% +%%% A range can begin with '-inf' OR end with '+inf'. +%%% +%%% Atoms are greater than all integers. Therefore, we don't +%%% need any special handling of '+inf'. +%%% + +normalize({'-inf','-inf'}) -> + {'-inf',-1}; +normalize({'-inf','+inf'}) -> + any; +normalize({'+inf','+inf'}) -> + {0,'+inf'}; +normalize({Min,Max}=T) -> + true = inf_ge(Max, Min), + T. + +clamp(V, A, B) -> + inf_min(inf_max(V, A), B). + +inf_min(A, B) when A =:= '-inf'; B =:= '-inf' -> '-inf'; +inf_min(A, B) when A =< B -> A; +inf_min(A, B) when A > B -> B. + +inf_max('-inf', B) -> B; +inf_max(A, '-inf') -> A; +inf_max(A, B) when A >= B -> A; +inf_max(A, B) when A < B -> B. + +inf_neg('-inf') -> '+inf'; +inf_neg('+inf') -> '-inf'; +inf_neg(N) -> -N. + +inf_add(Int, N) when is_integer(Int) -> Int + N; +inf_add(Inf, _N) -> Inf. + +inf_bsr('-inf', _S) -> + '-inf'; +inf_bsr('+inf', _S) -> + '+inf'; +inf_bsr(N, S0) when S0 =:= '-inf'; S0 < 0 -> + S = inf_neg(S0), + if + S >= ?NUM_BITS, N < 0 -> '-inf'; + S >= ?NUM_BITS, N >= 0 -> '+inf'; + true -> N bsl S + end; +inf_bsr(N, '+inf') -> + if + N < 0 -> -1; + N >= 0 -> 0 + end; +inf_bsr(N, S) when S >= 0 -> + N bsr S. + +inf_bsl(N, S) -> + inf_bsr(N, inf_neg(S)). + +inf_lt(_, '-inf') -> false; +inf_lt('-inf', _) -> true; +inf_lt(A, B) -> A < B. + +inf_ge(_, '-inf') -> true; +inf_ge('-inf', _) -> false; +inf_ge(A, B) -> A >= B. + +inf_le(A, B) -> inf_ge(B, A). + +inf_gt(A, B) -> inf_lt(B, A). diff --git a/lib/compiler/src/beam_call_types.erl b/lib/compiler/src/beam_call_types.erl index bd1d672d4a..73f704695e 100644 --- a/lib/compiler/src/beam_call_types.erl +++ b/lib/compiler/src/beam_call_types.erl @@ -27,6 +27,17 @@ -export([will_succeed/3, types/3, arith_type/2]). %% +%% Define an upper limit for functions that return sizes of data +%% structures. The chosen value is about half the maxium size of a +%% smallnum. That means that adding a small constant to it will result +%% in a smallnum, but still the value is still sufficiently high to +%% make it impossible to reach in the foreseeable future. +%% +-define(SIZE_UPPER_LIMIT, ((1 bsl 58) - 1)). + +-define(UNICODE_TYPE, #t_integer{elements={0,16#10FFFF}}). + +%% %% Returns whether a call will succeed or not. %% %% Notes: @@ -42,23 +53,47 @@ -spec will_succeed(Mod, Func, ArgTypes) -> Result when Mod :: atom(), Func :: atom(), - ArgTypes :: [normal_type()], + ArgTypes :: [type()], Result :: 'yes' | 'no' | 'maybe'. -will_succeed(erlang, Op, [LHS, RHS]) - when Op =:= '+'; Op =:= '-'; Op =:= '*' -> +will_succeed(erlang, Op, [LHS, RHS]) when Op =:= '+'; + Op =:= '-'; + Op =:= '*' -> succeeds_if_smallish(LHS, RHS); -will_succeed(erlang, Op, [#t_integer{elements={_,_}}, - #t_integer{elements={Div,_}}]) - when (Op =:= 'div' orelse Op =:= 'rem'), Div > 0 -> - yes; -will_succeed(erlang, 'bsr', [#t_integer{elements={_,_}}, - #t_integer{elements={S,_}}]) - when S >= 0 -> - yes; -will_succeed(erlang, 'bsl', [#t_integer{}=LHS,#t_integer{elements={S,_}}]) - when S < 64 -> - succeeds_if_smallish(LHS); +will_succeed(erlang, Op, [LHS, RHS]) when Op =:= 'div'; + Op =:= 'rem' -> + case {meet(LHS, #t_integer{}), meet(RHS, #t_integer{})} of + {#t_integer{elements={_,_}}=LHS, + #t_integer{elements={Min,Max}}=RHS} + when is_integer(Min), Min > 0; + is_integer(Max), Max < -1 -> + 'yes'; + {#t_integer{}, #t_integer{}} -> + 'maybe'; + {_, _} -> + no + end; +will_succeed(erlang, 'bsr', [LHS, RHS]) -> + case {meet(LHS, #t_integer{}), meet(RHS, #t_integer{})} of + {#t_integer{elements={_,_}}=LHS, + #t_integer{elements={Shift,_}}=RHS} + when is_integer(Shift), Shift >= 0 -> + 'yes'; + {#t_integer{}, #t_integer{}} -> + 'maybe'; + {_, _} -> + no + end; +will_succeed(erlang, 'bsl', [LHS, RHS]) -> + case {meet(LHS, #t_integer{}), meet(RHS, #t_integer{})} of + {LHS, #t_integer{elements={Shift,_}}=RHS} + when is_integer(Shift), Shift < 64 -> + succeeds_if_smallish(LHS); + {#t_integer{}, #t_integer{}} -> + 'maybe'; + {_, _} -> + no + end; will_succeed(erlang, '++', [LHS, _RHS]) -> succeeds_if_type(LHS, proper_list()); will_succeed(erlang, '--', [_, _] = Args) -> @@ -66,25 +101,38 @@ will_succeed(erlang, '--', [_, _] = Args) -> will_succeed(erlang, BoolOp, [_, _] = Args) when BoolOp =:= 'and'; BoolOp =:= 'or' -> succeeds_if_types(Args, beam_types:make_boolean()); -will_succeed(erlang, Op, [_, _] = Args) when Op =:= 'band'; Op =:= 'bor'; Op =:= 'bxor' -> +will_succeed(erlang, Op, [_, _] = Args) when Op =:= 'band'; + Op =:= 'bor'; + Op =:= 'bxor' -> succeeds_if_types(Args, #t_integer{}); will_succeed(erlang, bit_size, [Arg]) -> succeeds_if_type(Arg, #t_bitstring{}); will_succeed(erlang, byte_size, [Arg]) -> succeeds_if_type(Arg, #t_bitstring{}); -will_succeed(erlang, element, [Pos, #t_tuple{size=Sz}] = Args) when Sz > 0 -> - SizeType = #t_integer{elements={1,Sz}}, - case beam_types:meet(Pos, SizeType) of - Pos -> - yes; +will_succeed(erlang, element, [Pos, Tuple]=Args) -> + case normalize(Tuple) of + #t_tuple{exact=Exact,size=Sz} when Sz >= 1 -> + case meet(Pos, #t_integer{elements={1,Sz}}) of + Pos -> yes; + none when Exact -> no; + _ -> 'maybe' + end; _ -> - fails_on_conflict(Args, [#t_integer{}, #t_tuple{}]) + Required = [#t_integer{elements={1,?MAX_TUPLE_SIZE}}, + #t_tuple{size=1}], + fails_on_conflict(Args, Required) end; will_succeed(erlang, hd, [Arg]) -> succeeds_if_type(Arg, #t_cons{}); -will_succeed(erlang, is_function, [_,#t_integer{elements={Min,_}}]) - when Min >= 0 -> - yes; +will_succeed(erlang, is_function, [_, Arity]) -> + case meet(Arity, #t_integer{}) of + #t_integer{elements={Min,_}}=Arity when is_integer(Min), Min >= 0 -> + yes; + #t_integer{} -> + 'maybe'; + _ -> + no + end; will_succeed(erlang, is_map_key, [_Key, Map]) -> succeeds_if_type(Map, #t_map{}); will_succeed(erlang, length, [Arg]) -> @@ -93,27 +141,40 @@ will_succeed(erlang, map_size, [Arg]) -> succeeds_if_type(Arg, #t_map{}); will_succeed(erlang, 'not', [Arg]) -> succeeds_if_type(Arg, beam_types:make_boolean()); -will_succeed(erlang, setelement, [#t_integer{elements={Min,Max}}, - #t_tuple{exact=Exact,size=Size}, _]) -> - if - 1 =< Min, Max =< Size -> - %% The index is always in range. - yes; - Max < 1; Exact, Size < Min -> - %% The index is always out of range. +will_succeed(erlang, setelement, [Pos, Tuple0, _Value]) -> + PosRange = #t_integer{elements={1,?MAX_TUPLE_SIZE}}, + case {meet(Pos, PosRange), meet(Tuple0, #t_tuple{size=1})} of + {none, _} -> no; - true -> + {_, none} -> + no; + {#t_integer{elements={Min,Max}}=Pos, Tuple} -> + MaxTupleSize = max_tuple_size(Tuple), + if + MaxTupleSize < Min -> + %% Index is always out of range. + no; + Tuple0 =:= Tuple, Max =< MaxTupleSize -> + %% We always have a tuple, and the index is always in + %% range. + yes; + true -> + %% We may or may not have a tuple, and the index may or may + %% not be in range if we do. + 'maybe' + end; + {_, _} -> 'maybe' end; will_succeed(erlang, size, [Arg]) -> - ArgType = beam_types:join(#t_tuple{}, #t_bitstring{}), + ArgType = join(#t_tuple{}, #t_bitstring{}), succeeds_if_type(Arg, ArgType); will_succeed(erlang, tuple_size, [Arg]) -> succeeds_if_type(Arg, #t_tuple{}); will_succeed(erlang, tl, [Arg]) -> succeeds_if_type(Arg, #t_cons{}); will_succeed(erlang, raise, [Class, _Reason, nil]) -> - case beam_types:meet(Class, #t_atom{elements=[error,exit,throw]}) of + case meet(Class, #t_atom{elements=[error,exit,throw]}) of Class -> no; none -> yes; _ -> 'maybe' @@ -139,8 +200,17 @@ will_succeed(Mod, Func, Args) -> end end. +max_tuple_size(#t_union{tuple_set=[_|_]=Set}=Union) -> + Union = meet(Union, #t_tuple{}), %Assertion. + Arities = [Arity || {{Arity, _Tag}, _Record} <- Set], + lists:max(Arities); +max_tuple_size(#t_tuple{exact=true,size=Size}) -> + Size; +max_tuple_size(#t_tuple{exact=false}) -> + ?MAX_TUPLE_SIZE. + fails_on_conflict([ArgType | Args], [Required | Types]) -> - case beam_types:meet(ArgType, Required) of + case meet(ArgType, Required) of none -> no; _ -> fails_on_conflict(Args, Types) end; @@ -157,7 +227,7 @@ succeeds_if_types([LHS, RHS], Required) -> end. succeeds_if_type(ArgType, Required) -> - case beam_types:meet(ArgType, Required) of + case meet(ArgType, Required) of ArgType -> yes; none -> no; _ -> 'maybe' @@ -167,7 +237,7 @@ succeeds_if_smallish(#t_integer{elements={Min,Max}}) when abs(Min) bsr 128 =:= 0, abs(Max) bsr 128 =:= 0 -> yes; succeeds_if_smallish(ArgType) -> - case succeeds_if_type(ArgType, number) of + case succeeds_if_type(ArgType, #t_number{}) of yes -> %% Could potentially fail with a `system_limit` exception. 'maybe'; @@ -185,20 +255,6 @@ succeeds_if_smallish(LHS, RHS) -> end. %% -%% Define an upper limit for functions that return sizes of data -%% structures. The chosen value is about half the maxium size of a -%% smallnum. That means that adding a small constant to it will result -%% in a smallnum, but still the value is still sufficiently high to -%% make it impossible to reach in the foreseeable future. -%% --define(SIZE_UPPER_LIMIT, ((1 bsl 58) - 1)). - -%% -%% The document maximum size of a tuple. -%% --define(MAX_TUPLE_SIZE, (1 bsl 24) - 1). - -%% %% Returns the inferred return and argument types for known functions, and %% whether it's safe to subtract argument types on failure. %% @@ -221,7 +277,7 @@ succeeds_if_smallish(LHS, RHS) -> types(erlang, 'map_size', [_]) -> sub_safe(#t_integer{elements={0,?SIZE_UPPER_LIMIT}}, [#t_map{}]); types(erlang, 'tuple_size', [Src]) -> - Min = case Src of + Min = case normalize(meet(Src, #t_tuple{})) of #t_tuple{size=Sz} -> Sz; _ -> 0 end, @@ -259,10 +315,10 @@ types(erlang, 'xor', [_,_]) -> sub_unsafe(Bool, [Bool, Bool]); %% Relational operators. -types(erlang, Op, [#t_integer{elements=Range1}, - #t_integer{elements=Range2}]) +types(erlang, Op, [Arg1, Arg2]) when Op =:= '<'; Op =:= '=<'; Op =:= '>='; Op =:= '>' -> - case beam_bounds:relop(Op, Range1, Range2) of + {R1,R2} = {get_range(Arg1), get_range(Arg2)}, + case beam_bounds:relop(Op, R1, R2) of 'maybe' -> sub_unsafe(beam_types:make_boolean(), [any, any]); Bool when is_boolean(Bool) -> @@ -281,7 +337,7 @@ types(erlang, is_boolean, [Type]) -> true -> sub_unsafe(#t_atom{elements=[true]}, [any]); false -> - case beam_types:meet(Type, #t_atom{}) of + case meet(Type, #t_atom{}) of #t_atom{elements=[_|_]=Es} -> case any(fun is_boolean/1, Es) of true -> @@ -300,7 +356,7 @@ types(erlang, is_float, [Type]) -> types(erlang, is_function, [Type, #t_integer{elements={Arity,Arity}}]) when Arity >= 0, Arity =< ?MAX_FUNC_ARGS -> RetType = - case beam_types:meet(Type, #t_fun{arity=Arity}) of + case meet(Type, #t_fun{arity=Arity}) of Type -> #t_atom{elements=[true]}; none -> #t_atom{elements=[false]}; _ -> beam_types:make_boolean() @@ -315,7 +371,7 @@ types(erlang, is_list, [Type]) -> types(erlang, is_map, [Type]) -> sub_unsafe_type_test(Type, #t_map{}); types(erlang, is_number, [Type]) -> - sub_unsafe_type_test(Type, number); + sub_unsafe_type_test(Type, #t_number{}); types(erlang, is_pid, [Type]) -> sub_unsafe_type_test(Type, pid); types(erlang, is_port, [Type]) -> @@ -327,68 +383,110 @@ types(erlang, is_tuple, [Type]) -> %% Bitwise ops types(erlang, 'band', [_,_]=Args) -> - sub_unsafe(erlang_band_type(Args), [#t_integer{}, #t_integer{}]); + sub_unsafe(beam_bounds_type('band', #t_integer{}, Args), + [#t_integer{}, #t_integer{}]); types(erlang, 'bor', [_,_]=Args) -> - sub_unsafe(erlang_bor_type(Args), [#t_integer{}, #t_integer{}]); + sub_unsafe(beam_bounds_type('bor', #t_integer{}, Args), + [#t_integer{}, #t_integer{}]); types(erlang, 'bxor', [_,_]) -> sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); types(erlang, 'bsl', [_,_]) -> sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); types(erlang, 'bsr', [_,_]=Args) -> - sub_unsafe(erlang_bsr_type(Args), [#t_integer{}, #t_integer{}]); + sub_unsafe(beam_bounds_type('bsr', #t_integer{}, Args), + [#t_integer{}, #t_integer{}]); types(erlang, 'bnot', [_]) -> sub_unsafe(#t_integer{}, [#t_integer{}]); %% Fixed-type arithmetic types(erlang, 'float', [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(erlang, 'round', [_]) -> - sub_unsafe(#t_integer{}, [number]); + sub_unsafe(#t_integer{}, [#t_number{}]); types(erlang, 'floor', [_]) -> - sub_unsafe(#t_integer{}, [number]); + sub_unsafe(#t_integer{}, [#t_number{}]); types(erlang, 'ceil', [_]) -> - sub_unsafe(#t_integer{}, [number]); + sub_unsafe(#t_integer{}, [#t_number{}]); types(erlang, 'trunc', [_]) -> - sub_unsafe(#t_integer{}, [number]); + sub_unsafe(#t_integer{}, [#t_number{}]); types(erlang, '/', [_,_]) -> - sub_unsafe(#t_float{}, [number, number]); + sub_unsafe(#t_float{}, [#t_number{}, #t_number{}]); types(erlang, 'div', [_,_]=Args) -> - ArgTypes = [#t_integer{}, #t_integer{}], - sub_unsafe(erlang_div_type(Args), ArgTypes); + sub_unsafe(beam_bounds_type('div', #t_integer{}, Args), + [#t_integer{}, #t_integer{}]); types(erlang, 'rem', Args) -> - ArgTypes = [#t_integer{}, #t_integer{}], - sub_unsafe(erlang_rem_type(Args), ArgTypes); + sub_unsafe(beam_bounds_type('rem', #t_integer{}, Args), + [#t_integer{}, #t_integer{}]); + +%% Some mixed-type arithmetic. +types(erlang, Op, [LHS, RHS]) when Op =:= '+'; Op =:= '-' -> + case get_range(LHS, RHS, #t_number{}) of + {Type, {A,B}, {C,_D}} when is_integer(C), C >= 0 -> + R = beam_bounds:bounds(Op, {A,B}, {C,'+inf'}), + RetType = case Type of + integer -> #t_integer{elements=R}; + number -> #t_number{elements=R} + end, + sub_unsafe(RetType, [#t_number{}, #t_number{}]); + {Type, {A,_B}, {C,D}} when Op =:= '+', is_integer(A), A >= 0 -> + R = beam_bounds:bounds(Op, {A,'+inf'}, {C,D}), + RetType = case Type of + integer -> #t_integer{elements=R}; + number -> #t_number{elements=R} + end, + sub_unsafe(RetType, [#t_number{}, #t_number{}]); + _ -> + mixed_arith_types([LHS, RHS]) + end; + +types(erlang, abs, [Type]) -> + case meet(Type, #t_number{}) of + #t_float{} -> + sub_unsafe(#t_float{}, [#t_float{}]); + #t_integer{elements=R} -> + RetType = #t_integer{elements=beam_bounds:bounds(abs, R)}, + sub_unsafe(RetType, [#t_integer{}]); + #t_number{elements=R} -> + RetType = #t_number{elements=beam_bounds:bounds(abs, R)}, + sub_unsafe(RetType, [#t_number{}]); + _ -> + sub_unsafe(#t_number{}, [#t_number{}]) + end; -%% Mixed-type arithmetic; '+'/2 and friends are handled in the catch-all +%% The rest of the mixed-type arithmetic is handled in the catch-all %% clause for the 'erlang' module. -types(erlang, 'abs', [_]=Args) -> - mixed_arith_types(Args); %% List operations types(erlang, '++', [LHS, RHS]) -> %% `[] ++ RHS` yields RHS, even if RHS is not a list. ListType = copy_list(LHS, same_length, proper), - RetType = beam_types:join(ListType, RHS), + RetType = join(ListType, RHS), sub_unsafe(RetType, [proper_list(), any]); types(erlang, '--', [LHS, _]) -> RetType = copy_list(LHS, new_length, proper), sub_unsafe(RetType, [proper_list(), proper_list()]); +types(erlang, atom_to_list, [_]) -> + sub_unsafe(proper_list(?UNICODE_TYPE), [#t_atom{}]); types(erlang, 'iolist_to_binary', [_]) -> %% Arg is an iodata(), despite its name. - ArgType = beam_types:join(#t_list{}, #t_bitstring{size_unit=8}), + ArgType = join(#t_list{}, #t_bitstring{size_unit=8}), sub_unsafe(#t_bitstring{size_unit=8}, [ArgType]); types(erlang, 'iolist_size', [_]) -> %% Arg is an iodata(), despite its name. The size is NOT limited %% by the size of memory. - ArgType = beam_types:join(#t_list{}, #t_bitstring{size_unit=8}), - sub_unsafe(#t_integer{}, [ArgType]); + ArgType = join(#t_list{}, #t_bitstring{size_unit=8}), + sub_unsafe(#t_integer{elements={0,'+inf'}}, [ArgType]); types(erlang, 'list_to_binary', [_]) -> %% Arg is an iolist(), despite its name. sub_unsafe(#t_bitstring{size_unit=8}, [#t_list{}]); types(erlang, 'list_to_bitstring', [_]) -> %% As list_to_binary but with bitstrings rather than binaries. sub_unsafe(#t_bitstring{}, [proper_list()]); +types(erlang, list_to_integer, [_]) -> + sub_unsafe(#t_integer{}, [proper_cons()]); +types(erlang, list_to_integer, [_, _]) -> + sub_unsafe(#t_integer{}, [proper_cons(), #t_integer{}]); %% Process operations types(erlang, alias, []) -> @@ -443,80 +541,72 @@ types(erlang, 'node', [_]) -> types(erlang, 'node', []) -> sub_unsafe(#t_atom{}, []); types(erlang, 'size', [_]) -> - ArgType = beam_types:join(#t_tuple{}, #t_bitstring{}), + ArgType = join(#t_tuple{}, #t_bitstring{}), sub_unsafe(#t_integer{}, [ArgType]); %% Tuple element ops -types(erlang, element, [PosType, TupleType]) -> - Index = case PosType of - #t_integer{elements={Same,Same}} when is_integer(Same) -> - Same; - _ -> - 0 - end, - - RetType = case TupleType of - #t_tuple{size=Sz,elements=Es} when Index =< Sz, - Index >= 1 -> - beam_types:get_tuple_element(Index, Es); - _ -> - any - end, - - ArgTypes = [#t_integer{elements={1,?MAX_TUPLE_SIZE}}, - #t_tuple{size=Index}], - - sub_unsafe(RetType, ArgTypes); +types(erlang, element, [Pos, Tuple0]) -> + PosRange = #t_integer{elements={1,?MAX_TUPLE_SIZE}}, + case meet(Pos, PosRange) of + #t_integer{elements={Index,Index}} when Index >= 1 -> + case normalize(meet(Tuple0, #t_tuple{size=Index})) of + #t_tuple{elements=Es}=Tuple -> + RetType = beam_types:get_tuple_element(Index, Es), + sub_unsafe(RetType, [PosRange, Tuple]); + none -> + sub_unsafe(none, [PosRange, #t_tuple{}]) + end; + _ -> + sub_unsafe(any, [PosRange, #t_tuple{}]) + end; types(erlang, setelement, [PosType, TupleType, ArgType]) -> - RetType = case {PosType,TupleType} of - {#t_integer{elements={Index,Index}}, - #t_tuple{elements=Es0,size=Size}=T} when Index >= 1 -> - %% This is an exact index, update the type of said - %% element or return 'none' if it's known to be out of - %% bounds. - Es = beam_types:set_tuple_element(Index, ArgType, Es0), - case T#t_tuple.exact of - false -> - T#t_tuple{size=max(Index, Size),elements=Es}; - true when Index =< Size -> - T#t_tuple{elements=Es}; - true -> - none - end; - {#t_integer{elements={Min,Max}}, - #t_tuple{elements=Es0,size=Size}=T} when Min >= 1 -> - %% We know this will land between Min and Max, so kill - %% the types for those indexes. - Es = discard_tuple_element_info(Min, Max, Es0), - case T#t_tuple.exact of - false -> - T#t_tuple{elements=Es,size=max(Min, Size)}; - true when Min =< Size -> - T#t_tuple{elements=Es,size=Size}; - true -> - none + PosRange = #t_integer{elements={1,?MAX_TUPLE_SIZE}}, + RetType = case meet(PosType, PosRange) of + #t_integer{elements={Same,Same}} -> + beam_types:update_tuple(TupleType, [{Same, ArgType}]); + #t_integer{} -> + case normalize(meet(TupleType, #t_tuple{size=1})) of + #t_tuple{}=T -> T#t_tuple{elements=#{}}; + none -> none end; - {_,#t_tuple{}=T} -> - %% Position unknown, so we have to discard all element - %% information. - T#t_tuple{elements=#{}}; - {#t_integer{elements={Min,_Max}},_} -> - #t_tuple{size=Min}; - {_,_} -> - #t_tuple{} + none -> + none end, - sub_unsafe(RetType, [#t_integer{}, #t_tuple{}, any]); + sub_unsafe(RetType, [PosRange, #t_tuple{size=1}, any]); types(erlang, make_fun, [_,_,Arity0]) -> - Type = case Arity0 of + Type = case meet(Arity0, #t_integer{}) of #t_integer{elements={Arity,Arity}} when Arity >= 0, Arity =< ?MAX_FUNC_ARGS -> #t_fun{arity=Arity}; + #t_integer{} -> + #t_fun{}; _ -> - #t_fun{} + none end, sub_unsafe(Type, [#t_atom{}, #t_atom{}, #t_integer{}]); +types(erlang, Op, [LHS,RHS]) when Op =:= min; Op =:= max -> + R1 = get_range(LHS), + R2 = get_range(RHS), + R = beam_bounds:bounds(Op, R1, R2), + + %% We cannot use mixed_arith_types/1 here as we will return either argument + %% unchanged. The result will not be converted to a float if one of the + %% arguments is a float. + %% + %% 1235.0 = 1 + 1234.0 + %% 1 = erlang:min(1, 1234.0) + RetType = case {LHS, RHS} of + {#t_integer{}, #t_integer{}} -> #t_integer{elements=R}; + {#t_integer{}, #t_number{}} -> #t_number{elements=R}; + {#t_number{}, #t_integer{}} -> #t_number{elements=R}; + {#t_number{}, #t_number{}} -> #t_number{elements=R}; + {_, _} -> join(LHS, RHS) + end, + + sub_unsafe(RetType, [any, any]); + types(erlang, Name, Args) -> Arity = length(Args), @@ -546,53 +636,53 @@ types(erlang, Name, Args) -> %% types(math, cos, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, cosh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, sin, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, sinh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, tan, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, tanh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, acos, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, acosh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, asin, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, asinh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, atan, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, atanh, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, erf, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, erfc, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, exp, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, log, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, log2, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, log10, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, sqrt, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, atan2, [_,_]) -> - sub_unsafe(#t_float{}, [number, number]); + sub_unsafe(#t_float{}, [#t_number{}, #t_number{}]); types(math, pow, [_,_]) -> - sub_unsafe(#t_float{}, [number, number]); + sub_unsafe(#t_float{}, [#t_number{}, #t_number{}]); types(math, ceil, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, floor, [_]) -> - sub_unsafe(#t_float{}, [number]); + sub_unsafe(#t_float{}, [#t_number{}]); types(math, fmod, [_,_]) -> - sub_unsafe(#t_float{}, [number, number]); + sub_unsafe(#t_float{}, [#t_number{}, #t_number{}]); types(math, pi, []) -> sub_unsafe(#t_float{}, []); @@ -620,11 +710,13 @@ types(lists, subtract, [_,_]=Args) -> %% Functions returning booleans. types(lists, all, [_,_]) -> %% This can succeed on improper lists if the fun returns 'false' for an - %% element before reaching the end. - sub_unsafe(beam_types:make_boolean(), [#t_fun{arity=1}, #t_list{}]); + %% element before reaching the end. It can also return 'true' for the + %% empty list, which means we cannot assume the predicate is a fun. + sub_unsafe(beam_types:make_boolean(), [any, #t_list{}]); types(lists, any, [_,_]) -> - %% Doesn't imply that the argument is a proper list; see lists:all/2 - sub_unsafe(beam_types:make_boolean(), [#t_fun{arity=1}, #t_list{}]); + %% Doesn't imply that the argument is a proper list, nor that the fun is + %% valid; see lists:all/2 + sub_unsafe(beam_types:make_boolean(), [any, #t_list{}]); types(lists, keymember, [_,_,_]) -> %% Doesn't imply that the argument is a proper list; see lists:all/2 sub_unsafe(beam_types:make_boolean(), [any, #t_integer{}, #t_list{}]); @@ -643,10 +735,10 @@ types(lists, suffix, [_,_]) -> %% Simple folds types(lists, foldl, [Fun, Init, List]) -> RetType = lists_fold_type(Fun, Init, List), - sub_unsafe(RetType, [#t_fun{arity=2}, any, proper_list()]); + sub_unsafe(RetType, [any, any, proper_list()]); types(lists, foldr, [Fun, Init, List]) -> RetType = lists_fold_type(Fun, Init, List), - sub_unsafe(RetType, [#t_fun{arity=2}, any, proper_list()]); + sub_unsafe(RetType, [any, any, proper_list()]); %% Functions returning plain lists. types(lists, droplast, [List]) -> @@ -656,17 +748,17 @@ types(lists, dropwhile, [_Fun, List]) -> %% If the element is found before the end of the list, we could return an %% improper list. RetType = copy_list(List, new_length, maybe_improper), - sub_unsafe(RetType, [#t_fun{arity=1}, #t_list{}]); + sub_unsafe(RetType, [any, #t_list{}]); types(lists, duplicate, [_Count, Element]) -> sub_unsafe(proper_list(Element), [#t_integer{}, any]); types(lists, filter, [_Fun, List]) -> RetType = copy_list(List, new_length, proper), - sub_unsafe(RetType, [#t_fun{arity=1}, proper_list()]); + sub_unsafe(RetType, [any, proper_list()]); types(lists, flatten, [_]) -> sub_unsafe(proper_list(), [proper_list()]); types(lists, map, [Fun, List]) -> RetType = lists_map_type(Fun, List), - sub_unsafe(RetType, [#t_fun{arity=1}, proper_list()]); + sub_unsafe(RetType, [any, proper_list()]); types(lists, reverse, [List]) -> RetType = copy_list(List, same_length, proper), sub_unsafe(RetType, [proper_list()]); @@ -674,9 +766,10 @@ types(lists, sort, [List]) -> RetType = copy_list(List, same_length, proper), sub_unsafe(RetType, [proper_list()]); types(lists, takewhile, [_Fun, List]) -> - %% Doesn't imply that the argument is a proper list; see lists:all/2 + %% Doesn't imply that the argument is a proper list, nor that the fun is + %% valid; see lists:all/2 RetType = copy_list(List, new_length, proper), - sub_unsafe(RetType, [#t_fun{arity=1}, #t_list{}]); + sub_unsafe(RetType, [any, #t_list{}]); types(lists, usort, [List]) -> %% The result is not quite the same length, but a non-empty list will stay %% non-empty. @@ -686,41 +779,51 @@ types(lists, zip, [_,_]=Lists) -> {RetType, ArgType} = lists_zip_types(Lists), sub_unsafe(RetType, [ArgType, ArgType]); types(lists, zipwith, [Fun | [_,_]=Lists]) -> + %% Doesn't imply that the argument is a fun, as a possible implementation + %% could succeed when both lists are empty. {RetType, ArgType} = lists_zipwith_types(Fun, Lists), - sub_unsafe(RetType, [#t_fun{arity=2}, ArgType, ArgType]); + sub_unsafe(RetType, [any, ArgType, ArgType]); %% Functions with complex return values. types(lists, keyfind, [KeyType,PosType,_]) -> %% Doesn't imply that the argument is a proper list; see lists:all/2 - TupleType = case PosType of + TupleType = case meet(PosType, #t_integer{}) of #t_integer{elements={Index,Index}} when is_integer(Index), Index >= 1 -> Es = beam_types:set_tuple_element(Index, KeyType, #{}), #t_tuple{size=Index,elements=Es}; - _ -> - #t_tuple{} + #t_integer{} -> + #t_tuple{}; + none -> + none end, - RetType = beam_types:join(TupleType, beam_types:make_atom(false)), - sub_unsafe(RetType, [any, #t_integer{}, #t_list{}]); + RetType = join(TupleType, beam_types:make_atom(false)), + sub_unsafe(RetType, [any, any, #t_list{}]); types(lists, MapFold, [Fun, Init, List]) when MapFold =:= mapfoldl; MapFold =:= mapfoldr -> RetType = lists_mapfold_type(Fun, Init, List), - sub_unsafe(RetType, [#t_fun{arity=2}, any, proper_list()]); + %% Doesn't imply that the argument is a proper list, nor that the fun is + %% valid; see lists:all/2 + sub_unsafe(RetType, [any, any, proper_list()]); types(lists, partition, [_Fun, List]) -> + %% Doesn't imply that the argument is a proper list, nor that the fun is + %% valid; see lists:all/2 ListType = copy_list(List, new_length, proper), RetType = make_two_tuple(ListType, ListType), - sub_unsafe(RetType, [#t_fun{arity=1}, proper_list()]); + sub_unsafe(RetType, [any, proper_list()]); types(lists, search, [_,_]) -> - %% Doesn't imply that the argument is a proper list; see lists:all/2 + %% Doesn't imply that the argument is a proper list, nor that the fun is + %% valid; see lists:all/2 TupleType = make_two_tuple(beam_types:make_atom(value), any), - RetType = beam_types:join(TupleType, beam_types:make_atom(false)), - sub_unsafe(RetType, [#t_fun{arity=1}, #t_list{}]); + RetType = join(TupleType, beam_types:make_atom(false)), + sub_unsafe(RetType, [any, #t_list{}]); types(lists, splitwith, [_Fun, List]) -> %% Only the elements in the left list are guaranteed to be visited, so both - %% the argument and the right list may be improper. + %% the argument and the right list may be improper. The fun isn't + %% guaranteed to be valid either if the list is empty. Left = copy_list(List, new_length, proper), Right = copy_list(List, new_length, maybe_improper), - sub_unsafe(make_two_tuple(Left, Right), [#t_fun{arity=1}, #t_list{}]); + sub_unsafe(make_two_tuple(Left, Right), [any, #t_list{}]); types(lists, unzip, [List]) -> RetType = lists_unzip_type(2, List), sub_unsafe(RetType, [proper_list()]); @@ -730,12 +833,14 @@ types(lists, unzip, [List]) -> %% types(maps, filter, [_Fun, Map]) -> - %% Conservatively assume that key/value types are unchanged. - RetType = case Map of + %% Conservatively assume that key/value types are unchanged. Note that we + %% cannot assume that Fun is a function on success, as a potential + %% implementation could short-circuit on the empty map. + RetType = case meet(Map, #t_map{}) of #t_map{}=T -> T; - _ -> #t_map{} + _ -> none end, - sub_unsafe(RetType, [#t_fun{arity=2}, #t_map{}]); + sub_unsafe(RetType, [any, #t_map{}]); types(maps, find, [Key, Map]) -> TupleType = case erlang_map_get_type(Key, Map) of none -> @@ -744,36 +849,39 @@ types(maps, find, [Key, Map]) -> make_two_tuple(beam_types:make_atom(ok), ValueType) end, %% error | {ok, Value} - RetType = beam_types:join(beam_types:make_atom(error), TupleType), + RetType = join(beam_types:make_atom(error), TupleType), sub_unsafe(RetType, [any, #t_map{}]); types(maps, fold, [Fun, Init, _Map]) -> - RetType = case Fun of + RetType = case meet(Fun, #t_fun{arity=3}) of #t_fun{type=Type} -> %% The map is potentially empty, so we have to assume it %% can return the initial value. - beam_types:join(Type, Init); + join(Type, Init); _ -> - any + %% A potential implementation could still succeed with a + %% non-fun for empty maps. + Init end, - sub_unsafe(RetType, [#t_fun{arity=3}, any, #t_map{}]); + sub_unsafe(RetType, [any, any, #t_map{}]); types(maps, from_keys, [Keys, Value]) -> KeyType = erlang_hd_type(Keys), - RetType = case KeyType of - none -> #t_map{super_key=none,super_value=none}; - _ -> #t_map{super_key=KeyType,super_value=Value} - end, + ValueType = case KeyType of + none -> none; + _ -> Value + end, + RetType = #t_map{super_key=KeyType,super_value=ValueType}, sub_unsafe(RetType, [proper_list(), any]); types(maps, from_list, [Pairs]) -> PairType = erlang_hd_type(Pairs), - RetType = case beam_types:normalize(PairType) of + RetType = case normalize(meet(PairType, #t_tuple{exact=true,size=2})) of #t_tuple{elements=Es} -> SKey = beam_types:get_tuple_element(1, Es), SValue = beam_types:get_tuple_element(2, Es), #t_map{super_key=SKey,super_value=SValue}; - none -> + none when PairType =:= none -> #t_map{super_key=none,super_value=none}; - _ -> - #t_map{} + none when PairType =/= none -> + none end, sub_unsafe(RetType, [proper_list()]); types(maps, get, [_Key, _Map]=Args) -> @@ -781,47 +889,51 @@ types(maps, get, [_Key, _Map]=Args) -> types(maps, get, [Key, Map, Default]) -> RetType = case erlang_map_get_type(Key, Map) of none -> Default; - ValueType -> beam_types:join(ValueType, Default) + ValueType -> join(ValueType, Default) end, sub_unsafe(RetType, [any, #t_map{}, any]); types(maps, keys, [Map]) -> - RetType = case Map of + RetType = case meet(Map, #t_map{}) of #t_map{super_key=none} -> nil; #t_map{super_key=SKey} -> proper_list(SKey); - _ -> proper_list() + _ -> none end, sub_unsafe(RetType, [#t_map{}]); -types(maps, map, [Fun, Map]) -> - RetType = case {Fun, Map} of - {#t_fun{type=FunRet}, #t_map{super_value=SValue0}} -> - SValue = beam_types:join(FunRet, SValue0), +types(maps, map, [Fun, Map0]) -> + RetType = case {meet(Fun, #t_fun{arity=2}), meet(Map0, #t_map{})} of + {#t_fun{type=FunRet}, #t_map{super_value=SValue0}=Map} -> + SValue = join(FunRet, SValue0), Map#t_map{super_value=SValue}; - _ -> - #t_map{} + {none, #t_map{}} -> + %% A potential implementation could still work on empty + %% maps even when the fun is broken. + #t_map{super_key=none,super_value=none}; + {_, none} -> + none end, - sub_unsafe(RetType, [#t_fun{arity=2}, #t_map{}]); + sub_unsafe(RetType, [any, #t_map{}]); types(maps, merge, [A, B]) -> - RetType = case {A, B} of + RetType = case {meet(A, #t_map{}), meet(B, #t_map{})} of {#t_map{super_key=SKeyA,super_value=SValueA}, #t_map{super_key=SKeyB,super_value=SValueB}} -> - SKey = beam_types:join(SKeyA, SKeyB), - SValue = beam_types:join(SValueA, SValueB), + SKey = join(SKeyA, SKeyB), + SValue = join(SValueA, SValueB), #t_map{super_key=SKey,super_value=SValue}; _ -> - #t_map{} + none end, sub_unsafe(RetType, [#t_map{}, #t_map{}]); types(maps, new, []) -> RetType = #t_map{super_key=none,super_value=none}, sub_unsafe(RetType, []); types(maps, put, [Key, Value, Map]) -> - RetType = case Map of + RetType = case meet(Map, #t_map{}) of #t_map{super_key=SKey0,super_value=SValue0} -> - SKey = beam_types:join(Key, SKey0), - SValue = beam_types:join(Value, SValue0), + SKey = join(Key, SKey0), + SValue = join(Value, SValue0), #t_map{super_key=SKey,super_value=SValue}; _ -> - #t_map{} + none end, sub_unsafe(RetType, [any, any, #t_map{}]); types(maps, remove, [Key, Map]) -> @@ -832,48 +944,49 @@ types(maps, take, [Key, Map]) -> none -> none; ValueType -> - MapType = beam_types:meet(Map, #t_map{}), + MapType = meet(Map, #t_map{}), make_two_tuple(ValueType, MapType) end, %% error | {Value, Map} - RetType = beam_types:join(beam_types:make_atom(error), TupleType), + RetType = join(beam_types:make_atom(error), TupleType), sub_unsafe(RetType, [any, #t_map{}]); types(maps, to_list, [Map]) -> - RetType = case Map of + RetType = case meet(Map, #t_map{}) of #t_map{super_key=SKey,super_value=SValue} -> proper_list(make_two_tuple(SKey, SValue)); _ -> - proper_list() + none end, sub_unsafe(RetType, [#t_map{}]); -types(maps, update_with, [_Key, Fun, Map]) -> - RetType = case {Fun, Map} of - {#t_fun{type=FunRet}, #t_map{super_value=SValue0}} -> - SValue = beam_types:join(FunRet, SValue0), +types(maps, update_with, [_Key, Fun, Map0]) -> + RetType = case {meet(Fun, #t_fun{arity=1}), meet(Map0, #t_map{})} of + {#t_fun{type=FunRet}, #t_map{super_value=SValue0}=Map} + when FunRet =/= none -> + SValue = join(FunRet, SValue0), Map#t_map{super_value=SValue}; _ -> - #t_map{} + none end, sub_unsafe(RetType, [any, #t_fun{arity=1}, #t_map{}]); types(maps, values, [Map]) -> - RetType = case Map of + RetType = case meet(Map, #t_map{}) of #t_map{super_value=none} -> nil; #t_map{super_value=SValue} -> proper_list(SValue); - _ -> proper_list() + _ -> none end, sub_unsafe(RetType, [#t_map{}]); -types(maps, with, [Keys, Map]) -> - RetType = case {erlang_hd_type(Keys), Map} of +types(maps, with, [Keys, Map0]) -> + RetType = case {erlang_hd_type(Keys), meet(Map0, #t_map{})} of {none, _} -> #t_map{super_key=none,super_value=none}; - {KeysType, #t_map{super_key=SKey0}} -> + {KeysType, #t_map{super_key=SKey0}=Map} -> %% Since we know that the Map will only contain the pairs %% pointed out by Keys, we can restrict the types to %% those in the list. - SKey = beam_types:meet(KeysType, SKey0), + SKey = meet(KeysType, SKey0), Map#t_map{super_key=SKey}; {_, _} -> - #t_map{} + none end, sub_unsafe(RetType, [proper_list(), #t_map{}]); types(maps, without, [Keys, Map]) -> @@ -887,32 +1000,31 @@ types(_, _, Args) -> -spec arith_type(Op, ArgTypes) -> RetType when Op :: beam_ssa:op(), - ArgTypes :: [normal_type()], + ArgTypes :: [type()], RetType :: type(). -arith_type({bif,'+'}, [#t_integer{elements=Range1}, - #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'+'(Range1, Range2)}; -arith_type({bif,'-'}, [#t_integer{elements=Range1}, - #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'-'(Range1, Range2)}; -arith_type({bif,'*'}, [#t_integer{elements=Range1}, - #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'*'(Range1, Range2)}; -arith_type({bif,'div'}, ArgTypes) -> - erlang_div_type(ArgTypes); -arith_type({bif,'rem'}, ArgTypes) -> - erlang_rem_type(ArgTypes); -arith_type({bif,'band'}, ArgTypes) -> - erlang_band_type(ArgTypes); -arith_type({bif,'bor'}, Args) -> - erlang_bor_type(Args); -arith_type({bif,'bxor'}, Args) -> - erlang_bxor_type(Args); -arith_type({bif,'bsr'}, Args) -> - erlang_bsr_type(Args); -arith_type({bif,'bsl'}, Args) -> - erlang_bsl_type(Args); +arith_type({bif,'-'}, [Arg]) -> + ArgTypes = [#t_integer{elements={0,0}},Arg], + beam_bounds_type('-', #t_number{}, ArgTypes); +arith_type({bif,'bnot'}, [Arg0]) -> + case meet(Arg0, #t_integer{}) of + none -> + none; + #t_integer{elements=R} -> + #t_integer{elements=beam_bounds:bounds('bnot', R)} + end; +arith_type({bif,Op}, [_,_]=ArgTypes) when Op =:= '+'; + Op =:= '-'; + Op =:= '*' -> + beam_bounds_type(Op, #t_number{}, ArgTypes); +arith_type({bif,Op}, [_,_]=ArgTypes) when Op =:= 'band'; + Op =:= 'bor'; + Op =:= 'bsl'; + Op =:= 'bsr'; + Op =:= 'bxor'; + Op =:= 'div'; + Op =:= 'rem' -> + beam_bounds_type(Op, #t_integer{}, ArgTypes); arith_type(_Op, _Args) -> any. @@ -920,107 +1032,96 @@ arith_type(_Op, _Args) -> %% Function-specific helpers. %% -mixed_arith_types([FirstType | _]=Args0) -> +mixed_arith_types(Args0) -> + [FirstType|_] = Args = [meet(A, #t_number{}) || A <- Args0], RetType = foldl(fun(#t_integer{}, #t_integer{}) -> #t_integer{}; - (#t_integer{}, number) -> number; + (#t_integer{}, #t_number{}) -> #t_number{}; (#t_integer{}, #t_float{}) -> #t_float{}; (#t_float{}, #t_integer{}) -> #t_float{}; - (#t_float{}, number) -> #t_float{}; + (#t_float{}, #t_number{}) -> #t_float{}; (#t_float{}, #t_float{}) -> #t_float{}; - (number, #t_integer{}) -> number; - (number, #t_float{}) -> #t_float{}; - (number, number) -> number; - (any, _) -> number; + (#t_number{}, #t_integer{}) -> #t_number{}; + (#t_number{}, #t_float{}) -> #t_float{}; + (#t_number{}, #t_number{}) -> #t_number{}; (_, _) -> none - end, FirstType, Args0), - sub_unsafe(RetType, [number || _ <- Args0]). + end, FirstType, Args), + sub_unsafe(RetType, [#t_number{} || _ <- Args]). erlang_hd_type(Src) -> - case beam_types:meet(Src, #t_cons{}) of + case meet(Src, #t_cons{}) of #t_cons{type=Type} -> Type; none -> none end. erlang_tl_type(Src) -> - case beam_types:meet(Src, #t_cons{}) of - #t_cons{terminator=Term}=Cons -> beam_types:join(Cons, Term); + case meet(Src, #t_cons{}) of + #t_cons{terminator=Term}=Cons -> join(Cons, Term); none -> none end. -erlang_band_type([#t_integer{elements=Range1}, #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'band'(Range1, Range2)}; -erlang_band_type([#t_integer{elements=Range}, _RHS]) -> - #t_integer{elements=beam_bounds:'band'(Range, any)}; -erlang_band_type([_LHS, #t_integer{elements=Range}]) -> - #t_integer{elements=beam_bounds:'band'(any, Range)}; -erlang_band_type([_, _]) -> - #t_integer{}. - -erlang_bor_type([#t_integer{elements=Range1}, #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'bor'(Range1, Range2)}; -erlang_bor_type([_, _]) -> - #t_integer{}. - -erlang_bxor_type([#t_integer{elements=Range1}, #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'bxor'(Range1, Range2)}; -erlang_bxor_type([_, _]) -> - #t_integer{}. - -erlang_bsr_type([#t_integer{elements=Range1}, #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'bsr'(Range1, Range2)}; -erlang_bsr_type([_, _]) -> - #t_integer{}. - -erlang_bsl_type([#t_integer{elements=Range1}, #t_integer{elements=Range2}]) -> - #t_integer{elements=beam_bounds:'bsl'(Range1, Range2)}; -erlang_bsl_type([_, _]) -> - #t_integer{}. - -erlang_div_type(ArgTypes) -> - case ArgTypes of - [#t_integer{elements=Range1},#t_integer{elements=Range2}]-> - #t_integer{elements=beam_bounds:'div'(Range1, Range2)}; - _ -> - #t_integer{} +beam_bounds_type(Op, Type, [LHS, RHS]) -> + case get_range(LHS, RHS, Type) of + {_, none, _} -> + none; + {_, _, none} -> + none; + {float, _R1, _R2} -> + #t_float{}; + {integer, R1, R2} -> + #t_integer{elements=beam_bounds:bounds(Op, R1, R2)}; + {number, R1, R2} -> + #t_number{elements=beam_bounds:bounds(Op, R1, R2)} end. -erlang_rem_type([LHS0, #t_integer{elements=Range2}]) -> - Range1 = case LHS0 of - #t_integer{elements=R1} -> R1; - _ -> any - end, - #t_integer{elements=beam_bounds:'rem'(Range1, Range2)}; -erlang_rem_type(_) -> - #t_integer{}. +get_range(LHS, RHS, Type) -> + get_range(meet(LHS, Type), meet(RHS, Type)). + +get_range(#t_float{}=LHS, #t_float{}=RHS) -> + {float, get_range(LHS), get_range(RHS)}; +get_range(#t_integer{}=LHS, #t_integer{}=RHS) -> + {integer, get_range(LHS), get_range(RHS)}; +get_range(LHS, RHS) -> + {number, get_range(LHS), get_range(RHS)}. + +get_range(#t_float{}) -> any; +get_range(#t_integer{elements=R}) -> R; +get_range(#t_number{elements=R}) -> R; +get_range(_) -> none. erlang_map_get_type(Key, Map) -> - case Map of + case meet(Map, #t_map{}) of #t_map{super_key=SKey,super_value=SValue} -> - case beam_types:meet(SKey, Key) of + case meet(SKey, Key) of none -> none; _ -> SValue end; - _ -> - any + none -> + none end. -lists_fold_type(_Fun, Init, nil) -> +lists_fold_type(Fun, Init, List) -> + lists_fold_type_1(meet(Fun, #t_fun{arity=2}), + Init, + meet(List, #t_list{})). + +lists_fold_type_1(_Fun, Init, nil) -> Init; -lists_fold_type(#t_fun{type=Type}, _Init, #t_cons{}) -> +lists_fold_type_1(#t_fun{type=Type}, _Init, #t_cons{}) -> %% The list is non-empty so it's safe to ignore Init. Type; -lists_fold_type(#t_fun{type=Type}, Init, #t_list{}) -> +lists_fold_type_1(#t_fun{type=Type}, Init, #t_list{}) -> %% The list is possibly empty so we have to assume it can return the %% initial value, whose type can differ significantly from the fun's %% return value. - beam_types:join(Type, Init); -lists_fold_type(_Fun, _Init, _List) -> + join(Type, Init); +lists_fold_type_1(_Fun, _Init, _List) -> any. -lists_map_type(#t_fun{type=Type}, Types) -> - lists_map_type_1(Types, Type); -lists_map_type(_Fun, Types) -> - lists_map_type_1(Types, any). +lists_map_type(Fun, Types) -> + case meet(Fun, #t_fun{arity=1}) of + #t_fun{type=Type} -> lists_map_type_1(Types, Type); + none -> none + end. lists_map_type_1(nil, _ElementType) -> nil; @@ -1036,35 +1137,40 @@ lists_map_type_1(_, none) -> lists_map_type_1(_, ElementType) -> proper_list(ElementType). -lists_mapfold_type(#t_fun{type=#t_tuple{size=2,elements=Es}}, Init, List) -> - ElementType = beam_types:get_tuple_element(1, Es), - AccType = beam_types:get_tuple_element(2, Es), - lists_mapfold_type_1(List, ElementType, Init, AccType); -lists_mapfold_type(#t_fun{type=none}, _Init, #t_cons{}) -> - %% The list is non-empty and the fun never returns. - none; -lists_mapfold_type(#t_fun{type=none}, Init, _List) -> - %% The fun never returns, so the only way we could return normally is - %% if the list is empty, in which case we'll return [] and the initial - %% value. - make_two_tuple(nil, Init); -lists_mapfold_type(_Fun, Init, List) -> - lists_mapfold_type_1(List, any, Init, any). - -lists_mapfold_type_1(nil, _ElementType, Init, _AccType) -> - make_two_tuple(nil, Init); +lists_mapfold_type(Fun, Init, List) -> + case {meet(Fun, #t_fun{type=#t_tuple{size=2}}), meet(List, #t_list{})} of + {_, nil} -> + make_two_tuple(nil, Init); + {#t_fun{type=#t_tuple{elements=Es}}, ListType} -> + ElementType = beam_types:get_tuple_element(1, Es), + AccType = beam_types:get_tuple_element(2, Es), + lists_mapfold_type_1(ListType, ElementType, Init, AccType); + {#t_fun{type=none}, #t_list{}} -> + %% The fun never returns, so the only way we could return normally + %% is if the list is empty, in which case we'll return [] and the + %% initial value. + make_two_tuple(nil, Init); + _ -> + none + end. + lists_mapfold_type_1(#t_cons{}, ElementType, _Init, AccType) -> %% The list has at least one element, so it's safe to ignore Init. make_two_tuple(proper_cons(ElementType), AccType); lists_mapfold_type_1(_, ElementType, Init, AccType0) -> %% We can only rely on AccType when we know the list is non-empty, so we %% have to join it with the initial value in case the list is empty. - AccType = beam_types:join(AccType0, Init), + AccType = join(AccType0, Init), make_two_tuple(proper_list(ElementType), AccType). lists_unzip_type(Size, List) -> - Es = lut_make_elements(lut_list_types(Size, List), 1, #{}), - #t_tuple{size=Size,exact=true,elements=Es}. + case meet(List, #t_list{type=#t_tuple{exact=true,size=Size}}) of + none -> + none; + ListType -> + Es = lut_make_elements(lut_list_types(Size, ListType), 1, #{}), + #t_tuple{size=Size,exact=true,elements=Es} + end. lut_make_elements([Type | Types], Index, Es0) -> Es = beam_types:set_tuple_element(Index, Type, Es0), @@ -1072,16 +1178,16 @@ lut_make_elements([Type | Types], Index, Es0) -> lut_make_elements([], _Index, Es) -> Es. -lut_list_types(Size, #t_cons{type=#t_tuple{size=Size,elements=Es}}) -> +lut_list_types(Size, #t_cons{type=Tuple}) -> + #t_tuple{size=Size,elements=Es} = normalize(Tuple), Types = lut_element_types(1, Size, Es), [proper_cons(T) || T <- Types]; -lut_list_types(Size, #t_list{type=#t_tuple{size=Size,elements=Es}}) -> +lut_list_types(Size, #t_list{type=Tuple}) -> + #t_tuple{size=Size,elements=Es} = normalize(Tuple), Types = lut_element_types(1, Size, Es), [proper_list(T) || T <- Types]; lut_list_types(Size, nil) -> - lists:duplicate(Size, nil); -lut_list_types(Size, _) -> - lists:duplicate(Size, proper_list()). + lists:duplicate(Size, nil). lut_element_types(Index, Max, #{}) when Index > Max -> []; @@ -1093,42 +1199,43 @@ lut_element_types(Index, Max, Es) -> %% length, so if one of them is #t_cons{}, we can infer that all of them are %% #t_cons{} on success. -lists_zip_types(Types) -> - lists_zip_types_1(Types, false, #{}, 1). +lists_zip_types(Types0) -> + Types = [meet(T, #t_list{terminator=nil}) || T <- Types0], + lists_zip_types_1(Types, fun proper_list/1, #{}, 1). -lists_zip_types_1([nil | _], _AnyCons, _Es, _N) -> +lists_zip_types_1([none | _], _ListFun, _Es, _N) -> + %% At least one of the lists is not a proper list + {none, nil}; +lists_zip_types_1([nil | _], _ListFun, _Es, _N) -> %% Early exit; we know the result is [] on success. {nil, nil}; -lists_zip_types_1([#t_cons{type=Type,terminator=nil} | Lists], - _AnyCons, Es0, N) -> +lists_zip_types_1([#t_cons{type=Type} | Lists], _ListFun, Es0, N) -> Es = beam_types:set_tuple_element(N, Type, Es0), - lists_zip_types_1(Lists, true, Es, N + 1); -lists_zip_types_1([#t_list{type=Type,terminator=nil} | Lists], - AnyCons, Es0, N) -> + %% The result will be cons. + lists_zip_types_1(Lists, fun proper_cons/1, Es, N + 1); +lists_zip_types_1([#t_list{type=Type} | Lists], ListFun, Es0, N) -> Es = beam_types:set_tuple_element(N, Type, Es0), - lists_zip_types_1(Lists, AnyCons, Es, N + 1); -lists_zip_types_1([_ | Lists], AnyCons, Es, N) -> - lists_zip_types_1(Lists, AnyCons, Es, N + 1); -lists_zip_types_1([], true, Es, N) -> - %% At least one element was cons, so we know it's non-empty on success. - ElementType = #t_tuple{exact=true,size=(N - 1),elements=Es}, - RetType = proper_cons(ElementType), - ArgType = proper_cons(), - {RetType, ArgType}; -lists_zip_types_1([], false, Es, N) -> - ElementType = #t_tuple{exact=true,size=(N - 1),elements=Es}, - RetType = proper_list(ElementType), - ArgType = proper_list(), + lists_zip_types_1(Lists, ListFun, Es, N + 1); +lists_zip_types_1([], ListFun, Es, N) -> + ElementType = #t_tuple{exact=true,size=N-1,elements=Es}, + RetType = ListFun(ElementType), + ArgType = ListFun(any), {RetType, ArgType}. -lists_zipwith_types(#t_fun{type=Type}, Types) -> - lists_zipwith_type_1(Types, Type); -lists_zipwith_types(_Fun, Types) -> - lists_zipwith_type_1(Types, any). +lists_zipwith_types(Fun, Types0) -> + ElementType = case meet(Fun, #t_fun{}) of + #t_fun{type=T} -> T; + none -> none + end, + Types = [meet(T, #t_list{terminator=nil}) || T <- Types0], + lists_zipwith_type_1(Types, ElementType). lists_zipwith_type_1([nil | _], _ElementType) -> %% Early exit; we know the result is [] on success. {nil, nil}; +lists_zipwith_type_1([none | _], _ElementType) -> + %% Early exit; at least one argument cannot be a proper list. + {none, any}; lists_zipwith_type_1([#t_cons{} | _Lists], none) -> %% Early exit; the list is non-empty and we know the fun never %% returns. @@ -1138,7 +1245,7 @@ lists_zipwith_type_1([#t_cons{} | _Lists], ElementType) -> RetType = proper_cons(ElementType), ArgType = proper_cons(), {RetType, ArgType}; -lists_zipwith_type_1([_ | Lists], ElementType) -> +lists_zipwith_type_1([#t_list{} | Lists], ElementType) -> lists_zipwith_type_1(Lists, ElementType); lists_zipwith_type_1([], none) -> %% Since we know the fun won't return, the only way we could return @@ -1149,16 +1256,19 @@ lists_zipwith_type_1([], ElementType) -> ArgType = proper_list(), {RetType, ArgType}. -maps_remove_type(Key, #t_map{super_key=SKey0}=Map) -> - case beam_types:is_singleton_type(Key) of - true -> - SKey = beam_types:subtract(SKey0, Key), - Map#t_map{super_key=SKey}; - false -> - Map - end; -maps_remove_type(_Key, _Map) -> - #t_map{}. +maps_remove_type(Key, Map0) -> + case meet(Map0, #t_map{}) of + #t_map{super_key=SKey0}=Map -> + case beam_types:is_singleton_type(Key) of + true -> + SKey = beam_types:subtract(SKey0, Key), + Map#t_map{super_key=SKey}; + false -> + Map + end; + none -> + none + end. %%% %%% Generic helpers @@ -1166,7 +1276,7 @@ maps_remove_type(_Key, _Map) -> sub_unsafe_type_test(ArgType, Required) -> RetType = - case beam_types:meet(ArgType, Required) of + case meet(ArgType, Required) of ArgType -> #t_atom{elements=[true]}; none -> #t_atom{elements=[false]}; _ -> beam_types:make_boolean() @@ -1179,12 +1289,6 @@ sub_unsafe(RetType, ArgTypes) -> sub_safe(RetType, ArgTypes) -> {RetType, ArgTypes, true}. -discard_tuple_element_info(Min, Max, Es) -> - foldl(fun(El, Acc) when Min =< El, El =< Max -> - maps:remove(El, Acc); - (_El, Acc) -> Acc - end, Es, maps:keys(Es)). - proper_cons() -> #t_cons{terminator=nil}. @@ -1203,27 +1307,27 @@ proper_list(ElementType) -> List :: type(), Length :: same_length | new_length, Proper :: proper | maybe_improper. -copy_list(#t_cons{terminator=Term}=T, Length, maybe_improper) -> - copy_list_1(T, Length, Term); -copy_list(#t_list{terminator=Term}=T, Length, maybe_improper) -> - copy_list_1(T, Length, Term); -copy_list(T, Length, proper) -> - copy_list_1(T, Length, nil); -copy_list(T, Length, _Proper) -> - copy_list_1(T, Length, any). - -copy_list_1(#t_cons{}=T, same_length, Terminator) -> - T#t_cons{terminator=Terminator}; -copy_list_1(#t_cons{type=Type}, new_length, Terminator) -> - #t_list{type=Type,terminator=Terminator}; -copy_list_1(#t_list{}=T, _Length, Terminator) -> - T#t_list{terminator=Terminator}; -copy_list_1(nil, _Length, _Terminator) -> - nil; -copy_list_1(_, _Length, Terminator) -> - #t_list{terminator=Terminator}. +copy_list(List0, Length, Proper) -> + case {meet(List0, #t_list{}), Length, Proper} of + {#t_cons{type=Type,terminator=Term}, new_length, maybe_improper} -> + #t_list{type=Type,terminator=Term}; + {#t_cons{type=Type}, new_length, proper} -> + #t_list{type=Type,terminator=nil}; + {#t_cons{}=T, _, proper} -> + T#t_cons{terminator=nil}; + {#t_list{}=T, _, proper} -> + T#t_list{terminator=nil}; + {none, _, _} -> + none; + {List, _, _} -> + List + end. make_two_tuple(Type1, Type2) -> Es0 = beam_types:set_tuple_element(1, Type1, #{}), Es = beam_types:set_tuple_element(2, Type2, Es0), #t_tuple{size=2,exact=true,elements=Es}. + +normalize(T) -> beam_types:normalize(T). +join(A, B) -> beam_types:join(A, B). +meet(A, B) -> beam_types:meet(A, B). diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 6b41c15970..5f1ec19d59 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -269,12 +269,17 @@ beam_disasm_types(none) -> beam_disasm_types(<<?BEAM_TYPES_VERSION:32,Count:32,Table/binary>>) -> Res = gb_trees:from_orddict(disasm_types(Table, 0)), Count = gb_trees:size(Res), %Assertion. - Res. - -disasm_types(<<Type:18/binary,Rest/binary>>, Index) -> - [{Index,beam_types:decode_ext(Type)}|disasm_types(Rest, Index+1)]; -disasm_types(<<>>, _) -> - []. + Res; +beam_disasm_types(<<_/binary>>) -> + none. + +disasm_types(Types0, Index) -> + case beam_types:decode_ext(Types0) of + done -> + []; + {Types,Rest} -> + [{Index,Types}|disasm_types(Rest, Index+1)] + end. %%----------------------------------------------------------------------- %% Disassembles the code chunk of a BEAM file: @@ -412,6 +417,10 @@ disasm_instr(B, Bs, Atoms, Literals, Types) -> disasm_init_yregs(Bs, Atoms, Literals, Types); bs_create_bin -> disasm_bs_create_bin(Bs, Atoms, Literals, Types); + bs_match -> + disasm_bs_match(Bs, Atoms, Literals, Types); + update_record -> + disasm_update_record(Bs, Atoms, Literals, Types); _ -> try decode_n_args(Arity, Bs, Atoms, Literals, Types) of {Args, RestBs} -> @@ -488,6 +497,27 @@ disasm_bs_create_bin(Bs0, Atoms, Literals, Types) -> {List, RestBs} = decode_n_args(Len, Bs7, Atoms, Literals, Types), {{bs_create_bin, [{A1,A2,A3,A4,A5,Z,U,List}]}, RestBs}. +disasm_bs_match(Bs0, Atoms, Literals, Types) -> + {A1, Bs1} = decode_arg(Bs0, Atoms, Literals, Types), + {A2, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + Bs5 = Bs2, + {Z, Bs6} = decode_arg(Bs5, Atoms, Literals, Types), + {U, Bs7} = decode_arg(Bs6, Atoms, Literals, Types), + {u, Len} = U, + {List, RestBs} = decode_n_args(Len, Bs7, Atoms, Literals, Types), + {{bs_match, [{A1,A2,Z,U,List}]}, RestBs}. + +disasm_update_record(Bs1, Atoms, Literals, Types) -> + {Hint, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {Size, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), + {Src, Bs4} = decode_arg(Bs3, Atoms, Literals, Types), + {Dst, Bs6} = decode_arg(Bs4, Atoms, Literals, Types), + {Z, Bs7} = decode_arg(Bs6, Atoms, Literals, Types), + {U, Bs8} = decode_arg(Bs7, Atoms, Literals, Types), + {u, Len} = U, + {List, RestBs} = decode_n_args(Len, Bs8, Atoms, Literals, Types), + {{update_record, [Hint,Size,Src,Dst,{{Z,U,List}}]}, RestBs}. + %%----------------------------------------------------------------------- %% decode_arg([Byte]) -> {Arg, [Byte]} %% @@ -1241,6 +1271,15 @@ resolve_inst({badrecord,[Arg]},_,_,_) -> {badrecord,resolve_arg(Arg)}; %% +%% OTP 26. +%% + +resolve_inst({update_record,[Hint,Size,Src,Dst,List]},_,_,_) -> + {update_record,Hint,Size,Src,Dst,List}; +resolve_inst({bs_match,[{Fail,Ctx,{z,1},{u,_},Args}]},_,_,_) -> + {bs_match,Fail,Ctx,{list,Args}}; + +%% %% Catches instructions that are not yet handled. %% resolve_inst(X,_,_,_) -> ?exit({resolve_inst,X}). diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 555af00ccc..6d3a04b3b3 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -599,20 +599,23 @@ find_fixpoint(OptFun, Is0) -> opt([{test,is_eq_exact,{f,L},_}|[{jump,{f,L}}|_]=Is], Acc, St) -> %% The is_eq_exact test is not needed. opt(Is, Acc, St); -opt([{test,Test0,{f,L}=Lbl,Ops}=I|[{jump,To}|Is]=Is0], Acc, St) -> +opt([{test,Test0,{f,L}=Lbl,Ops}=I0|[{jump,To}|Is]=Is0], Acc, St) -> case is_label_defined(Is, L) of false -> + I = is_lt_to_is_ge(I0), opt(Is0, [I|Acc], label_used(Lbl, St)); true -> case invert_test(Test0) of not_possible -> + I = is_lt_to_is_ge(I0), opt(Is0, [I|Acc], label_used(Lbl, St)); Test -> %% Invert the test and remove the jump. opt([{test,Test,To,Ops}|Is], Acc, St) end end; -opt([{test,_,{f,_}=Lbl,_}=I|Is], Acc, St) -> +opt([{test,_,{f,_}=Lbl,_}=I0|Is], Acc, St) -> + I = is_lt_to_is_ge(I0), opt(Is, [I|Acc], label_used(Lbl, St)); opt([{test,_,{f,_}=Lbl,_,_,_}=I|Is], Acc, St) -> opt(Is, [I|Acc], label_used(Lbl, St)); @@ -673,6 +676,17 @@ opt([], Acc, #st{replace=Replace0}) when Replace0 =/= #{} -> opt([], Acc, #st{replace=Replace}) when Replace =:= #{} -> reverse(Acc). +is_lt_to_is_ge({test,is_lt,Lbl,Args}=I) -> + case Args of + [{integer,N},{tr,_,#t_integer{}}=Src] -> + {test,is_ge,Lbl,[Src,{integer,N+1}]}; + [{tr,_,#t_integer{}}=Src,{integer,N}] -> + {test,is_ge,Lbl,[{integer,N-1},Src]}; + [_,_] -> + I + end; +is_lt_to_is_ge(I) -> I. + prune_redundant_values([_Val,F|Vls], F) -> prune_redundant_values(Vls, F); prune_redundant_values([Val,Lbl|Vls], F) -> @@ -911,6 +925,8 @@ instr_labels({bs_start_match4,Fail,_,_,_}) -> {f,L} -> [L]; {atom,_} -> [] end; +instr_labels({bs_match,{f,Fail},_Ctx,_List}) -> + [Fail]; instr_labels(_) -> []. diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl index 5037811614..5f743f7efd 100644 --- a/lib/compiler/src/beam_ssa.erl +++ b/lib/compiler/src/beam_ssa.erl @@ -105,7 +105,8 @@ %% To avoid the collapsing, change the value of SET_LIMIT to 50 in the %% file erl_types.erl in the dialyzer application. --type prim_op() :: 'bs_extract' | 'bs_get_tail' | 'bs_init_writable' | +-type prim_op() :: 'bs_create_bin' | + 'bs_extract' | 'bs_ensure' | 'bs_get_tail' | 'bs_init_writable' | 'bs_match' | 'bs_start_match' | 'bs_test_tail' | 'build_stacktrace' | 'call' | 'catch_end' | @@ -115,20 +116,28 @@ 'is_nonempty_list' | 'is_tagged_tuple' | 'kill_try_tag' | 'landingpad' | - 'make_fun' | 'match_fail' | 'new_try_tag' | 'old_make_fun' | + 'make_fun' | 'match_fail' | 'new_try_tag' | + 'nif_start' | + 'old_make_fun' | 'peek_message' | 'phi' | 'put_list' | 'put_map' | 'put_tuple' | - 'raw_raise' | 'recv_next' | 'remove_message' | 'resume' | + 'raw_raise' | + 'recv_marker_bind' | + 'recv_marker_clear' | + 'recv_marker_reserve' | + 'recv_next' | 'remove_message' | 'resume' | + 'update_tuple' | 'update_record' | 'wait_timeout'. -type float_op() :: 'checkerror' | 'clearerror' | 'convert' | 'get' | 'put' | '+' | '-' | '*' | '/'. %% Primops only used internally during code generation. --type cg_prim_op() :: 'bs_get' | 'bs_get_position' | 'bs_match_string' | +-type cg_prim_op() :: 'bs_checked_get' | 'bs_checked_skip' | + 'bs_get' | 'bs_get_position' | 'bs_match_string' | 'bs_restore' | 'bs_save' | 'bs_set_position' | 'bs_skip' | 'copy' | 'match_fail' | 'put_tuple_arity' | - 'put_tuple_element' | 'put_tuple_elements' | - 'set_tuple_element' | 'succeeded'. + 'set_tuple_element' | 'succeeded' | + 'update_record'. -import(lists, [foldl/3,mapfoldl/3,member/2,reverse/1,sort/1]). @@ -221,6 +230,8 @@ no_side_effect(#b_set{op=Op}) -> put_tuple -> true; raw_raise -> true; {succeeded,guard} -> true; + update_record -> true; + update_tuple -> true; _ -> false end. @@ -239,7 +250,7 @@ no_side_effect(#b_set{op=Op}) -> Count :: label(), Result :: {block_map(), label()}. -insert_on_edges(Insertions, Blocks, Count) -> +insert_on_edges(Insertions, Blocks, Count) when is_map(Blocks) -> %% Sort insertions to simplify the handling of duplicates. insert_on_edges_1(sort(Insertions), Blocks, Count). @@ -374,11 +385,9 @@ successors(#b_blk{last=Terminator}) -> normalize(#b_set{op={bif,Bif},args=Args}=Set) -> case {is_commutative(Bif),Args} of - {false,_} -> - Set; - {true,[#b_literal{}=Lit,#b_var{}=Var]} -> + {true, [#b_literal{}=Lit,#b_var{}=Var]} -> Set#b_set{args=[Var,Lit]}; - {true,_} -> + {_, _} -> Set end; normalize(#b_set{}=Set) -> @@ -428,7 +437,7 @@ successors(L, Blocks) -> Blocks :: block_map(), Def :: ordsets:ordset(var_name()). -def(Ls, Blocks) -> +def(Ls, Blocks) when is_map(Blocks) -> Blks = [map_get(L, Blocks) || L <- Ls], def_1(Blks, []). @@ -439,7 +448,7 @@ def(Ls, Blocks) -> Def :: ordsets:ordset(var_name()), Unused :: ordsets:ordset(var_name()). -def_unused(Ls, Unused, Blocks) -> +def_unused(Ls, Unused, Blocks) when is_map(Blocks) -> Blks = [map_get(L, Blocks) || L <- Ls], Preds = sets:from_list(Ls, [{version, 2}]), def_unused_1(Blks, Preds, [], Unused). @@ -461,7 +470,7 @@ def_unused(Ls, Unused, Blocks) -> Labels :: [label()], Blocks :: block_map(), Result :: {dominator_map(), numbering_map()}. -dominators(Labels, Blocks) -> +dominators(Labels, Blocks) when is_map(Blocks) -> Preds = predecessors(Blocks), dominators_from_predecessors(Labels, Preds). @@ -469,7 +478,7 @@ dominators(Labels, Blocks) -> Labels :: [label()], Preds :: predecessor_map(), Result :: {dominator_map(), numbering_map()}. -dominators_from_predecessors(Top0, Preds) -> +dominators_from_predecessors(Top0, Preds) when is_map(Preds) -> Df = maps:from_list(number(Top0, 0)), [{0,[]}|Top] = [{L,map_get(L, Preds)} || L <- Top0], @@ -483,7 +492,7 @@ dominators_from_predecessors(Top0, Preds) -> %% and Dominators and Numbering as returned from dominators/1. -spec common_dominators([label()], dominator_map(), numbering_map()) -> [label()]. -common_dominators(Ls, Dom, Numbering) -> +common_dominators(Ls, Dom, Numbering) when is_map(Dom) -> Doms = [map_get(L, Dom) || L <- Ls], dom_intersection(Doms, Numbering). @@ -493,7 +502,7 @@ common_dominators(Ls, Dom, Numbering) -> Acc0 :: any(), Blocks :: block_map(). -fold_instrs(Fun, Labels, Acc0, Blocks) -> +fold_instrs(Fun, Labels, Acc0, Blocks) when is_map(Blocks) -> fold_instrs_1(Labels, Fun, Blocks, Acc0). %% mapfold_blocks(Fun, [Label], Acc, BlockMap) -> {BlockMap,Acc}. @@ -506,7 +515,7 @@ fold_instrs(Fun, Labels, Acc0, Blocks) -> Acc :: any(), Blocks :: block_map(), Result :: {block_map(), any()}. -mapfold_blocks(Fun, Labels, Acc, Blocks) -> +mapfold_blocks(Fun, Labels, Acc, Blocks) when is_map(Blocks) -> foldl(fun(Lbl, A) -> mapfold_blocks_1(Fun, Lbl, A) end, {Blocks, Acc}, Labels). @@ -525,7 +534,7 @@ mapfold_blocks_1(Fun, Lbl, {Blocks0, Acc0}) -> Blocks0 :: block_map(), Blocks :: block_map(). -mapfold_instrs(Fun, Labels, Acc0, Blocks) -> +mapfold_instrs(Fun, Labels, Acc0, Blocks) when is_map(Blocks) -> mapfold_instrs_1(Labels, Fun, Blocks, Acc0). -spec flatmapfold_instrs(Fun, Labels, Acc0, Blocks0) -> {Blocks,Acc} when @@ -536,7 +545,7 @@ mapfold_instrs(Fun, Labels, Acc0, Blocks) -> Blocks0 :: block_map(), Blocks :: block_map(). -flatmapfold_instrs(Fun, Labels, Acc0, Blocks) -> +flatmapfold_instrs(Fun, Labels, Acc0, Blocks) when is_map(Blocks) -> flatmapfold_instrs_1(Labels, Fun, Blocks, Acc0). -type fold_fun() :: fun((label(), b_blk(), any()) -> any()). @@ -551,7 +560,7 @@ flatmapfold_instrs(Fun, Labels, Acc0, Blocks) -> Acc0 :: any(), Blocks :: #{label():=b_blk()}. -fold_blocks(Fun, Labels, Acc0, Blocks) -> +fold_blocks(Fun, Labels, Acc0, Blocks) when is_map(Blocks) -> fold_blocks_1(Labels, Fun, Blocks, Acc0). %% linearize(Blocks) -> [{BlockLabel,#b_blk{}}]. @@ -565,7 +574,7 @@ fold_blocks(Fun, Labels, Acc0, Blocks) -> Blocks :: block_map(), Linear :: [{label(),b_blk()}]. -linearize(Blocks) -> +linearize(Blocks) when is_map(Blocks) -> Seen = sets:new([{version, 2}]), {Linear0,_} = linearize_1([0], Blocks, Seen, []), Linear = fix_phis(Linear0, #{}), @@ -583,7 +592,7 @@ rpo(Blocks) -> Blocks :: block_map(), Labels :: [label()]. -rpo(From, Blocks) -> +rpo(From, Blocks) when is_map(Blocks) -> Seen = sets:new([{version, 2}]), {Ls,_} = rpo_1(From, Blocks, Seen, []), Ls. @@ -600,7 +609,7 @@ rpo(From, Blocks) -> Blocks :: block_map(), Labels :: [label()]. -between(From, To, Preds, Blocks) -> +between(From, To, Preds, Blocks) when is_map(Preds), is_map(Blocks) -> %% Gather the predecessors of `To` and then walk forward from `From`, %% skipping the blocks that don't precede `To`. %% @@ -618,7 +627,7 @@ between(From, To, Preds, Blocks) -> rename_vars(Rename, Labels, Blocks) when is_list(Rename) -> rename_vars(maps:from_list(Rename), Labels, Blocks); -rename_vars(Rename, Labels, Blocks) when is_map(Rename)-> +rename_vars(Rename, Labels, Blocks) when is_map(Rename), is_map(Blocks) -> Preds = sets:from_list(Labels, [{version, 2}]), F = fun(#b_set{op=phi,args=Args0}=Set) -> Args = rename_phi_vars(Args0, Preds, Rename), @@ -649,7 +658,7 @@ rename_vars(Rename, Labels, Blocks) when is_map(Rename)-> Blocks :: block_map(), Count :: label(). -split_blocks(Ls, P, Blocks, Count) -> +split_blocks(Ls, P, Blocks, Count) when is_map(Blocks) -> split_blocks_1(Ls, P, Blocks, Count). -spec trim_unreachable(SSA0) -> SSA when @@ -698,7 +707,7 @@ definitions(Labels, Blocks) -> -spec uses(Labels, Blocks) -> usage_map() when Labels :: [label()], Blocks :: block_map(). -uses(Labels, Blocks) -> +uses(Labels, Blocks) when is_map(Blocks) -> fold_blocks(fun fold_uses_block/3, Labels, #{}, Blocks). fold_uses_block(Lbl, #b_blk{is=Is,last=Last}, UseMap0) -> diff --git a/lib/compiler/src/beam_ssa_bc_size.erl b/lib/compiler/src/beam_ssa_bc_size.erl index c48a7c8cf6..a93fe8f2f3 100644 --- a/lib/compiler/src/beam_ssa_bc_size.erl +++ b/lib/compiler/src/beam_ssa_bc_size.erl @@ -44,7 +44,7 @@ -spec opt(st_map()) -> st_map(). -opt(StMap) -> +opt(StMap) when is_map(StMap) -> opt(maps:keys(StMap), StMap). opt([Id|Ids], StMap0) -> @@ -199,7 +199,7 @@ setup_call_bs([], [], #{}, NewBs) -> NewBs. calc_size([{L,#b_blk{is=Is,last=Last}}|Blks], Map0) -> case maps:take(L, Map0) of - {Bs0,Map1} -> + {Bs0,Map1} when is_map(Bs0) -> Bs1 = calc_size_is(Is, Bs0), Map2 = update_successors(Last, Bs1, Map1), case get_ret(Last, Bs1) of diff --git a/lib/compiler/src/beam_ssa_bsm.erl b/lib/compiler/src/beam_ssa_bsm.erl index f84ae7dced..71736b3bc0 100644 --- a/lib/compiler/src/beam_ssa_bsm.erl +++ b/lib/compiler/src/beam_ssa_bsm.erl @@ -473,10 +473,27 @@ combine_matches(#b_function{bs=Blocks0,cnt=Counter0}=F, ModInfo) -> %% so we can reuse the RPO computed for Blocks0. Blocks2 = beam_ssa:rename_vars(State#cm.renames, RPO, Blocks1), - {Blocks, Counter} = alias_matched_binaries(Blocks2, Counter0, - State#cm.match_aliases), - - F#b_function{ bs=beam_ssa:trim_unreachable(Blocks), + %% Replacing variables with the atom `true` can cause + %% branches to phi nodes to be omitted, with the phi nodes + %% still referencing the unreachable blocks. Therefore, + %% trim now to update the phi nodes. + Blocks3 = beam_ssa:trim_unreachable(Blocks2), + + Aliases = State#cm.match_aliases, + {Blocks4, Counter} = alias_matched_binaries(Blocks3, Counter0, + Aliases), + Blocks = if + map_size(Aliases) =:= 0 -> + %% No need to trim because there were no aliases. + Blocks4; + true -> + %% Play it safe. It is unclear whether + %% the call to alias_matched_binaries/3 + %% could ever make any blocks + %% unreachable. + beam_ssa:trim_unreachable(Blocks4) + end, + F#b_function{ bs=Blocks, cnt=Counter }; false -> F diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl index 22c8f82e39..94ffe42a45 100644 --- a/lib/compiler/src/beam_ssa_codegen.erl +++ b/lib/compiler/src/beam_ssa_codegen.erl @@ -29,7 +29,7 @@ -include("beam_ssa.hrl"). -include("beam_asm.hrl"). --import(lists, [foldl/3,keymember/3,keysort/2,map/2,mapfoldl/3, +-import(lists, [append/1,foldl/3,keymember/3,keysort/2,map/2,mapfoldl/3, member/2,reverse/1,reverse/2,sort/1, splitwith/2,takewhile/2]). @@ -46,8 +46,9 @@ -spec module(beam_ssa:b_module(), [compile:option()]) -> {'ok',beam_asm:module_code()}. -module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> - {Asm,St} = functions(Fs, {atom,Mod}), +module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, Opts) -> + NoBsMatch = member(no_bs_match, Opts), + {Asm,St} = functions(Fs, NoBsMatch, {atom,Mod}), {ok,{Mod,Es,Attrs,Asm,St#cg.lcount}}. -record(need, {h=0 :: non_neg_integer(), % heap words @@ -77,7 +78,8 @@ module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> -record(cg_ret, {arg :: cg_value(), dealloc=none :: 'none' | pos_integer() }). --record(cg_switch, {arg :: cg_value(), +-record(cg_switch, {anno=#{} :: anno(), + arg :: cg_value(), fail :: ssa_label(), list :: [sw_list_item()] }). @@ -107,11 +109,11 @@ module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> -type ssa_register() :: xreg() | yreg() | freg() | zreg(). -functions(Forms, AtomMod) -> - mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, +functions(Forms, NoBsMatch, AtomMod) -> + mapfoldl(fun (F, St) -> function(F, NoBsMatch, AtomMod, St) end, #cg{lcount=1}, Forms). -function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> +function(#b_function{anno=Anno,bs=Blocks}, NoBsMatch, AtomMod, St0) -> #{func_info:={_,Name,Arity}} = Anno, try assert_exception_block(Blocks), %Assertion. @@ -124,7 +126,7 @@ function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> Labels = (St4#cg.labels)#{0=>Entry,?EXCEPTION_BLOCK=>0}, St5 = St4#cg{labels=Labels,used_labels=gb_sets:singleton(Entry), ultimate_fail=Ult}, - {Body,St} = cg_fun(Blocks, St5#cg{fc_label=Fi}), + {Body,St} = cg_fun(Blocks, NoBsMatch, St5#cg{fc_label=Fi}), Asm = [{label,Fi},line(Anno), {func_info,AtomMod,{atom,Name},Arity}] ++ add_parameter_annos(Body, Anno) ++ @@ -165,16 +167,20 @@ add_parameter_annos([{label, _}=Entry | Body], Anno) -> [Entry | sort(Annos)] ++ Body. -cg_fun(Blocks, St0) -> +cg_fun(Blocks, NoBsMatch, St0) -> Linear0 = linearize(Blocks), - St = collect_catch_labels(Linear0, St0), + St1 = collect_catch_labels(Linear0, St0), Linear1 = need_heap(Linear0), - Linear2 = prefer_xregs(Linear1, St), - Linear3 = liveness(Linear2, St), - Linear4 = defined(Linear3, St), - Linear5 = opt_allocate(Linear4, St), + Linear2 = prefer_xregs(Linear1, St1), + Linear3 = liveness(Linear2, St1), + Linear4 = defined(Linear3, St1), + Linear5 = opt_allocate(Linear4, St1), Linear = fix_wait_timeout(Linear5), - cg_linear(Linear, St). + {Asm,St} = cg_linear(Linear, St1), + case NoBsMatch of + true -> {Asm,St}; + false -> {bs_translate(Asm),St} + end. %% collect_catch_labels(Linear, St) -> St. %% Collect all catch labels (labels for blocks that begin @@ -351,6 +357,8 @@ classify_heap_need({float,Op}, _Args) -> get -> put_float; _ -> neutral end; +classify_heap_need(update_record, [_Flag, #b_literal{val=Size} |_ ]) -> + {put, Size + 1}; classify_heap_need(Name, _Args) -> classify_heap_need(Name). @@ -366,6 +374,9 @@ classify_heap_need(Name, _Args) -> %% Note: Only handle operations in this function that are not handled %% by classify_heap_need/2. +classify_heap_need(bs_ensure) -> gc; +classify_heap_need(bs_checked_get) -> gc; +classify_heap_need(bs_checked_skip) -> gc; classify_heap_need(bs_get) -> gc; classify_heap_need(bs_get_tail) -> gc; classify_heap_need(bs_init_writable) -> gc; @@ -670,6 +681,7 @@ need_live_anno(Op) -> case Op of {bif,_} -> true; bs_create_bin -> true; + bs_checked_get -> true; bs_get -> true; bs_get_position -> true; bs_get_tail -> true; @@ -677,6 +689,7 @@ need_live_anno(Op) -> bs_skip -> true; call -> true; put_map -> true; + update_record -> true; _ -> false end. @@ -803,6 +816,7 @@ need_y_init(#cg_set{op=bs_skip,args=[#b_literal{val=Type}|_]}) -> end; need_y_init(#cg_set{op=bs_start_match}) -> true; need_y_init(#cg_set{op=put_map}) -> true; +need_y_init(#cg_set{op=update_record}) -> true; need_y_init(#cg_set{}) -> false. %% opt_allocate([{BlockLabel,Block}], #st{}) -> [BeamInstruction]. @@ -986,8 +1000,8 @@ cg_block(Is0, Last, Next, St0) -> end. cg_switch(Is0, Last, St0) -> - #cg_switch{arg=Src0,fail=Fail0,list=List0} = Last, - Src = beam_arg(Src0, St0), + #cg_switch{anno=Anno,arg=Src0,fail=Fail0,list=List0} = Last, + Src1 = beam_arg(Src0, St0), {Fail1,St1} = use_block_label(Fail0, St0), Fail = ensure_label(Fail1, St1), {List1,St2} = @@ -997,13 +1011,14 @@ cg_switch(Is0, Last, St0) -> end, St1, List0), {Is1,St} = cg_block(Is0, none, St2), case reverse(Is1) of - [{bif,tuple_size,_,[Tuple],{z,_}=Src}|More] -> + [{bif,tuple_size,_,[Tuple],{z,_}=Src1}|More] -> List = map(fun({integer,Arity}) -> Arity; ({f,_}=F) -> F end, List1), Is = reverse(More, [{select_tuple_arity,Tuple,Fail,{list,List}}]), {Is,St}; _ -> + [Src] = typed_args([Src0], Anno, St), SelectVal = {select_val,Src,Fail,{list,List1}}, {Is1 ++ [SelectVal],St} end. @@ -1112,10 +1127,14 @@ cg_block([#cg_set{op=bs_create_bin,dst=Dst0,args=Args0,anno=Anno}=I, Live = get_live(I), Dst = beam_arg(Dst0, St), Args = bs_args(Args1), + Unit0 = maps:get(unit, Anno, 1), Unit = case Args of - [{atom,append},_Seg,U|_] -> U; - [{atom,private_append},_Seg,U|_] -> U; - _ -> 1 + [{atom,append},_Seg,U|_] -> + max(U, Unit0); + [{atom,private_append},_Seg,U|_] -> + max(U, Unit0); + _ -> + Unit0 end, Is = [Line,{bs_create_bin,Fail,Alloc,Live,Unit,Dst,{list,Args}}], {Is,St}; @@ -1128,6 +1147,13 @@ cg_block([#cg_set{op=bs_start_match, Live = get_live(I), Is = Pre ++ [{test,bs_start_match3,Fail,Live,[Bin],Dst}], {Is,St}; +cg_block([#cg_set{op=bs_ensure,args=Ss0}, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + %% Temporary instruction that will be incorporated into a bs_match + %% instruction by the bs_translate sub pass. + [Ctx,{integer,Size},{integer,Unit}] = beam_args(Ss0, St), + Is = [{test,bs_ensure,Fail,[Ctx,Size,Unit]}], + {Is,St}; cg_block([#cg_set{op=bs_get}=Set, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> {cg_bs_get(Fail, Set, St),St}; @@ -1179,6 +1205,10 @@ cg_block([#cg_set{op={float,convert},dst=Dst0,args=Args0,anno=Anno}, [Src] = typed_args(Args0, Anno, St), Dst = beam_arg(Dst0, St), {[line(Anno),{fconv,Src,Dst}], St}; +cg_block([#cg_set{op=bs_skip,args=Args0,anno=Anno}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + Args = typed_args(Args0, Anno, St), + {cg_bs_skip(bif_fail(Fail), Args, I),St}; cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=I, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> [Dst|Args] = beam_args([Dst0|Args0], St), @@ -1294,6 +1324,13 @@ cg_block([#cg_set{op=wait_timeout,dst=Bool,args=Args0}], {Bool,Fail}, St) -> [{wait_timeout,Fail,Timeout},timeout] end, {Is,St}; +cg_block([#cg_set{op=has_map_field,dst=Dst0,args=Args0,anno=Anno}|T], Context, St0) -> + [Map,Key] = typed_args(Args0, Anno, St0), + Dst = beam_arg(Dst0, St0), + I = {bif,is_map_key,{f,0},[Key,Map],Dst}, + {Is0,St} = cg_block(T, Context, St0), + Is = [I|Is0], + {Is,St}; cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set], none, St) -> [Dst|Args] = beam_args([Dst0|Args0], St), Is = cg_instr(Op, Args, Dst, Set), @@ -1710,6 +1747,17 @@ cg_instr(bs_start_match, [{atom,new}, Src0], Dst, Set) -> {Src, Pre} = force_reg(Src0, Dst), Live = get_live(Set), Pre ++ [{bs_start_match4,{atom,no_fail},Live,Src,Dst}]; +cg_instr(bs_checked_get, [Kind,Ctx,{literal,Flags},{integer,Size},{integer,Unit}], Dst, Set) -> + %% Temporary instruction that will be incorporated into a bs_match + %% instruction by the bs_translate sub pass. + Live = get_live(Set), + [{bs_checked_get,Live,Kind,Ctx,field_flags(Flags, Set),Size,Unit,Dst}]; +cg_instr(bs_checked_get, [{atom,binary},Ctx,{literal,_Flags}, + {atom,all},{integer,Unit}], Dst, Set) -> + %% Temporary instruction that will be incorporated into a bs_match + %% instruction by the bs_translate sub pass. + Live = get_live(Set), + [{bs_checked_get_tail,Live,Ctx,Unit,Dst}]; cg_instr(bs_get_tail, [Src], Dst, Set) -> Live = get_live(Set), [{bs_get_tail,Src,Dst,Live}]; @@ -1730,6 +1778,13 @@ cg_instr(is_nonempty_list, Ss, Dst, Set) -> cg_instr(Op, Args, Dst, _Set) -> cg_instr(Op, Args, Dst). +cg_instr(bs_checked_skip, [_Type,Ctx,_Flags,{integer,Sz},{integer,U}], {z,_}) + when is_integer(Sz) -> + %% Temporary instruction that will be incorporated into a bs_match + %% instruction by the bs_translate sub pass. + [{bs_checked_skip,Ctx,Sz*U}]; +cg_instr(bs_checked_skip, [_Type,_Ctx,_Flags,{atom,all},{integer,_U}], {z,_}) -> + []; cg_instr(bs_init_writable, Args, Dst) -> setup_args(Args) ++ [bs_init_writable|copy({x,0}, Dst)]; cg_instr(bs_set_position, [Ctx,Pos], _Dst) -> @@ -1748,8 +1803,6 @@ cg_instr(get_tl=Op, [Src], Dst) -> [{Op,Src,Dst}]; cg_instr(get_tuple_element=Op, [Src,{integer,N}], Dst) -> [{Op,Src,N,Dst}]; -cg_instr(has_map_field, [Map,Key], Dst) -> - [{bif,is_map_key,{f,0},[Key,Map],Dst}]; cg_instr(nif_start, [], _Dst) -> [nif_start]; cg_instr(put_list=Op, [Hd,Tl], Dst) -> @@ -1769,10 +1822,11 @@ cg_instr(recv_marker_reserve, [], Dst) -> cg_instr(remove_message, [], _Dst) -> [remove_message]; cg_instr(resume, [A,B], _Dst) -> - [{bif,raise,{f,0},[A,B],{x,0}}]. + [{bif,raise,{f,0},[A,B],{x,0}}]; +cg_instr(update_record, [Hint, {integer,Size}, Src | Ss0], Dst) -> + Ss = cg_update_record_list(Ss0, []), + [{update_record,Hint,Size,Src,Dst,{list,Ss}}]. -cg_test(bs_skip, Fail, Args, _Dst, I) -> - cg_bs_skip(Fail, Args, I); cg_test({float,Op0}, Fail, Args, Dst, #cg_set{anno=Anno}) -> Op = case Op0 of '+' -> fadd; @@ -1787,12 +1841,24 @@ cg_test(peek_message, Fail, [], Dst, _I) -> cg_test(put_map, Fail, [{atom,exact},SrcMap|Ss], Dst, #cg_set{anno=Anno}=Set) -> Live = get_live(Set), [line(Anno),{put_map_exact,Fail,SrcMap,Dst,Live,{list,Ss}}]; +cg_test(set_tuple_element=Op, Fail, Args, Dst, Set) -> + {f,0} = Fail, %Assertion. + cg_instr(Op, Args, Dst, Set); cg_test(raw_raise, _Fail, Args, Dst, _I) -> cg_instr(raw_raise, Args, Dst); cg_test(resume, _Fail, [_,_]=Args, Dst, _I) -> cg_instr(resume, Args, Dst). -cg_bs_get(Fail, #cg_set{dst=Dst0,args=[#b_literal{val=Type}|Ss0]}=Set, St) -> +cg_update_record_list([{integer, Index}, Value], []) -> + [Index, Value]; +cg_update_record_list([{integer, Index}, Value | Updates], Acc) -> + cg_update_record_list(Updates, [{Index, Value} | Acc]); +cg_update_record_list([], Acc) -> + append([[Index, Value] || {Index, Value} <- sort(Acc)]). + +cg_bs_get(Fail, #cg_set{dst=Dst0,args=Args,anno=Anno}=Set, St) -> + [{atom,Type}|Ss0] = typed_args(Args, Anno, St), + Dst = beam_arg(Dst0, St), Op = case Type of integer -> bs_get_integer2; float -> bs_get_float2; @@ -1801,8 +1867,7 @@ cg_bs_get(Fail, #cg_set{dst=Dst0,args=[#b_literal{val=Type}|Ss0]}=Set, St) -> utf16 -> bs_get_utf16; utf32 -> bs_get_utf32 end, - [Dst|Ss1] = beam_args([Dst0|Ss0], St), - Ss = case Ss1 of + Ss = case Ss0 of [Ctx,{literal,Flags},Size,{integer,Unit}] -> %% Plain integer/float/binary. [Ctx,Size,Unit,field_flags(Flags, Set)]; @@ -1966,8 +2031,8 @@ translate_terminator(#b_br{bool=#b_literal{val=true},succ=Succ}) -> #cg_br{bool=#b_literal{val=true},succ=Succ,fail=Succ}; translate_terminator(#b_br{bool=Bool,succ=Succ,fail=Fail}) -> #cg_br{bool=Bool,succ=Succ,fail=Fail}; -translate_terminator(#b_switch{arg=Bool,fail=Fail,list=List}) -> - #cg_switch{arg=Bool,fail=Fail,list=List}. +translate_terminator(#b_switch{anno=Anno,arg=Bool,fail=Fail,list=List}) -> + #cg_switch{anno=Anno,arg=Bool,fail=Fail,list=List}. translate_phis(L, #cg_br{succ=Target,fail=Target}, Blocks) -> #b_blk{is=Is} = maps:get(Target, Blocks), @@ -2113,6 +2178,104 @@ break_up_cycle_1(Dst, [{move,S,D}|Path], Acc) -> break_up_cycle_1(Dst, Path, [{swap,S,D}|Acc]). %%% +%%% Collect and translate binary match instructions, producing a +%%% bs_match instruction. +%%% + +bs_translate([{bs_get_tail,_,_,_}=I|Is]) -> + %% A lone bs_get_tail. There is no advantage to incorporating it into + %% a bs_match instruction. + [I|bs_translate(Is)]; +bs_translate([I|Is0]) -> + case bs_translate_instr(I) of + none -> + [I|bs_translate(Is0)]; + {Ctx,Fail0,First} -> + {Instrs0,Fail,Is} = bs_translate_collect(Is0, Ctx, Fail0, [First]), + Instrs = bs_eq_fixup(Instrs0), + [{bs_match,Fail,Ctx,{commands,Instrs}}|bs_translate(Is)] + end; +bs_translate([]) -> []. + +bs_translate_collect([I|Is]=Is0, Ctx, Fail, Acc) -> + case bs_translate_instr(I) of + {Ctx,Fail,Instr} -> + bs_translate_collect(Is, Ctx, Fail, [Instr|Acc]); + {Ctx,{f,0},Instr} -> + bs_translate_collect(Is, Ctx, Fail, [Instr|Acc]); + {_,_,_} -> + {bs_translate_fixup(Acc),Fail,Is0}; + none -> + {bs_translate_fixup(Acc),Fail,Is0} + end. + +bs_translate_fixup([{get_tail,_,_,_}=GT,{test_tail,Bits}|Is0]) -> + Is = reverse(Is0), + bs_translate_fixup_tail(Is, Bits) ++ [GT]; +bs_translate_fixup([{test_tail,Bits}|Is0]) -> + Is = reverse(Is0), + bs_translate_fixup_tail(Is, Bits); +bs_translate_fixup(Is) -> + reverse(Is). + +bs_eq_fixup([{'=:=',nil,Bits,Value}|Is]) -> + EqInstrs = bs_eq_fixup_split(Bits, <<Value:Bits>>), + EqInstrs ++ bs_eq_fixup(Is); +bs_eq_fixup([I|Is]) -> + [I|bs_eq_fixup(Is)]; +bs_eq_fixup([]) -> []. + +%% In the 32-bit runtime system, each integer to be matched must +%% fit in a SIGNED 32-bit word. Therefore, we will split the +%% instruction into multiple instructions each matching at most +%% 31 bits. +bs_eq_fixup_split(Bits, Value) when Bits =< 31 -> + <<I:Bits>> = Value, + [{'=:=',nil,Bits,I}]; +bs_eq_fixup_split(Bits, Value0) -> + <<I:31,Value/bits>> = Value0, + [{'=:=',nil,31,I} | bs_eq_fixup_split(Bits - 31, Value)]. + +bs_translate_fixup_tail([{ensure_at_least,Bits0,_}|Is], Bits) -> + [{ensure_exactly,Bits0+Bits}|Is]; +bs_translate_fixup_tail([I|Is], Bits) -> + [I|bs_translate_fixup_tail(Is, Bits)]; +bs_translate_fixup_tail([], Bits) -> + [{ensure_exactly,Bits}]. + +bs_translate_instr({test,bs_ensure,Fail,[Ctx,Size,Unit]}) -> + {Ctx,Fail,{ensure_at_least,Size,Unit}}; +bs_translate_instr({bs_checked_get,Live,{atom,Type},Ctx,{field_flags,Flags0}, + Size,Unit,Dst}) -> + %% Only keep flags that have a meaning for binary matching and are + %% distinct from the default value. + Flags = [Flag || Flag <- Flags0, + case Flag of + little -> true; + native -> true; + big -> false; + signed -> true; + unsigned -> false; + {anno,_} -> false + end], + {Ctx,{f,0},{Type,Live,{literal,Flags},Size,Unit,Dst}}; +bs_translate_instr({bs_checked_skip,Ctx,Stride}) -> + {Ctx,{f,0},{skip,Stride}}; +bs_translate_instr({bs_checked_get_tail,Live,Ctx,Unit,Dst}) -> + {Ctx,{f,0},{get_tail,Live,Unit,Dst}}; +bs_translate_instr({bs_get_tail,Ctx,Dst,Live}) -> + {Ctx,{f,0},{get_tail,Live,1,Dst}}; +bs_translate_instr({test,bs_test_tail2,Fail,[Ctx,Bits]}) -> + {Ctx,Fail,{test_tail,Bits}}; +bs_translate_instr({test,bs_match_string,Fail,[Ctx,Bits,{string,String}]}) + when bit_size(String) =< 64 -> + <<Value:Bits,_/bitstring>> = String, + Live = nil, + {Ctx,Fail,{'=:=',Live,Bits,Value}}; +bs_translate_instr(_) -> none. + + +%%% %%% General utility functions. %%% diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl index e10189afd8..11e065e6a6 100644 --- a/lib/compiler/src/beam_ssa_opt.erl +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -93,7 +93,7 @@ skip_removed(FuncIds, StMap) -> fixpoint(_FuncIds, _Order, _Passes, StMap, FuncDb, 0) -> %% Too many repetitions. Give up and return what we have. {StMap, FuncDb}; -fixpoint(FuncIds0, Order0, Passes, StMap0, FuncDb0, N) -> +fixpoint(FuncIds0, Order0, Passes, StMap0, FuncDb0, N) when is_map(StMap0) -> {StMap, FuncDb} = phase(FuncIds0, Passes, StMap0, FuncDb0), Repeat = changed(FuncIds0, FuncDb0, FuncDb, StMap0, StMap), case sets:is_empty(Repeat) of @@ -243,6 +243,7 @@ prologue_passes(Opts) -> ?PASS(ssa_opt_linearize), ?PASS(ssa_opt_tuple_size), ?PASS(ssa_opt_record), + ?PASS(ssa_opt_update_tuple), ?PASS(ssa_opt_cse), % Helps the first type pass. ?PASS(ssa_opt_live)], % ... passes_1(Ps, Opts). @@ -288,6 +289,7 @@ epilogue_passes(Opts) -> ?PASS(ssa_opt_sink), ?PASS(ssa_opt_blockify), ?PASS(ssa_opt_redundant_br), + ?PASS(ssa_opt_bs_ensure), ?PASS(ssa_opt_merge_blocks), ?PASS(ssa_opt_get_tuple_element), ?PASS(ssa_opt_tail_literals), @@ -299,14 +301,16 @@ epilogue_passes(Opts) -> passes_1(Ps, Opts0) -> Negations = [{list_to_atom("no_"++atom_to_list(N)),N} || {N,_} <- Ps], - Opts = proplists:substitute_negations(Negations, Opts0), + Expansions = [{no_bs_match,[no_ssa_opt_bs_ensure,no_bs_match]}], + Opts = proplists:normalize(Opts0, [{expand,Expansions}, + {negations,Negations}]), [case proplists:get_value(Name, Opts, true) of true -> P; false -> {NoName,Name} = keyfind(Name, 2, Negations), {NoName,fun(S) -> S end} - end || {Name,_}=P <- Ps]. + end || {Name,_}=P <- Ps]. %% Builds a function information map with basic information about incoming and %% outgoing local calls, as well as whether the function is exported. @@ -393,7 +397,7 @@ fdb_update(Caller, Callee, FuncDb) -> %% Functions where module-level optimization is disabled are added last in %% arbitrary order. -get_call_order_po(StMap, FuncDb) -> +get_call_order_po(StMap, FuncDb) when is_map(FuncDb) -> Order = gco_po(FuncDb), Order ++ sort([K || K <- maps:keys(StMap), not is_map_key(K, FuncDb)]). @@ -491,7 +495,7 @@ ssa_opt_split_blocks({#opt_st{ssa=Blocks0,cnt=Count0}=St, FuncDb}) -> %%% different registers). %%% -ssa_opt_coalesce_phis({#opt_st{ssa=Blocks0}=St, FuncDb}) -> +ssa_opt_coalesce_phis({#opt_st{ssa=Blocks0}=St, FuncDb}) when is_map(Blocks0) -> Ls = beam_ssa:rpo(Blocks0), Blocks = c_phis_1(Ls, Blocks0), {St#opt_st{ssa=Blocks}, FuncDb}. @@ -876,6 +880,73 @@ is_tagged_tuple_4([_|Is], Bool, TagVar) -> is_tagged_tuple_4([], _, _) -> no. %%% +%%% Replaces setelement/3 with the update_tuple psuedo-instruction, and merges +%%% multiple such calls into the same instruction. +%%% +ssa_opt_update_tuple({#opt_st{ssa=Linear0}=St, FuncDb}) -> + {St#opt_st{ssa=update_tuple_opt(Linear0, #{})}, FuncDb}. + +update_tuple_opt([{L, #b_blk{is=Is0}=B} | Bs], SetOps0) -> + {Is, SetOps} = update_tuple_opt_is(Is0, SetOps0, []), + [{L, B#b_blk{is=Is}} | update_tuple_opt(Bs, SetOps)]; +update_tuple_opt([], _SetOps) -> + []. + +update_tuple_opt_is([#b_set{op=call, + dst=Dst, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=setelement}}, + #b_literal{val=N}=Index, + Src, + Value]}=I0 | Is], + SetOps0, Acc) when is_integer(N), N >= 1 -> + SetOps1 = SetOps0#{ Dst => {Src, Index, Value} }, + SetOps = maps:remove(Value, SetOps1), + + Args = update_tuple_merge(Src, SetOps, [Index, Value], + sets:new([{version,2}])), + I = I0#b_set{op=update_tuple,dst=Dst,args=Args}, + + update_tuple_opt_is(Is, SetOps, [I | Acc]); +update_tuple_opt_is([#b_set{op=Op}=I | Is], SetOps0, Acc) -> + case {Op, beam_ssa:clobbers_xregs(I)} of + {_, true} -> + %% Merging setelement across stack frames is potentially very + %% expensive as the update list needs to be saved on the stack, so + %% we discard our state whenever we need one. + update_tuple_opt_is(Is, #{}, [I | Acc]); + {{succeeded, _}, false} -> + %% This is a psuedo-op used to link ourselves with our catch block, + %% so it doesn't really count as a use. + update_tuple_opt_is(Is, SetOps0, [I | Acc]); + {_, false} -> + %% It's pointless to merge us with later ops if our result is used + %% and needs to be created anyway. + SetOps = maps:without(beam_ssa:used(I), SetOps0), + update_tuple_opt_is(Is, SetOps, [I | Acc]) + end; +update_tuple_opt_is([], SetOps, Acc) -> + {reverse(Acc), SetOps}. + +update_tuple_merge(Src, SetOps, Updates0, Seen0) -> + %% Note that we're merging in reverse order, so Updates0 contains the + %% updates made *after* this one. + case SetOps of + #{ Src := {Ancestor, Index, Value} } -> + %% Drop redundant updates, which can happen when when a record is + %% updated in several branches and one of them overwrites a + %% previous index. + Updates = case sets:is_element(Index, Seen0) of + false -> [Index, Value | Updates0]; + true -> Updates0 + end, + Seen = sets:add_element(Index, Seen0), + update_tuple_merge(Ancestor, SetOps, Updates, Seen); + #{} -> + [Src | Updates0] + end. + +%%% %%% Common subexpression elimination (CSE). %%% %%% Eliminate repeated evaluation of identical expressions. To avoid @@ -957,6 +1028,30 @@ cse_is([#b_set{op={succeeded,_},dst=Bool,args=[Src]}=I0|Is], Es, Sub0, Acc) -> Sub = Sub0#{Bool=>#b_literal{val=true}}, cse_is(Is, Es, Sub, Acc) end; +cse_is([#b_set{op=put_map,dst=Dst,args=[_Kind,Map|_]}=I0|Is], + Es0, Sub0, Acc) -> + I1 = sub(I0, Sub0), + {ok,ExprKey} = cse_expr(I1), + case Es0 of + #{ExprKey:=PrevPutMap} -> + Sub = Sub0#{Dst=>PrevPutMap}, + cse_is(Is, Es0, Sub, Acc); + #{Map:=PutMap} -> + case combine_put_maps(PutMap, I1) of + none -> + Es1 = Es0#{ExprKey=>Dst}, + Es = cse_add_inferred_exprs(I1, Es1), + cse_is(Is, Es, Sub0, [I1|Acc]); + I -> + Es1 = Es0#{ExprKey=>Dst}, + Es = cse_add_inferred_exprs(I1, Es1), + cse_is(Is, Es, Sub0, [I|Acc]) + end; + #{} -> + Es1 = Es0#{ExprKey=>Dst}, + Es = cse_add_inferred_exprs(I1, Es1), + cse_is(Is, Es, Sub0, [I1|Acc]) + end; cse_is([#b_set{dst=Dst}=I0|Is], Es0, Sub0, Acc) -> I = sub(I0, Sub0), case beam_ssa:clobbers_xregs(I) of @@ -1005,8 +1100,9 @@ cse_add_inferred_exprs(#b_set{op={bif,tl},dst=Tl,args=[List]}, Es) -> Es#{{get_tl,[List]} => Tl}; cse_add_inferred_exprs(#b_set{op={bif,map_get},dst=Value,args=[Key,Map]}, Es) -> Es#{{get_map_element,[Map,Key]} => Value}; -cse_add_inferred_exprs(#b_set{op=put_map,dst=Map,args=[_,_|Args]}, Es) -> - cse_add_map_get(Args, Map, Es); +cse_add_inferred_exprs(#b_set{op=put_map,dst=Map,args=[_,_|Args]}=I, Es0) -> + Es = cse_add_map_get(Args, Map, Es0), + Es#{Map => I}; cse_add_inferred_exprs(_, Es) -> Es. cse_add_map_get([Key,Value|T], Map, Es0) -> @@ -1045,6 +1141,42 @@ cse_suitable(#b_set{anno=Anno,op={bif,Name},args=Args}) -> erl_internal:bool_op(Name, Arity)); cse_suitable(#b_set{}) -> false. +combine_put_maps(#b_set{dst=Prev,args=[#b_literal{val=assoc},Map|Args1]}, + #b_set{args=[#b_literal{val=assoc},Prev|Args2]}=I) -> + case are_map_keys_literals(Args1) andalso are_map_keys_literals(Args2) of + true -> + Args = combine_put_map_args(Args1, Args2), + I#b_set{args=[#b_literal{val=assoc},Map|Args]}; + false -> + none + end; +combine_put_maps(#b_set{}, #b_set{}) -> + none. + +combine_put_map_args(Args1, Args2) -> + Keys = sets:from_list(get_map_keys(Args2), [{version,2}]), + combine_put_map_args_1(Args1, Args2, Keys). + +combine_put_map_args_1([Key,Value|T], Tail, Keys) -> + case sets:is_element(Key, Keys) of + true -> + combine_put_map_args_1(T, Tail, Keys); + false -> + [Key,Value|combine_put_map_args_1(T, Tail, Keys)] + end; +combine_put_map_args_1([], Tail, _Keys) -> Tail. + +get_map_keys([Key,_|T]) -> + [Key|get_map_keys(T)]; +get_map_keys([]) -> []. + +are_map_keys_literals([#b_literal{},_Value|Args]) -> + are_map_keys_literals(Args); +are_map_keys_literals([#b_var{}|_]) -> + false; +are_map_keys_literals([]) -> + true. + %%% %%% Using floating point instructions. %%% @@ -1715,7 +1847,8 @@ bsm_skip([], _) -> []. bsm_skip_is([I0|Is], Extracted) -> case I0 of - #b_set{op=bs_match, + #b_set{anno=Anno0, + op=bs_match, dst=Ctx, args=[#b_literal{val=T}=Type,PrevCtx|Args0]} when T =/= float, T =/= string, T =/= skip -> @@ -1727,7 +1860,8 @@ bsm_skip_is([I0|Is], Extracted) -> false -> %% The value is never extracted. Args = [#b_literal{val=skip},PrevCtx,Type|Args0], - I0#b_set{args=Args} + Anno = maps:remove(arg_types, Anno0), + I0#b_set{anno=Anno,args=Args} end, [I|Is]; #b_set{} -> @@ -2322,7 +2456,7 @@ ssa_opt_sink({#opt_st{ssa=Linear}=St, FuncDb}) -> {do_ssa_opt_sink(Defs, St), FuncDb} end. -do_ssa_opt_sink(Defs, #opt_st{ssa=Linear}=St) -> +do_ssa_opt_sink(Defs, #opt_st{ssa=Linear}=St) when is_map(Defs) -> %% Find all the blocks that use variables defined by %% get_tuple_element instructions. Used = used_blocks(Linear, Defs, []), @@ -2552,7 +2686,7 @@ gc_update_successors(Blk, GC, WillGC) -> %% Return an gbset of block labels for the blocks that are not %% suitable for sinking of get_tuple_element instructions. -unsuitable(Linear, Blocks, Predecessors) -> +unsuitable(Linear, Blocks, Predecessors) when is_map(Blocks), is_map(Predecessors) -> Unsuitable0 = unsuitable_1(Linear), Unsuitable1 = unsuitable_recv(Linear, Blocks, Predecessors), gb_sets:from_list(Unsuitable0 ++ Unsuitable1). @@ -2560,6 +2694,7 @@ unsuitable(Linear, Blocks, Predecessors) -> unsuitable_1([{L,#b_blk{is=[#b_set{op=Op}=I|_]}}|Bs]) -> Unsuitable = case Op of bs_extract -> true; + bs_match -> true; {float,_} -> true; landingpad -> true; _ -> beam_ssa:is_loop_header(I) @@ -2791,6 +2926,7 @@ collect_get_tuple_element(Is, _Src, Acc) -> ssa_opt_unfold_literals({St,FuncDb}) -> #opt_st{ssa=Blocks0,args=Args,anno=Anno} = St, + true = is_map(Blocks0), %Assertion. ParamInfo = maps:get(parameter_info, Anno, #{}), LitMap = collect_arg_literals(Args, ParamInfo, 0, #{}), case map_size(LitMap) of @@ -2955,6 +3091,7 @@ unfold_arg(Expr, _LitMap, _X) -> ssa_opt_tail_literals({St,FuncDb}) -> #opt_st{cnt=Count0,ssa=Blocks0} = St, + true = is_map(Blocks0), %Assertion. {Count, Blocks} = opt_tail_literals(beam_ssa:rpo(Blocks0), Count0, Blocks0), {St#opt_st{cnt=Count,ssa=Blocks},FuncDb}. @@ -3039,7 +3176,7 @@ is_tail_literal(_Is, _Last, _Blocks) -> %%% ret Var %%% -ssa_opt_redundant_br({#opt_st{ssa=Blocks0}=St, FuncDb}) -> +ssa_opt_redundant_br({#opt_st{ssa=Blocks0}=St, FuncDb}) when is_map(Blocks0) -> Blocks = redundant_br(beam_ssa:rpo(Blocks0), Blocks0), {St#opt_st{ssa=Blocks}, FuncDb}. @@ -3123,6 +3260,156 @@ redundant_br_safe_bool(Is, Bool) -> end. %%% +%%% Add the bs_ensure instruction before a sequence of `bs_match` +%%% (SSA) instructions, each having a literal size and the +%%% same failure label. +%%% +%%% This is the first part of building the `bs_match` (BEAM) +%%% instruction that can match multiple segments having the same +%%% failure label. +%%% + +ssa_opt_bs_ensure({#opt_st{ssa=Blocks0,cnt=Count0}=St, FuncDb}) when is_map(Blocks0) -> + RPO = beam_ssa:rpo(Blocks0), + Seen = sets:new([{version,2}]), + {Blocks,Count} = ssa_opt_bs_ensure(RPO, Seen, Count0, Blocks0), + {St#opt_st{ssa=Blocks,cnt=Count}, FuncDb}. + +ssa_opt_bs_ensure([L|Ls], Seen0, Count0, Blocks0) -> + case sets:is_element(L, Seen0) of + true -> + %% This block is already covered by a `bs_ensure` + %% instruction. + ssa_opt_bs_ensure(Ls, Seen0, Count0, Blocks0); + false -> + case is_bs_match_blk(L, Blocks0) of + no -> + ssa_opt_bs_ensure(Ls, Seen0, Count0, Blocks0); + {yes,Size0,#b_br{succ=Succ,fail=Fail}} -> + {Size,Blocks1,Seen} = + ssa_opt_bs_ensure_collect(Succ, Fail, + Blocks0, Seen0, Size0), + Blocks2 = annotate_match(L, Blocks1), + {Blocks,Count} = build_bs_ensure_match(L, Size, Count0, Blocks2), + ssa_opt_bs_ensure(Ls, Seen, Count, Blocks) + end + end; +ssa_opt_bs_ensure([], _Seen, Count, Blocks) -> + {Blocks,Count}. + +ssa_opt_bs_ensure_collect(L, Fail, Blocks0, Seen0, Acc0) -> + case is_bs_match_blk(L, Blocks0) of + no -> + {Acc0,Blocks0,Seen0}; + {yes,Size,#b_br{succ=Succ,fail=Fail}} -> + case update_size(Size, Acc0) of + no -> + {Acc0,Blocks0,Seen0}; + Acc -> + Seen = sets:add_element(L, Seen0), + Blocks = annotate_match(L, Blocks0), + ssa_opt_bs_ensure_collect(Succ, Fail, Blocks, Seen, Acc) + end; + {yes,_,_} -> + {Acc0,Blocks0,Seen0} + end. + +annotate_match(L, Blocks) -> + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), + Is = [case I of + #b_set{op=bs_match} -> + beam_ssa:add_anno(ensured, true, I); + #b_set{} -> + I + end || I <- Is0], + Blk = Blk0#b_blk{is=Is}, + Blocks#{L := Blk}. + +update_size({{PrevCtx,NewCtx},Size,Unit}, {{_,PrevCtx},Sum,Unit0}) -> + {{PrevCtx,NewCtx},Sum + Size,max(Unit, Unit0)}; +update_size(_, _) -> + no. + +is_bs_match_blk(L, Blocks) -> + Blk = map_get(L, Blocks), + case Blk of + #b_blk{is=Is,last=#b_br{bool=#b_var{}}=Last} -> + case is_bs_match_is(Is) of + no -> + no; + {yes,CtxSizeUnit} -> + {yes,CtxSizeUnit,Last} + end; + #b_blk{} -> + no + end. + +is_bs_match_is([#b_set{op=bs_match,dst=Dst}=I, + #b_set{op={succeeded,guard},args=[Dst]}]) -> + case is_viable_match(I) of + no -> + no; + {yes,{Ctx,Size,Unit}} when Size bsr 24 =:= 0 -> + %% Only include matches of reasonable size. + {yes,{{Ctx,Dst},Size,Unit}}; + {yes,_} -> + %% Too large size. + no + end; +is_bs_match_is([_|Is]) -> + is_bs_match_is(Is); +is_bs_match_is([]) -> no. + +is_viable_match(#b_set{op=bs_match,args=Args}) -> + case Args of + [#b_literal{val=binary},Ctx,_,#b_literal{val=all},#b_literal{val=U}] + when is_integer(U), 1 =< U, U =< 256 -> + {yes,{Ctx,0,U}}; + [#b_literal{val=binary},Ctx,_,#b_literal{val=Size},#b_literal{val=U}] + when is_integer(Size) -> + {yes,{Ctx,Size*U,1}}; + [#b_literal{val=integer},Ctx,_,#b_literal{val=Size},#b_literal{val=U}] + when is_integer(Size) -> + {yes,{Ctx,Size*U,1}}; + [#b_literal{val=skip},Ctx,_,_,#b_literal{val=all},#b_literal{val=U}] -> + {yes,{Ctx,0,U}}; + [#b_literal{val=skip},Ctx,_,_,#b_literal{val=Size},#b_literal{val=U}] + when is_integer(Size) -> + {yes,{Ctx,Size*U,1}}; + [#b_literal{val=string},Ctx,#b_literal{val=Str}] when bit_size(Str) =< 64 -> + {yes,{Ctx,bit_size(Str),1}}; + _ -> + no + end. + +build_bs_ensure_match(L, {_,Size,Unit}, Count0, Blocks0) -> + BsMatchL = Count0, + Count1 = Count0 + 1, + {NewCtx,Count2} = new_var('@context', Count1), + {SuccBool,Count} = new_var('@ssa_bool', Count2), + + BsMatchBlk0 = map_get(L, Blocks0), + + #b_blk{is=MatchIs,last=#b_br{fail=Fail}} = BsMatchBlk0, + {Prefix,Suffix0} = splitwith(fun(#b_set{op=Op}) -> Op =/= bs_match end, MatchIs), + [BsMatch0|Suffix1] = Suffix0, + #b_set{args=[Type,_Ctx|Args]} = BsMatch0, + BsMatch = BsMatch0#b_set{args=[Type,NewCtx|Args]}, + Suffix = [BsMatch|Suffix1], + BsMatchBlk = BsMatchBlk0#b_blk{is=Suffix}, + + #b_set{args=[_,Ctx|_]} = keyfind(bs_match, #b_set.op, MatchIs), + Is = Prefix ++ [#b_set{op=bs_ensure, + dst=NewCtx, + args=[Ctx,#b_literal{val=Size},#b_literal{val=Unit}]}, + #b_set{op={succeeded,guard},dst=SuccBool,args=[NewCtx]}], + Blk = #b_blk{is=Is,last=#b_br{bool=SuccBool,succ=BsMatchL,fail=Fail}}, + + Blocks = Blocks0#{L := Blk, BsMatchL => BsMatchBlk}, + + {Blocks,Count}. + +%%% %%% Common utilities. %%% diff --git a/lib/compiler/src/beam_ssa_pp.erl b/lib/compiler/src/beam_ssa_pp.erl index 4e6974385e..41703b348a 100644 --- a/lib/compiler/src/beam_ssa_pp.erl +++ b/lib/compiler/src/beam_ssa_pp.erl @@ -179,10 +179,11 @@ format_terminator(#b_br{anno=A,bool=Bool,succ=Succ,fail=Fail}, FuncAnno) -> format_label(Succ), format_label(Fail)]); format_terminator(#b_switch{anno=A,arg=Arg,fail=Fail,list=List}, FuncAnno) -> + [format_instr_anno(A, FuncAnno, [Arg]), io_lib:format(" ~sswitch ~ts, ~ts, ~ts\n", [format_i_number(A),format_arg(Arg, FuncAnno), format_label(Fail), - format_switch_list(List, FuncAnno)]); + format_switch_list(List, FuncAnno)])]; format_terminator(#b_ret{anno=A,arg=Arg}, FuncAnno) -> io_lib:format(" ~sret ~ts\n", [format_i_number(A),format_arg(Arg, FuncAnno)]). @@ -333,6 +334,12 @@ format_type(#t_integer{elements={X,X}}) -> io_lib:format("~p", [X]); format_type(#t_integer{elements={Low,High}}) -> io_lib:format("~p..~p", [Low,High]); +format_type(#t_number{elements=any}) -> + "number()"; +format_type(#t_number{elements={X,X}}) -> + io_lib:format("number(~p)", [X]); +format_type(#t_number{elements={Low,High}}) -> + io_lib:format("number(~p, ~p)", [Low,High]); format_type(#t_list{type=ET,terminator=nil}) -> ["list(", format_type(ET), ")"]; format_type(#t_list{type=ET,terminator=TT}) -> @@ -347,6 +354,8 @@ format_type(#t_tuple{elements=Es,exact=Ex,size=S}) -> ["{", string:join(format_tuple_elems(S, Ex, Es, 1), ", "), "}"]; +format_type(other) -> + "other()"; format_type(pid) -> "pid()"; format_type(port) -> diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl index bb9aa75797..13b48c897a 100644 --- a/lib/compiler/src/beam_ssa_pre_codegen.erl +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -73,8 +73,8 @@ -import(lists, [all/2,any/2,append/1,duplicate/2, foldl/3,last/1,member/2,partition/2, - reverse/1,reverse/2,seq/2,sort/1, - splitwith/2,usort/1,zip/2]). + reverse/1,reverse/2,seq/2,sort/1,sort/2, + usort/1,zip/2]). -spec module(beam_ssa:b_module(), [compile:option()]) -> {'ok',beam_ssa:b_module()}. @@ -119,7 +119,7 @@ passes(Opts) -> ?PASS(fix_bs), ?PASS(sanitize), ?PASS(expand_match_fail), - ?PASS(use_set_tuple_element), + ?PASS(expand_update_tuple), ?PASS(place_frames), ?PASS(fix_receives), @@ -203,20 +203,23 @@ assert_no_ces(_, _, Blocks) -> Blocks. fix_bs(#st{ssa=Blocks,cnt=Count0}=St) -> F = fun(#b_set{op=bs_start_match,dst=Dst}, A) -> %% Mark the root of the match context list. - [{Dst,{context,Dst}}|A]; + A#{Dst => {context,Dst}}; + (#b_set{op=bs_ensure,dst=Dst,args=[ParentCtx|_]}, A) -> + %% Link this match context to the previous match context. + A#{Dst => ParentCtx}; (#b_set{op=bs_match,dst=Dst,args=[_,ParentCtx|_]}, A) -> - %% Link this match context the previous match context. - [{Dst,ParentCtx}|A]; + %% Link this match context to the previous match context. + A#{Dst => ParentCtx}; (_, A) -> A end, RPO = beam_ssa:rpo(Blocks), - case beam_ssa:fold_instrs(F, RPO, [], Blocks) of - [] -> + CtxChain = beam_ssa:fold_instrs(F, RPO, #{}, Blocks), + case map_size(CtxChain) of + 0 -> %% No binary matching in this function. St; - [_|_]=M -> - CtxChain = maps:from_list(M), + _ -> Linear0 = beam_ssa:linearize(Blocks), %% Insert position instructions where needed. @@ -347,6 +350,33 @@ bs_restores_is([#b_set{op=bs_start_match,dst=Start}|Is], FPos = SPos0, SPos = SPos0#{Start=>Start}, bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=bs_ensure,dst=NewPos,args=Args}|Is], + CtxChain, SPos0, _FPos, Rs0) -> + Start = bs_subst_ctx(NewPos, CtxChain), + [FromPos|_] = Args, + case SPos0 of + #{Start := FromPos} -> + %% Same position, no restore needed. + SPos = SPos0#{Start := NewPos}, + FPos = SPos0, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs0); + #{} -> + SPos = SPos0#{Start := NewPos}, + FPos = SPos0#{Start := FromPos}, + Rs = Rs0#{NewPos=>{Start,FromPos}}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs) + end; +bs_restores_is([#b_set{anno=#{ensured := _}, + op=bs_match,dst=NewPos,args=Args}|Is], + CtxChain, SPos0, _FPos, Rs) -> + %% This match instruction will be a part of a `bs_match` BEAM + %% instruction, so there will never be a restore to this + %% position. + Start = bs_subst_ctx(NewPos, CtxChain), + [_,FromPos|_] = Args, + SPos = SPos0#{Start := NewPos}, + FPos = SPos0#{Start := FromPos}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); bs_restores_is([#b_set{op=bs_match,dst=NewPos,args=Args}=I|Is], CtxChain, SPos0, _FPos, Rs0) -> Start = bs_subst_ctx(NewPos, CtxChain), @@ -483,13 +513,13 @@ bs_restore_args([], Pos, _CtxChain, _Dst, Rs) -> %% Insert all bs_save and bs_restore instructions. bs_insert_bsm3(Blocks, Saves, Restores) -> - bs_insert_1(Blocks, [], Saves, Restores, fun(I) -> I end). + bs_insert_1(Blocks, [], Saves, Restores). -bs_insert_1([{L,#b_blk{is=Is0}=Blk} | Bs], Deferred0, Saves, Restores, XFrm) -> +bs_insert_1([{L,#b_blk{is=Is0}=Blk} | Bs], Deferred0, Saves, Restores) -> Is1 = bs_insert_deferred(Is0, Deferred0), - {Is, Deferred} = bs_insert_is(Is1, Saves, Restores, XFrm, []), - [{L,Blk#b_blk{is=Is}} | bs_insert_1(Bs, Deferred, Saves, Restores, XFrm)]; -bs_insert_1([], [], _, _, _) -> + {Is, Deferred} = bs_insert_is(Is1, Saves, Restores, []), + [{L,Blk#b_blk{is=Is}} | bs_insert_1(Bs, Deferred, Saves, Restores)]; +bs_insert_1([], [], _, _) -> []. bs_insert_deferred([#b_set{op=bs_extract}=I | Is], Deferred) -> @@ -497,8 +527,7 @@ bs_insert_deferred([#b_set{op=bs_extract}=I | Is], Deferred) -> bs_insert_deferred(Is, Deferred) -> Deferred ++ Is. -bs_insert_is([#b_set{dst=Dst}=I0|Is], Saves, Restores, XFrm, Acc0) -> - I = XFrm(I0), +bs_insert_is([#b_set{dst=Dst}=I|Is], Saves, Restores, Acc0) -> Pre = case Restores of #{Dst:=R} -> [R]; #{} -> [] @@ -513,9 +542,9 @@ bs_insert_is([#b_set{dst=Dst}=I0|Is], Saves, Restores, XFrm, Acc0) -> %% Defer the save sequence to the success block. {reverse(Acc, Is), Post}; _ -> - bs_insert_is(Is, Saves, Restores, XFrm, Post ++ Acc) + bs_insert_is(Is, Saves, Restores, Post ++ Acc) end; -bs_insert_is([], _, _, _, Acc) -> +bs_insert_is([], _, _, Acc) -> {reverse(Acc), []}. %% Translate bs_match instructions to bs_get, bs_match_string, @@ -534,18 +563,45 @@ bs_instrs([{L,#b_blk{is=Is0}=Blk}|Bs], CtxChain, Acc0) -> bs_instrs(Bs, CtxChain, [{L,Blk#b_blk{is=Is}}|Acc0]) end; bs_instrs([], _, Acc) -> - reverse(Acc). + bs_rewrite_skip(Acc). + +bs_rewrite_skip([{L,#b_blk{is=Is0,last=Last0}=Blk}|Bs]) -> + case bs_rewrite_skip_is(Is0, []) of + no -> + [{L,Blk}|bs_rewrite_skip(Bs)]; + {yes,Is} -> + #b_br{succ=Succ} = Last0, + Last = beam_ssa:normalize(Last0#b_br{fail=Succ}), + [{L,Blk#b_blk{is=Is,last=Last}}|bs_rewrite_skip(Bs)] + end; +bs_rewrite_skip([]) -> + []. + +bs_rewrite_skip_is([#b_set{anno=#{ensured := true},op=bs_skip}=I0, + #b_set{op={succeeded,guard}}], Acc) -> + I = I0#b_set{op=bs_checked_skip}, + {yes,reverse(Acc, [I])}; +bs_rewrite_skip_is([I|Is], Acc) -> + bs_rewrite_skip_is(Is, [I|Acc]); +bs_rewrite_skip_is([], _Acc) -> + no. bs_instrs_is([#b_set{op={succeeded,_}}=I|Is], CtxChain, Acc) -> %% This instruction refers to a specific operation, so we must not %% substitute the context argument. bs_instrs_is(Is, CtxChain, [I | Acc]); -bs_instrs_is([#b_set{op=Op,args=Args0}=I0|Is], CtxChain, Acc) -> +bs_instrs_is([#b_set{anno=Anno0,op=Op,args=Args0}=I0|Is], CtxChain, Acc) -> Args = [bs_subst_ctx(A, CtxChain) || A <- Args0], I1 = I0#b_set{args=Args}, I = case {Op,Args} of {bs_match,[#b_literal{val=skip},Ctx,Type|As]} -> - I1#b_set{op=bs_skip,args=[Type,Ctx|As]}; + Anno = case Anno0 of + #{arg_types := #{4 := SizeType}} -> + Anno0#{arg_types := #{3 => SizeType}}; + #{} -> + Anno0 + end, + I1#b_set{anno=Anno,op=bs_skip,args=[Type,Ctx|As]}; {bs_match,[#b_literal{val=string},Ctx|As]} -> I1#b_set{op=bs_match_string,args=[Ctx|As]}; {_,_} -> @@ -560,10 +616,19 @@ bs_instrs_is([], _, Acc) -> bs_combine(Dst, Ctx, [{L,#b_blk{is=Is0}=Blk}|Acc]) -> [#b_set{}=Succeeded, - #b_set{op=bs_match,args=[Type,_|As]}=BsMatch|Is1] = reverse(Is0), - Is = reverse(Is1, [BsMatch#b_set{op=bs_get,dst=Dst,args=[Type,Ctx|As]}, - Succeeded#b_set{args=[Dst]}]), - [{L,Blk#b_blk{is=Is}}|Acc]. + #b_set{anno=Anno,op=bs_match,args=[Type,_|As]}=BsMatch|Is1] = reverse(Is0), + if + is_map_key(ensured, Anno) -> + Is = reverse(Is1, [BsMatch#b_set{op=bs_checked_get,dst=Dst, + args=[Type,Ctx|As]}]), + #b_blk{last=#b_br{succ=Succ}=Br0} = Blk, + Br = beam_ssa:normalize(Br0#b_br{fail=Succ}), + [{L,Blk#b_blk{is=Is,last=Br}}|Acc]; + true -> + Is = reverse(Is1, [BsMatch#b_set{op=bs_get,dst=Dst,args=[Type,Ctx|As]}, + Succeeded#b_set{args=[Dst]}]), + [{L,Blk#b_blk{is=Is}}|Acc] + end. bs_subst_ctx(#b_var{}=Var, CtxChain) -> case CtxChain of @@ -589,84 +654,89 @@ bs_subst_ctx(Other, _CtxChain) -> sanitize(#st{ssa=Blocks0,cnt=Count0}=St) -> Ls = beam_ssa:rpo(Blocks0), - {Blocks,Count} = sanitize(Ls, Blocks0, Count0, #{}), + {Blocks,Count} = sanitize(Ls, Blocks0, Count0, #{}, #{0 => reachable}), St#st{ssa=Blocks,cnt=Count}. -sanitize([L|Ls], Blocks0, Count0, Values0) -> - #b_blk{is=Is0,last=Last0} = Blk0 = map_get(L, Blocks0), - case sanitize_is(Is0, Last0, Blocks0, Count0, Values0, false, []) of - no_change -> - sanitize(Ls, Blocks0, Count0, Values0); - {Is,Last,Count,Values} -> - Blk = Blk0#b_blk{is=Is,last=Last}, - Blocks = Blocks0#{L:=Blk}, - sanitize(Ls, Blocks, Count, Values) - end; -sanitize([], Blocks0, Count, Values) -> - Blocks = if - map_size(Values) =:= 0 -> - Blocks0; - true -> - RPO = beam_ssa:rpo(Blocks0), - beam_ssa:rename_vars(Values, RPO, Blocks0) - end, +sanitize([L|Ls], InBlocks, Count0, Values0, Blocks0) -> + case is_map_key(L, Blocks0) of + false -> + %% This block will never be reached. Discard it. + sanitize(Ls, InBlocks, Count0, Values0, Blocks0); + true -> + #b_blk{is=Is0,last=Last0} = Blk0 = map_get(L, InBlocks), + case sanitize_is(Is0, Last0, InBlocks, Blocks0, Count0, Values0, false, []) of + no_change -> + Blk = sanitize_last(Blk0, Values0), + Blocks1 = Blocks0#{L := Blk}, + Blocks = sanitize_reachable(Blk0, Blocks1), + sanitize(Ls, InBlocks, Count0, Values0, Blocks); + {Is,Last,Count,Values} -> + Blk1 = Blk0#b_blk{is=Is,last=Last}, + Blk = sanitize_last(Blk1, Values), + Blocks1 = Blocks0#{L := Blk}, + Blocks = sanitize_reachable(Blk, Blocks1), + sanitize(Ls, InBlocks, Count, Values, Blocks) + end + end; +sanitize([], _InBlocks, Count, _Values, Blocks) -> + {Blocks,Count}. - %% Unreachable blocks can cause problems for the dominator calculations. - Ls = beam_ssa:rpo(Blocks), - Reachable = gb_sets:from_list(Ls), - {case map_size(Blocks) =:= gb_sets:size(Reachable) of - true -> Blocks; - false -> remove_unreachable(Ls, Blocks, Reachable, []) - end, Count}. +sanitize_reachable(Blk, Blocks) -> + foldl(fun(S, A) when is_map_key(S, A) -> A; + (S, A) -> A#{S => reachable} + end, Blocks, beam_ssa:successors(Blk)). sanitize_is([#b_set{op=get_map_element,args=Args0}=I0|Is], - Last, Blocks, Count0, Values, Changed, Acc) -> + Last, InBlocks, Blocks, Count0, Values, Changed, Acc) -> case sanitize_args(Args0, Values) of [#b_literal{}=Map,Key] -> %% Bind the literal map to a variable. {MapVar,Count} = new_var('@ssa_map', Count0), I = I0#b_set{args=[MapVar,Key]}, Copy = #b_set{op=copy,dst=MapVar,args=[Map]}, - sanitize_is(Is, Last, Blocks, Count, Values, true, [I,Copy|Acc]); + sanitize_is(Is, Last, InBlocks, Blocks, Count, + Values, true, [I,Copy|Acc]); [_,_]=Args0 -> - sanitize_is(Is, Last, Blocks, Count0, Values, Changed, [I0|Acc]); + sanitize_is(Is, Last, InBlocks, Blocks, Count0, + Values, Changed, [I0|Acc]); [_,_]=Args -> I = I0#b_set{args=Args}, - sanitize_is(Is, Last, Blocks, Count0, Values, true, [I|Acc]) + sanitize_is(Is, Last, InBlocks, Blocks, Count0, + Values, true, [I|Acc]) end; sanitize_is([#b_set{op=call,dst=CallDst}=Call, #b_set{op={succeeded,body},dst=SuccDst,args=[CallDst]}=Succ], #b_br{bool=SuccDst,succ=SuccLbl,fail=?EXCEPTION_BLOCK}=Last0, - Blocks, Count, Values, Changed, Acc) -> - case Blocks of - #{ SuccLbl := #b_blk{is=[],last=#b_ret{arg=CallDst}=Last} } -> + InBlocks, Blocks, Count, Values, Changed, Acc) -> + case InBlocks of + #{SuccLbl := #b_blk{is=[],last=#b_ret{arg=CallDst}=Last}} -> %% Tail call that may fail, translate the terminator to an ordinary %% return to simplify code generation. - do_sanitize_is(Call, [], Last, Blocks, Count, Values, - true, Acc); + do_sanitize_is(Call, [], Last, InBlocks, Blocks, + Count, Values, true, Acc); #{} -> - do_sanitize_is(Call, [Succ], Last0, Blocks, Count, Values, - Changed, Acc) + do_sanitize_is(Call, [Succ], Last0, InBlocks, Blocks, + Count, Values, Changed, Acc) end; sanitize_is([#b_set{op=Op,dst=Dst}=Fail, #b_set{op={succeeded,body},args=[Dst]}], #b_br{fail=?EXCEPTION_BLOCK}, - Blocks, Count, Values, _Changed, Acc) + InBlocks, Blocks, Count, Values, _Changed, Acc) when Op =:= match_fail; Op =:= resume -> %% Match failure or rethrow without a local handler. Translate the %% terminator to an ordinary return to simplify code generation. Last = #b_ret{arg=Dst}, - do_sanitize_is(Fail, [], Last, Blocks, Count, Values, true, Acc); + do_sanitize_is(Fail, [], Last, InBlocks, Blocks, Count, Values, true, Acc); sanitize_is([#b_set{op=match_fail,dst=RaiseDst}, #b_set{op={succeeded,guard},dst=SuccDst,args=[RaiseDst]}], #b_br{bool=SuccDst}=Last0, - Blocks, Count, Values, _Changed, Acc) -> + InBlocks, Blocks, Count, Values, _Changed, Acc) -> %% Match failures may be present in guards when optimizations are turned %% off. They must be treated as if they always fail. Last = beam_ssa:normalize(Last0#b_br{bool=#b_literal{val=false}}), - sanitize_is([], Last, Blocks, Count, Values, true, Acc); + sanitize_is([], Last, InBlocks, Blocks, Count, Values, true, Acc); sanitize_is([#b_set{op={succeeded,_Kind},dst=Dst,args=[Arg0]}=I0], - #b_br{bool=Dst}=Last, _Blocks, Count, Values, _Changed, Acc) -> + #b_br{bool=Dst}=Last, _InBlocks, _Blocks, Count, Values, _Changed, Acc) -> %% We no longer need to distinguish between guard and body checks, so we'll %% rewrite this as a plain 'succeeded'. case sanitize_arg(Arg0, Values) of @@ -678,7 +748,7 @@ sanitize_is([#b_set{op={succeeded,_Kind},dst=Dst,args=[Arg0]}=I0], {reverse(Acc), Last, Count, Values#{ Dst => Value }} end; sanitize_is([#b_set{op={succeeded,Kind},args=[Arg0]} | Is], - Last, Blocks, Count, Values, _Changed, Acc) -> + Last, InBlocks, Blocks, Count, Values, _Changed, Acc) -> %% We're no longer branching on this instruction and can safely remove it. [] = Is, #b_br{succ=Same,fail=Same} = Last, %Assertion. if @@ -687,24 +757,41 @@ sanitize_is([#b_set{op={succeeded,Kind},args=[Arg0]} | Is], %% in a try/catch; rewrite the terminator to a return. body = Kind, %Assertion. Arg = sanitize_arg(Arg0, Values), - sanitize_is(Is, #b_ret{arg=Arg}, Blocks, Count, Values, true, Acc); + sanitize_is(Is, #b_ret{arg=Arg}, InBlocks, Blocks, + Count, Values, true, Acc); Same =/= ?EXCEPTION_BLOCK -> %% We either always succeed, or always fail to somewhere other than %% the exception block. true = Kind =:= guard orelse Kind =:= body, %Assertion. - sanitize_is(Is, Last, Blocks, Count, Values, true, Acc) + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values, true, Acc) end; -sanitize_is([#b_set{op=bs_test_tail}=I], Last, Blocks, Count, Values, - Changed, Acc) -> +sanitize_is([#b_set{op=bs_test_tail}=I], Last, InBlocks, Blocks, + Count, Values, Changed, Acc) -> case Last of #b_br{succ=Same,fail=Same} -> - sanitize_is([], Last, Blocks, Count, Values, true, Acc); + sanitize_is([], Last, InBlocks, Blocks, + Count, Values, true, Acc); _ -> - do_sanitize_is(I, [], Last, Blocks, Count, Values, Changed, Acc) + do_sanitize_is(I, [], Last, InBlocks, Blocks, + Count, Values, Changed, Acc) + end; +sanitize_is([#b_set{op=bs_get,args=Args0}=I0|Is], Last, InBlocks, Blocks, + Count, Values, Changed, Acc) -> + case {Args0,sanitize_args(Args0, Values)} of + {[_,_,_,#b_var{},_],[Type,Val,Flags,#b_literal{val=all},Unit]} -> + %% The size `all` is used for the size of the final binary + %% segment in a pattern. Using `all` explicitly is not allowed, + %% so we convert it to an obvious invalid size. + Args = [Type,Val,Flags,#b_literal{val=bad_size},Unit], + I = I0#b_set{args=Args}, + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values, true, [I|Acc]); + {_,Args} -> + I = I0#b_set{args=Args}, + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values, Changed, [I|Acc]) end; -sanitize_is([#b_set{}=I|Is], Last, Blocks, Count, Values, Changed, Acc) -> - do_sanitize_is(I, Is, Last, Blocks, Count, Values, Changed, Acc); -sanitize_is([], Last, _Blocks, Count, Values, Changed, Acc) -> +sanitize_is([#b_set{}=I|Is], Last, InBlocks, Blocks, Count, Values, Changed, Acc) -> + do_sanitize_is(I, Is, Last, InBlocks, Blocks, Count, Values, Changed, Acc); +sanitize_is([], Last, _InBlocks, _Blocks, Count, Values, Changed, Acc) -> case Changed of true -> {reverse(Acc), Last, Count, Values}; @@ -713,34 +800,59 @@ sanitize_is([], Last, _Blocks, Count, Values, Changed, Acc) -> end. do_sanitize_is(#b_set{op=Op,dst=Dst,args=Args0}=I0, - Is, Last, Blocks, Count, Values, Changed0, Acc) -> + Is, Last, InBlocks, Blocks, Count, Values, Changed0, Acc) -> Args = sanitize_args(Args0, Values), - case sanitize_instr(Op, Args, I0) of + case sanitize_instr(Op, Args, I0, Blocks) of {value,Value0} -> Value = #b_literal{val=Value0}, - sanitize_is(Is, Last, Blocks, Count, Values#{Dst=>Value}, + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values#{Dst=>Value}, true, Acc); {ok,I} -> - sanitize_is(Is, Last, Blocks, Count, Values, true, [I|Acc]); + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values, true, [I|Acc]); ok -> I = I0#b_set{args=Args}, Changed = Changed0 orelse Args =/= Args0, - sanitize_is(Is, Last, Blocks, Count, Values, Changed, [I|Acc]) + sanitize_is(Is, Last, InBlocks, Blocks, Count, Values, Changed, [I|Acc]) + end. + +sanitize_last(#b_blk{last=Last0}=Blk, Values) -> + Last = case Last0 of + #b_br{bool=#b_literal{}} -> + Last0; + #b_br{bool=Bool} -> + beam_ssa:normalize(Last0#b_br{bool=sanitize_arg(Bool, Values)}); + #b_ret{arg=Arg} -> + Last0#b_ret{arg=sanitize_arg(Arg, Values)}; + #b_switch{arg=Arg} -> + beam_ssa:normalize(Last0#b_switch{arg=sanitize_arg(Arg, Values)}) + end, + if + Last =/= Last0 -> + Blk#b_blk{last=Last}; + true -> + Blk end. sanitize_args(Args, Values) -> [sanitize_arg(Arg, Values) || Arg <- Args]. +sanitize_arg(#b_remote{mod=Mod0,name=Name0}=Remote, Values) -> + Mod = sanitize_arg(Mod0, Values), + Name = sanitize_arg(Name0, Values), + Remote#b_remote{mod=Mod,name=Name}; +sanitize_arg({#b_var{}=Var,L}, Values) -> + {sanitize_arg(Var, Values),L}; sanitize_arg(#b_var{}=Var, Values) -> case Values of - #{Var:=New} -> New; + #{Var := New} -> New; #{} -> Var end; sanitize_arg(Arg, _Values) -> Arg. - -sanitize_instr(phi, PhiArgs, _I) -> +sanitize_instr(phi, PhiArgs0, I, Blocks) -> + PhiArgs = [{V,L} || {V,L} <- PhiArgs0, + is_map_key(L, Blocks)], case phi_all_same_literal(PhiArgs) of true -> %% (Can only happen when some optimizations have been @@ -756,8 +868,11 @@ sanitize_instr(phi, PhiArgs, _I) -> [{#b_literal{val=Val},_}|_] = PhiArgs, {value,Val}; false -> - ok + {ok,I#b_set{args=PhiArgs}} end; +sanitize_instr(Op, Args, I, _Blocks) -> + sanitize_instr(Op, Args, I). + sanitize_instr({bif,Bif}, [#b_literal{val=Lit}], _I) -> case erl_bifs:is_pure(erlang, Bif, 1) of false -> @@ -783,7 +898,7 @@ sanitize_instr(bs_match, Args, I) -> %% value is never used, because the match can always fail (for example, %% if it is a NaN). [#b_literal{val=float}|_] = Args, %Assertion. - {ok,I#b_set{op=bs_get}}; + {ok,I#b_set{op=bs_get,args=Args}}; sanitize_instr(get_hd, [#b_literal{val=[Hd|_]}], _I) -> {value,Hd}; sanitize_instr(get_tl, [#b_literal{val=[_|Tl]}], _I) -> @@ -807,27 +922,11 @@ sanitize_instr(is_tagged_tuple, [#b_literal{val=Tuple}, true -> {value,false} end; +sanitize_instr(succeeded, [#b_literal{}], _I) -> + {value,true}; sanitize_instr(_, _, _) -> ok. -remove_unreachable([L|Ls], Blocks, Reachable, Acc) -> - #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), - case split_phis(Is0) of - {[_|_]=Phis,Rest} -> - Is = [prune_phi(Phi, Reachable) || Phi <- Phis] ++ Rest, - Blk = Blk0#b_blk{is=Is}, - remove_unreachable(Ls, Blocks, Reachable, [{L,Blk}|Acc]); - {[],_} -> - remove_unreachable(Ls, Blocks, Reachable, [{L,Blk0}|Acc]) - end; -remove_unreachable([], _Blocks, _, Acc) -> - maps:from_list(Acc). - -prune_phi(#b_set{args=Args0}=Phi, Reachable) -> - Args = [A || {_,Pred}=A <- Args0, - gb_sets:is_element(Pred, Reachable)], - Phi#b_set{args=Args}. - phi_all_same_literal([{#b_literal{}=Arg, _From} | Phis]) -> phi_all_same_literal_1(Phis, Arg); phi_all_same_literal([_|_]) -> @@ -966,128 +1065,87 @@ find_fc_errors([#b_function{bs=Blocks}|Fs], Acc0) -> find_fc_errors([], Acc) -> Acc. - +%%% expand_update_tuple(St0) -> St %%% -%%% Introduce the set_tuple_element instructions to make -%%% multiple-field record updates faster. +%%% Expands the update_tuple psuedo-instruction into its actual instructions. %%% -%%% The expansion of record field updates, when more than one field is -%%% updated, but not a majority of the fields, will create a sequence of -%%% calls to `erlang:setelement(Index, Value, Tuple)` where Tuple in the -%%% first call is the original record tuple, and in the subsequent calls -%%% Tuple is the result of the previous call. Furthermore, all Index -%%% values are constant positive integers, and the first call to -%%% `setelement` will have the greatest index. Thus all the following -%%% calls do not actually need to test at run-time whether Tuple has type -%%% tuple, nor that the index is within the tuple bounds. -%%% -%%% Since this optimization introduces destructive updates, it used to -%%% be done as the very last Core Erlang pass before going to -%%% lower-level code. However, it turns out that this kind of destructive -%%% updates are awkward also in SSA code and can prevent or complicate -%%% type analysis and aggressive optimizations. -%%% -%%% NOTE: Because there no write barriers in the system, this kind of -%%% optimization can only be done when we are sure that garbage -%%% collection will not be triggered between the creation of the tuple -%%% and the destructive updates - otherwise we might insert pointers -%%% from an older generation to a newer. -%%% - -use_set_tuple_element(#st{ssa=Blocks0}=St) -> - Uses = count_uses(Blocks0), - RPO = reverse(beam_ssa:rpo(Blocks0)), - Blocks = use_ste_1(RPO, Uses, Blocks0), - St#st{ssa=Blocks}. - -use_ste_1([L|Ls], Uses, Blocks) -> - #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), - case use_ste_is(Is0, Uses) of - Is0 -> - use_ste_1(Ls, Uses, Blocks); - Is -> - Blk = Blk0#b_blk{is=Is}, - use_ste_1(Ls, Uses, Blocks#{L:=Blk}) - end; -use_ste_1([], _, Blocks) -> Blocks. - -%%% Optimize within a single block. +expand_update_tuple(#st{ssa=Blocks0,cnt=Count0}=St) -> + Linear0 = beam_ssa:linearize(Blocks0), + {Linear, Count} = expand_update_tuple_1(Linear0, Count0, []), + Blocks = maps:from_list(Linear), + St#st{ssa=Blocks,cnt=Count}. -use_ste_is([#b_set{}=I|Is0], Uses) -> - Is = use_ste_is(Is0, Uses), - case extract_ste(I) of - none -> - [I|Is]; - Extracted -> - use_ste_call(Extracted, I, Is, Uses) +expand_update_tuple_1([{L, #b_blk{is=Is0}=B0} | Bs], Count0, Acc0) -> + case expand_update_tuple_is(Is0, Count0, []) of + {Is, Count} -> + expand_update_tuple_1(Bs, Count, [{L, B0#b_blk{is=Is}} | Acc0]); + {Is, NextIs, Count1} -> + %% There are `set_tuple_element` instructions that we must put into + %% a new block to avoid separating the `setelement` instruction from + %% its `succeeded` instruction. + #b_blk{last=Br} = B0, + #b_br{succ=Succ} = Br, + NextL = Count1, + Count = Count1 + 1, + NextBr = #b_br{bool=#b_literal{val=true},succ=Succ,fail=Succ}, + NextB = #b_blk{is=NextIs,last=NextBr}, + B = B0#b_blk{is=Is,last=Br#b_br{succ=NextL}}, + Acc = [{NextL, NextB}, {L, B} | Acc0], + expand_update_tuple_1(Bs, Count, Acc) + end; +expand_update_tuple_1([], Count, Acc) -> + {Acc, Count}. + +expand_update_tuple_is([#b_set{op=update_tuple, args=[Src | Args]}=I0 | Is], + Count0, Acc) -> + {SetElement, Sets, Count} = expand_update_tuple_list(Args, I0, Src, Count0), + case {Sets, Is} of + {[_ | _], [#b_set{op=succeeded}]} -> + {reverse(Acc, [SetElement | Is]), reverse(Sets), Count}; + {_, _} -> + expand_update_tuple_is(Is, Count, Sets ++ [SetElement | Acc]) end; -use_ste_is([], _Uses) -> []. +expand_update_tuple_is([I | Is], Count, Acc) -> + expand_update_tuple_is(Is, Count, [I | Acc]); +expand_update_tuple_is([], Count, Acc) -> + {reverse(Acc), Count}. -use_ste_call({Dst0,Pos0,_Var0,_Val0}, Call1, Is0, Uses) -> - case get_ste_call(Is0, []) of - {Prefix,{Dst1,Pos1,Dst0,Val1},Call2,Is} - when Pos1 > 0, Pos0 > Pos1 -> - case is_single_use(Dst0, Uses) of - true -> - Call = Call1#b_set{dst=Dst1}, - Args = [Val1,Dst1,#b_literal{val=Pos1-1}], - Dsetel = Call2#b_set{op=set_tuple_element, - dst=Dst0, - args=Args}, - [Call|Prefix] ++ [Dsetel|Is]; - false -> - [Call1|Is0] - end; - _ -> - [Call1|Is0] - end. - -get_ste_call([#b_set{op=get_tuple_element}=I|Is], Acc) -> - get_ste_call(Is, [I|Acc]); -get_ste_call([#b_set{op=call}=I|Is], Acc) -> - case extract_ste(I) of - none -> - none; - Extracted -> - {reverse(Acc),Extracted,I,Is} - end; -get_ste_call(_, _) -> none. - -extract_ste(#b_set{op=call,dst=Dst, - args=[#b_remote{mod=#b_literal{val=M}, - name=#b_literal{val=F}}|Args]}) -> - case {M,F,Args} of - {erlang,setelement,[#b_literal{val=Pos},Tuple,Val]} -> - {Dst,Pos,Tuple,Val}; - {_,_,_} -> - none - end; -extract_ste(#b_set{}) -> none. - -%% Count how many times each variable is used. - -count_uses(Blocks) -> - count_uses_blk(maps:values(Blocks), #{}). - -count_uses_blk([#b_blk{is=Is,last=Last}|Bs], CountMap0) -> - F = fun(I, CountMap) -> - foldl(fun(Var, Acc) -> - case Acc of - #{Var:=2} -> Acc; - #{Var:=C} -> Acc#{Var:=C+1}; - #{} -> Acc#{Var=>1} - end - end, CountMap, beam_ssa:used(I)) - end, - CountMap = F(Last, foldl(F, CountMap0, Is)), - count_uses_blk(Bs, CountMap); -count_uses_blk([], CountMap) -> CountMap. - -is_single_use(V, Uses) -> - case Uses of - #{V:=1} -> true; - #{} -> false - end. +%% Expands an update_tuple list into setelement/3 + set_tuple_element. +%% +%% Note that it returns the instructions in reverse order. +expand_update_tuple_list(Args, I0, Src, Count0) -> + [Index, Value | Rest] = sort_update_tuple(Args, []), + + %% set_tuple_element is destructive, so we have to start off with a + %% setelement/3 call to give them something to work on. + I = I0#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=setelement}, + arity=3}, + Index, Src, Value]}, + {Sets, Count} = expand_update_tuple_list_1(Rest, I#b_set.dst, Count0, []), + {I, Sets, Count}. + +expand_update_tuple_list_1([], _Src, Count, Acc) -> + {Acc, Count}; +expand_update_tuple_list_1([Index0, Value | Updates], Src, Count0, Acc) -> + %% Change to the 0-based indexing used by `set_tuple_element`. + Index = #b_literal{val=(Index0#b_literal.val - 1)}, + {Dst, Count} = new_var('@ssa_dummy', Count0), + SetOp = #b_set{op=set_tuple_element, + dst=Dst, + args=[Value, Src, Index]}, + expand_update_tuple_list_1(Updates, Src, Count, [SetOp | Acc]). + +%% Sorts updates so that the highest index comes first, letting us use +%% set_tuple_element for all subsequent operations as we know their indexes +%% will be valid. +sort_update_tuple([_Index, _Value]=Args, []) -> + Args; +sort_update_tuple([#b_literal{}=Index, Value | Updates], Acc) -> + sort_update_tuple(Updates, [{Index, Value} | Acc]); +sort_update_tuple([], Acc) -> + append([[Index, Value] || {Index, Value} <- sort(fun erlang:'>='/2, Acc)]). %%% %%% Find out where frames should be placed. @@ -2528,6 +2586,7 @@ reserve_zreg([#b_set{op=Op,dst=Dst} | Is], Last, ShortLived, A) -> end; reserve_zreg([], _, _, A) -> A. +use_zreg(bs_checked_skip) -> yes; use_zreg(bs_match_string) -> yes; use_zreg(bs_set_position) -> yes; use_zreg(kill_try_tag) -> yes; @@ -3143,9 +3202,6 @@ rel2fam(S0) -> S = sofs:rel2fam(S1), sofs:to_external(S). -split_phis(Is) -> - splitwith(fun(#b_set{op=Op}) -> Op =:= phi end, Is). - is_yreg({y,_}) -> true; is_yreg({x,_}) -> false; is_yreg({z,_}) -> false; diff --git a/lib/compiler/src/beam_ssa_recv.erl b/lib/compiler/src/beam_ssa_recv.erl index f1d58ffb16..f0dc752f52 100644 --- a/lib/compiler/src/beam_ssa_recv.erl +++ b/lib/compiler/src/beam_ssa_recv.erl @@ -342,7 +342,7 @@ si_remote_call_1(Dst, [Callee | Args], Lbl, Blocks) -> none end, case MFA of - {erlang,alias,A} when 0 =< A, A =< 1 -> + {erlang,alias,A} when is_integer(A), 0 =< A, A =< 1 -> {makes_ref, Lbl, Dst}; {erlang,demonitor,2} -> case Args of @@ -357,12 +357,12 @@ si_remote_call_1(Dst, [Callee | Args], Lbl, Blocks) -> end; {erlang,make_ref,0} -> {makes_ref, Lbl, Dst}; - {erlang,monitor,A} when 2 =< A, A =< 3 -> + {erlang,monitor,A} when is_integer(A), 2 =< A, A =< 3 -> {makes_ref, Lbl, Dst}; - {erlang,spawn_monitor,A} when 1 =< A, A =< 4 -> + {erlang,spawn_monitor,A} when is_integer(A), 1 =< A, A =< 4 -> RPO = beam_ssa:rpo([Lbl], Blocks), si_ref_in_tuple(RPO, Blocks, Dst); - {erlang,spawn_request,A} when 1 =< A, A =< 5 -> + {erlang,spawn_request,A} when is_integer(A), 1 =< A, A =< 5 -> {makes_ref, Lbl, Dst}; _ -> %% As an aside, spawn_opt/2-5 is trivially supported by handling it diff --git a/lib/compiler/src/beam_ssa_share.erl b/lib/compiler/src/beam_ssa_share.erl index bd21655914..13b6038868 100644 --- a/lib/compiler/src/beam_ssa_share.erl +++ b/lib/compiler/src/beam_ssa_share.erl @@ -51,7 +51,7 @@ module(#b_module{body=Fs0}=Module, _Opts) -> Blocks0 :: beam_ssa:block_map(), Blk :: beam_ssa:b_blk(). -block(#b_blk{is=Is0,last=Last0}=Blk, Blocks) -> +block(#b_blk{is=Is0,last=Last0}=Blk, Blocks) when is_map(Blocks) -> case share_terminator(Last0, Blocks) of none -> Blk; @@ -75,7 +75,7 @@ block(#b_blk{is=Is0,last=Last0}=Blk, Blocks) -> %%% Local functions. %%% -function(#b_function{anno=Anno,bs=Blocks0}=F) -> +function(#b_function{anno=Anno,bs=Blocks0}=F) when is_map(Blocks0) -> try PO = reverse(beam_ssa:rpo(Blocks0)), {Blocks1,Changed} = blocks(PO, Blocks0, false), diff --git a/lib/compiler/src/beam_ssa_throw.erl b/lib/compiler/src/beam_ssa_throw.erl index c2dc552119..63a5541d01 100644 --- a/lib/compiler/src/beam_ssa_throw.erl +++ b/lib/compiler/src/beam_ssa_throw.erl @@ -442,7 +442,7 @@ ois_is([#b_set{op={bif,is_list},dst=Dst,args=[Src]} | Is], Ts) -> ois_is([#b_set{op={bif,is_map},dst=Dst,args=[Src]} | Is], Ts) -> ois_type_test(Src, Dst, #t_map{}, Is, Ts); ois_is([#b_set{op={bif,is_number},dst=Dst,args=[Src]} | Is], Ts) -> - ois_type_test(Src, Dst, number, Is, Ts); + ois_type_test(Src, Dst, #t_number{}, Is, Ts); ois_is([#b_set{op={bif,is_tuple},dst=Dst,args=[Src]} | Is], Ts) -> ois_type_test(Src, Dst, #t_tuple{}, Is, Ts); ois_is([#b_set{op=is_nonempty_list,dst=Dst,args=[Src]} | Is], Ts) -> diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index 77b723caa0..b293250365 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -533,7 +533,9 @@ opt_bs([{L, #b_blk{is=Is0,last=Last0}=Blk0} | Bs], SuccTypes = update_success_types(Last1, Ts, Ds, Meta, SuccTypes0), UsedOnce = Meta#metadata.used_once, - {Last, Ls1} = update_successors(Last1, Ts, Ds, Ls0, UsedOnce), + {Last2, Ls1} = update_successors(Last1, Ts, Ds, Ls0, UsedOnce), + + Last = opt_anno_types(Last2, Ts), Ls = Ls1#{ L := {outgoing, Ts} }, %Assertion. @@ -598,7 +600,17 @@ opt_anno_types(#b_set{op=Op,args=Args}=I, Ts) -> case benefits_from_type_anno(Op, Args) of true -> opt_anno_types_1(I, Args, Ts, 0, #{}); false -> I - end. + end; +opt_anno_types(#b_switch{anno=Anno0,arg=Arg}=I, Ts) -> + case concrete_type(Arg, Ts) of + any -> + I; + Type -> + Anno = Anno0#{arg_types => #{0 => Type}}, + I#b_switch{anno=Anno} + end; +opt_anno_types(I, _Ts) -> + I. opt_anno_types_1(I, [#b_var{}=Var | Args], Ts, Index, Acc0) -> case concrete_type(Var, Ts) of @@ -634,6 +646,8 @@ benefits_from_type_anno({bif,Op}, Args) -> not erl_internal:bool_op(Op, length(Args)); benefits_from_type_anno(bs_create_bin, _Args) -> true; +benefits_from_type_anno(bs_match, _Args) -> + true; benefits_from_type_anno(is_tagged_tuple, _Args) -> true; benefits_from_type_anno(call, [#b_var{} | _]) -> @@ -906,8 +920,20 @@ update_anno_types_1([#b_var{}=V|As], Ts, Index, ArgTypes) -> case beam_types:meet(T0, T1) of any -> update_anno_types_1(As, Ts, Index + 1, ArgTypes); + none -> + %% This instruction will never be reached. This happens when + %% compiling code such as the following: + %% + %% f(X) when is_integer(X), 0 =< X, X < 64 -> + %% (X = bnot X) + 1. + %% + %% The main type optimization sub pass will not find out + %% that `(X = bnot X)` will never succeed and that the `+` + %% operator is never executed, but this sub pass will. + %% This happens very rarely; therefore, don't bother removing + %% the unreachable instruction. + update_anno_types_1(As, Ts, Index + 1, ArgTypes); T -> - true = T =/= none, %Assertion. update_anno_types_1(As, Ts, Index + 1, ArgTypes#{Index => T}) end; update_anno_types_1([_|As], Ts, Index, ArgTypes) -> @@ -1012,10 +1038,19 @@ simplify(#b_set{op=bs_match,dst=Dst,args=Args0}=I0, Ts0, Ds0, _Ls, Sub) -> Ts = update_types(I, Ts0, Ds0), Ds = Ds0#{ Dst => I }, {I, Ts, Ds}; +simplify(#b_set{op=bs_create_bin=Op,dst=Dst,args=Args0,anno=Anno}=I0, + Ts0, Ds0, _Ls, Sub) -> + Args = simplify_args(Args0, Ts0, Sub), + I1 = I0#b_set{args=Args}, + #t_bitstring{size_unit=Unit} = T = type(Op, Args, Anno, Ts0, Ds0), + I = beam_ssa:add_anno(unit, Unit, I1), + Ts = Ts0#{ Dst => T }, + Ds = Ds0#{ Dst => I }, + {I, Ts, Ds}; simplify(#b_set{dst=Dst,args=Args0}=I0, Ts0, Ds0, _Ls, Sub) -> Args = simplify_args(Args0, Ts0, Sub), I1 = beam_ssa:normalize(I0#b_set{args=Args}), - case simplify(I1, Ts0) of + case simplify(I1, Ts0, Ds0) of #b_set{}=I -> Ts = update_types(I, Ts0, Ds0), Ds = Ds0#{ Dst => I }, @@ -1026,54 +1061,67 @@ simplify(#b_set{dst=Dst,args=Args0}=I0, Ts0, Ds0, _Ls, Sub) -> Sub#{ Dst => Var } end. -simplify(#b_set{op={bif,'and'},args=Args}=I, Ts) -> +simplify(#b_set{op={bif,'band'},args=Args}=I, Ts, Ds) -> + case normalized_types(Args, Ts) of + [#t_integer{elements=R},#t_integer{elements={M,M}}] -> + case beam_bounds:is_masking_redundant(R, M) of + true -> + %% M is mask that will have no effect. + hd(Args); + false -> + eval_bif(I, Ts, Ds) + end; + [_,_] -> + eval_bif(I, Ts, Ds) + end; +simplify(#b_set{op={bif,'and'},args=Args}=I, Ts, Ds) -> case is_safe_bool_op(Args, Ts) of true -> case Args of [_,#b_literal{val=false}=Res] -> Res; [Res,#b_literal{val=true}] -> Res; - _ -> eval_bif(I, Ts) + _ -> eval_bif(I, Ts, Ds) end; false -> I end; -simplify(#b_set{op={bif,'or'},args=Args}=I, Ts) -> +simplify(#b_set{op={bif,'or'},args=Args}=I, Ts, Ds) -> case is_safe_bool_op(Args, Ts) of true -> case Args of [Res,#b_literal{val=false}] -> Res; [_,#b_literal{val=true}=Res] -> Res; - _ -> eval_bif(I, Ts) + _ -> eval_bif(I, Ts, Ds) end; false -> I end; -simplify(#b_set{op={bif,element},args=[#b_literal{val=Index},Tuple]}=I0, Ts) -> +simplify(#b_set{op={bif,element},args=[#b_literal{val=Index},Tuple]}=I0, Ts, Ds) -> case normalized_type(Tuple, Ts) of #t_tuple{size=Size} when is_integer(Index), 1 =< Index, Index =< Size -> I = I0#b_set{op=get_tuple_element, args=[Tuple,#b_literal{val=Index-1}]}, - simplify(I, Ts); + simplify(I, Ts, Ds); _ -> - eval_bif(I0, Ts) + eval_bif(I0, Ts, Ds) end; -simplify(#b_set{op={bif,hd},args=[List]}=I, Ts) -> +simplify(#b_set{op={bif,hd},args=[List]}=I, Ts, Ds) -> case normalized_type(List, Ts) of #t_cons{} -> I#b_set{op=get_hd}; _ -> - eval_bif(I, Ts) + eval_bif(I, Ts, Ds) end; -simplify(#b_set{op={bif,tl},args=[List]}=I, Ts) -> +simplify(#b_set{op={bif,tl},args=[List]}=I, Ts, Ds) -> case normalized_type(List, Ts) of #t_cons{} -> I#b_set{op=get_tl}; _ -> - eval_bif(I, Ts) + eval_bif(I, Ts, Ds) end; -simplify(#b_set{op={bif,size},args=[Term]}=I, Ts) -> +simplify(#b_set{op={bif,size},args=[Term]}=I, Ts, Ds) -> case normalized_type(Term, Ts) of #t_tuple{} -> simplify(I#b_set{op={bif,tuple_size}}, Ts); @@ -1081,49 +1129,57 @@ simplify(#b_set{op={bif,size},args=[Term]}=I, Ts) -> %% If the bitstring is a binary (the size in bits is %% evenly divisibly by 8), byte_size/1 gives %% the same result as size/1. - simplify(I#b_set{op={bif,byte_size}}, Ts); + simplify(I#b_set{op={bif,byte_size}}, Ts, Ds); _ -> - eval_bif(I, Ts) + eval_bif(I, Ts, Ds) end; -simplify(#b_set{op={bif,tuple_size},args=[Term]}=I, Ts) -> +simplify(#b_set{op={bif,tuple_size},args=[Term]}=I, Ts, _Ds) -> case normalized_type(Term, Ts) of #t_tuple{size=Size,exact=true} -> #b_literal{val=Size}; _ -> I end; -simplify(#b_set{op={bif,is_map_key},args=[Key,Map]}=I, Ts) -> +simplify(#b_set{op={bif,is_map_key},args=[Key,Map]}=I, Ts, _Ds) -> case normalized_type(Map, Ts) of #t_map{} -> I#b_set{op=has_map_field,args=[Map,Key]}; _ -> I end; -simplify(#b_set{op={bif,Op0},args=Args}=I, Ts) when Op0 =:= '=='; - Op0 =:= '/=' -> - Types = normalized_types(Args, Ts), - EqEq0 = case {beam_types:meet(Types),beam_types:join(Types)} of - {none,any} -> true; - {#t_integer{},#t_integer{}} -> true; - {#t_float{},#t_float{}} -> true; - {#t_bitstring{},_} -> true; - {#t_atom{},_} -> true; - {_,_} -> false - end, - EqEq = EqEq0 orelse any_non_numeric_argument(Args, Ts), +simplify(#b_set{op={bif,Op0},args=[A,B]}=I, Ts, Ds) when Op0 =:= '=='; + Op0 =:= '/=' -> + EqEq = case {number_type(A, Ts),number_type(B, Ts)} of + {none,_} -> + %% The LHS does not contain any numbers. A strict + %% test is always safe. + true; + {_,none} -> + %% The RHS does not contain any numbers. A strict + %% test is always safe. + true; + {#t_integer{},#t_integer{}} -> + %% Both side contain integers but no floats. + true; + {#t_float{},#t_float{}} -> + %% Both side contain floats but no integers. + true; + {_,_} -> + false + end, case EqEq of true -> Op = case Op0 of '==' -> '=:='; '/=' -> '=/=' end, - simplify(I#b_set{op={bif,Op}}, Ts); + simplify(I#b_set{op={bif,Op}}, Ts, Ds); false -> - eval_bif(I, Ts) + eval_bif(I, Ts, Ds) end; -simplify(#b_set{op={bif,'=:='},args=[Same,Same]}, _Ts) -> +simplify(#b_set{op={bif,'=:='},args=[Same,Same]}, _Ts, _Ds) -> #b_literal{val=true}; -simplify(#b_set{op={bif,'=:='},args=[LHS,RHS]}=I, Ts) -> +simplify(#b_set{op={bif,'=:='},args=[LHS,RHS]}=I, Ts, Ds) -> LType = concrete_type(LHS, Ts), RType = concrete_type(RHS, Ts), case beam_types:meet(LType, RType) of @@ -1145,12 +1201,12 @@ simplify(#b_set{op={bif,'=:='},args=[LHS,RHS]}=I, Ts) -> %% comparison operator (such as >=) that can be %% translated to test instruction, this %% optimization will eliminate one instruction. - simplify(I#b_set{op={bif,'not'},args=[LHS]}, Ts); + simplify(I#b_set{op={bif,'not'},args=[LHS]}, Ts, Ds); {_,_} -> - eval_bif(I, Ts) + eval_bif(I, Ts, Ds) end end; -simplify(#b_set{op={bif,is_list},args=[Src]}=I0, Ts) -> +simplify(#b_set{op={bif,is_list},args=[Src]}=I0, Ts, Ds) -> case concrete_type(Src, Ts) of #t_union{list=#t_cons{}} -> I = I0#b_set{op=is_nonempty_list,args=[Src]}, @@ -1161,17 +1217,26 @@ simplify(#b_set{op={bif,is_list},args=[Src]}=I0, Ts) -> #t_union{list=nil} -> I0#b_set{op={bif,'=:='},args=[Src,#b_literal{val=[]}]}; _ -> - eval_bif(I0, Ts) + eval_bif(I0, Ts, Ds) end; -simplify(#b_set{op={bif,Op},args=Args}=I, Ts) -> +simplify(#b_set{op={bif,Op},args=[Term]}=I, Ts, _Ds) + when Op =:= trunc; Op =:= round; Op =:= ceil; Op =:= floor -> + case normalized_type(Term, Ts) of + #t_integer{} -> Term; + _ -> I + end; +simplify(#b_set{op={bif,Op},args=Args}=I, Ts, Ds) -> Types = normalized_types(Args, Ts), case is_float_op(Op, Types) of false -> - eval_bif(I, Ts); + eval_bif(I, Ts, Ds); true -> AnnoArgs = [anno_float_arg(A) || A <- Types], - eval_bif(beam_ssa:add_anno(float_op, AnnoArgs, I), Ts) + eval_bif(beam_ssa:add_anno(float_op, AnnoArgs, I), Ts, Ds) end; +simplify(I, Ts, _Ds) -> + simplify(I, Ts). + simplify(#b_set{op=bs_extract,args=[Ctx]}=I, Ts) -> case concrete_type(Ctx, Ts) of #t_bitstring{} -> @@ -1306,6 +1371,26 @@ simplify(#b_set{op=peek_message,args=[#b_literal{val=Val}]}=I, _Ts) -> simplify(#b_set{op=recv_marker_clear,args=[#b_literal{}]}, _Ts) -> %% Not a receive marker: see the 'peek_message' case. #b_literal{val=none}; +simplify(#b_set{op=update_tuple,args=[Src | Updates]}=I, Ts) -> + case {normalized_type(Src, Ts), update_tuple_highest_index(Updates, -1)} of + {#t_tuple{exact=true,size=Size}, Highest} when Highest =< Size, + Size < 512 -> + Args = [#b_literal{val=reuse}, + #b_literal{val=Size}, + Src | Updates], + simplify(I#b_set{op=update_record,args=Args}, Ts); + {_, _} -> + I + end; +simplify(#b_set{op=update_record,args=[Hint0, Size, Src | Updates0]}=I, Ts) -> + case simplify_update_record(Src, Hint0, Updates0, Ts) of + {changed, _, []} -> + Src; + {changed, Hint, [_|_]=Updates} -> + I#b_set{op=update_record,args=[Hint, Size, Src | Updates]}; + unchanged -> + I + end; simplify(I, _Ts) -> I. @@ -1345,14 +1430,14 @@ will_succeed_1(#b_set{op=bs_start_match,args=[_, Arg]}, _Src, Ts) -> end; will_succeed_1(#b_set{op={bif,Bif},args=BifArgs}, _Src, Ts) -> - ArgTypes = normalized_types(BifArgs, Ts), + ArgTypes = concrete_types(BifArgs, Ts), beam_call_types:will_succeed(erlang, Bif, ArgTypes); will_succeed_1(#b_set{op=call, args=[#b_remote{mod=#b_literal{val=Mod}, name=#b_literal{val=Func}} | CallArgs]}, _Src, Ts) -> - ArgTypes = normalized_types(CallArgs, Ts), + ArgTypes = concrete_types(CallArgs, Ts), beam_call_types:will_succeed(Mod, Func, ArgTypes); will_succeed_1(#b_set{op=get_hd}, _Src, _Ts) -> @@ -1363,8 +1448,14 @@ will_succeed_1(#b_set{op=has_map_field}, _Src, _Ts) -> yes; will_succeed_1(#b_set{op=get_tuple_element}, _Src, _Ts) -> yes; -will_succeed_1(#b_set{op=put_tuple}, _Src, _Ts) -> +will_succeed_1(#b_set{op=update_tuple,args=[Tuple | Updates]}, _Src, Ts) -> + TupleType = concrete_type(Tuple, Ts), + HighestIndex = update_tuple_highest_index(Updates, -1), + Args = [beam_types:make_integer(HighestIndex), TupleType, any], + beam_call_types:will_succeed(erlang, setelement, Args); +will_succeed_1(#b_set{op=update_record}, _Src, _Ts) -> yes; + will_succeed_1(#b_set{op=bs_create_bin}, _Src, _Ts) -> %% Intentionally don't try to determine whether construction will %% fail. Construction is unlikely to fail, and if it fails, the @@ -1401,6 +1492,46 @@ will_succeed_1(#b_set{op=wait_timeout}, _Src, _Ts) -> will_succeed_1(#b_set{}, _Src, _Ts) -> 'maybe'. +simplify_update_record(Src, Hint0, Updates, Ts) -> + case sur_1(Updates, concrete_type(Src, Ts), Ts, Hint0, []) of + {Hint0, []} -> + unchanged; + {Hint, Skipped} -> + {changed, Hint, sur_skip(Updates, Skipped)} + end. + +sur_1([Index, Arg | Updates], RecordType, Ts, Hint, Skipped) -> + FieldType = concrete_type(Arg, Ts), + IndexType = concrete_type(Index, Ts), + Singleton = beam_types:is_singleton_type(FieldType), + case beam_call_types:types(erlang, element, [IndexType, RecordType]) of + {FieldType, _, _} when Singleton -> + %% We can skip setting fields when we KNOW that they already have + %% the value we're trying to set. + sur_1(Updates, RecordType, Ts, Hint, Skipped ++ [Index]); + {ElementType, _, _} -> + case beam_types:meet(FieldType, ElementType) of + none -> + %% The element at the index can never have the same value + %% as the value we're trying to set. Tell the instruction + %% not to bother checking whether it's possible to reuse + %% the source. + sur_1(Updates, RecordType, Ts, + #b_literal{val=copy}, Skipped); + _ -> + sur_1(Updates, RecordType, Ts, Hint, Skipped) + end + end; +sur_1([], _RecordType, _Ts, Hint, Skipped) -> + {Hint, Skipped}. + +sur_skip([Index, _Arg | Updates], [Index | Skipped]) -> + sur_skip(Updates, Skipped); +sur_skip([Index, Arg | Updates], Skipped) -> + [Index, Arg | sur_skip(Updates, Skipped)]; +sur_skip([], []) -> + []. + simplify_is_record(I, #t_tuple{exact=Exact, size=Size, elements=Es}, @@ -1489,17 +1620,6 @@ simplify_remote_call(erlang, throw, [Term], Ts, I) -> beam_ssa:add_anno(thrown_type, Type, I); simplify_remote_call(erlang, '++', [#b_literal{val=[]},Tl], _Ts, _I) -> Tl; -simplify_remote_call(erlang, setelement, - [#b_literal{val=Pos}, - #b_literal{val=Tuple}, - #b_var{}=Value], _Ts, I) - when is_integer(Pos), 1 =< Pos, Pos =< tuple_size(Tuple) -> - %% Position is a literal integer and the shape of the - %% tuple is known. - Els0 = [#b_literal{val=El} || El <- tuple_to_list(Tuple)], - {Bef,[_|Aft]} = split(Pos - 1, Els0), - Els = Bef ++ [Value|Aft], - I#b_set{op=put_tuple,args=Els}; simplify_remote_call(Mod, Name, Args, _Ts, I) -> case erl_bifs:is_pure(Mod, Name, length(Args)) of true -> @@ -1533,51 +1653,58 @@ simplify_pure_call(Mod, Name, Args0, I) -> end end. -any_non_numeric_argument([#b_literal{val=Lit}|_], _Ts) -> - is_non_numeric(Lit); -any_non_numeric_argument([#b_var{}=V|T], Ts) -> - is_non_numeric_type(concrete_type(V, Ts)) orelse - any_non_numeric_argument(T, Ts); -any_non_numeric_argument([], _Ts) -> false. - -is_non_numeric([H|T]) -> - is_non_numeric(H) andalso is_non_numeric(T); -is_non_numeric(Tuple) when is_tuple(Tuple) -> - is_non_numeric_tuple(Tuple, tuple_size(Tuple)); -is_non_numeric(Map) when is_map(Map) -> - %% Starting from OTP 18, map keys are compared using `=:=`. - %% Therefore, we only need to check that the values in the map are - %% non-numeric. (Support for compiling BEAM files for OTP releases - %% older than OTP 18 has been dropped.) - is_non_numeric(maps:values(Map)); -is_non_numeric(Num) when is_number(Num) -> - false; -is_non_numeric(_) -> true. - -is_non_numeric_tuple(Tuple, El) when El >= 1 -> - is_non_numeric(element(El, Tuple)) andalso - is_non_numeric_tuple(Tuple, El-1); -is_non_numeric_tuple(_Tuple, 0) -> true. - -is_non_numeric_type(#t_atom{}) -> true; -is_non_numeric_type(#t_bitstring{}) -> true; -is_non_numeric_type(#t_cons{type=Type,terminator=Terminator}) -> - is_non_numeric_type(Type) andalso is_non_numeric_type(Terminator); -is_non_numeric_type(#t_list{type=Type,terminator=Terminator}) -> - is_non_numeric_type(Type) andalso is_non_numeric_type(Terminator); -is_non_numeric_type(#t_map{super_value=Value}) -> - is_non_numeric_type(Value); -is_non_numeric_type(nil) -> true; -is_non_numeric_type(#t_tuple{size=Size,exact=true,elements=Types}) - when map_size(Types) =:= Size -> - is_non_numeric_tuple_type(Size, Types); -is_non_numeric_type(_) -> false. - -is_non_numeric_tuple_type(0, _Types) -> - true; -is_non_numeric_tuple_type(Pos, Types) -> - is_non_numeric_type(map_get(Pos, Types)) andalso - is_non_numeric_tuple_type(Pos - 1, Types). +number_type(V, Ts) -> + number_type_1(concrete_type(V, Ts)). + +number_type_1(any) -> + #t_number{}; +number_type_1(Type) -> + N = beam_types:meet(Type, #t_number{}), + + List = case beam_types:meet(Type, #t_list{}) of + #t_cons{type=Head,terminator=Tail} -> + beam_types:join(number_type_1(Head), number_type_1(Tail)); + #t_list{type=Head,terminator=Tail} -> + beam_types:join(number_type_1(Head), number_type_1(Tail)); + nil -> + none; + none -> + none + end, + + Tup = case beam_types:meet(Type, #t_tuple{}) of + #t_tuple{size=Size,exact=true,elements=ElemTypes} + when map_size(ElemTypes) =:= Size -> + beam_types:join([number_type_1(ET) || + ET <- maps:values(ElemTypes)] ++ [none]); + #t_tuple{} -> + #t_number{}; + #t_union{} -> + #t_number{}; + none -> + none + end, + + Map = case beam_types:meet(Type, #t_map{}) of + #t_map{super_value=MapValue} -> + %% Starting from OTP 18, map keys are compared using + %% `=:=`. Therefore, we only need to use the values + %% in the map. + MapValue; + none -> + none + end, + + Fun = case beam_types:meet(Type, #t_fun{}) of + #t_fun{} -> + %% The environment could contain a number. We don't + %% keep track of the environment in #t_fun{}. + #t_number{}; + none -> + none + end, + + beam_types:join([N,List,Tup,Map,Fun]). make_literal_list(Args) -> make_literal_list(Args, []). @@ -1595,7 +1722,7 @@ is_safe_bool_op([LHS, RHS], Ts) -> beam_types:is_boolean_type(LType) andalso beam_types:is_boolean_type(RType). -eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts) -> +eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts, Ds) -> Arity = length(Args), case erl_bifs:is_pure(erlang, Bif, Arity) of false -> @@ -1603,7 +1730,7 @@ eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts) -> true -> case make_literal_list(Args) of none -> - eval_bif_1(I, Bif, concrete_types(Args, Ts)); + eval_bif_1(I, Bif, Ts, Ds); LitArgs -> try apply(erlang, Bif, LitArgs) of Val -> #b_literal{val=Val} @@ -1614,8 +1741,8 @@ eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts) -> end end. -eval_bif_1(I, Op, Types) -> - case Types of +eval_bif_1(#b_set{args=Args}=I, Op, Ts, Ds) -> + case concrete_types(Args, Ts) of [#t_integer{},#t_integer{elements={0,0}}] when Op =:= 'bor'; Op =:= 'bxor' -> #b_set{args=[Result,_]} = I, Result; @@ -1637,10 +1764,30 @@ eval_bif_1(I, Op, Types) -> false -> I end; + [#t_integer{},#t_integer{}] -> + reassociate(I, Ts, Ds); _ -> I end. +reassociate(#b_set{op={bif,OpB},args=[#b_var{}=V0,#b_literal{val=B0}]}=I, _Ts, Ds) + when OpB =:= '+'; OpB =:= '-' -> + case map_get(V0, Ds) of + #b_set{op={bif,OpA},args=[#b_var{}=V,#b_literal{val=A0}]} + when OpA =:= '+'; OpA =:= '-' -> + A = erlang:OpA(A0), + B = erlang:OpB(B0), + case A + B of + Combined when Combined < 0 -> + I#b_set{op={bif,'-'},args=[V,#b_literal{val=-Combined}]}; + Combined when Combined >= 0 -> + I#b_set{op={bif,'+'},args=[V,#b_literal{val=Combined}]} + end; + #b_set{} -> + I + end; +reassociate(I, _Ts, _Ds) -> I. + simplify_args(Args, Ts, Sub) -> [simplify_arg(Arg, Ts, Sub) || Arg <- Args]. @@ -1904,7 +2051,7 @@ update_types(#b_set{op=Op,dst=Dst,anno=Anno,args=Args}, Ts, Ds) -> Ts#{ Dst => T }. type({bif,Bif}, Args, _Anno, Ts, _Ds) -> - ArgTypes = normalized_types(Args, Ts), + ArgTypes = concrete_types(Args, Ts), case beam_call_types:types(erlang, Bif, ArgTypes) of {any, _, _} -> case {Bif, Args} of @@ -1917,11 +2064,12 @@ type({bif,Bif}, Args, _Anno, Ts, _Ds) -> {RetType, _, _} -> RetType end; -type(bs_create_bin, _Args, _Anno, _Ts, _Ds) -> - #t_bitstring{}; -type(bs_extract, [Ctx], _Anno, _Ts, Ds) -> +type(bs_create_bin, Args, _Anno, Ts, _Ds) -> + SizeUnit = bs_size_unit(Args, Ts), + #t_bitstring{size_unit=SizeUnit}; +type(bs_extract, [Ctx], _Anno, Ts, Ds) -> #b_set{op=bs_match,args=Args} = map_get(Ctx, Ds), - bs_match_type(Args); + bs_match_type(Args, Ts); type(bs_start_match, [_, Src], _Anno, Ts, _Ds) -> case beam_types:meet(#t_bs_matchable{}, concrete_type(Src, Ts)) of none -> @@ -1956,7 +2104,7 @@ type(bs_get_tail, [Ctx], _Anno, Ts, _Ds) -> type(call, [#b_remote{mod=#b_literal{val=Mod}, name=#b_literal{val=Name}}|Args], _Anno, Ts, _Ds) when is_atom(Mod), is_atom(Name) -> - ArgTypes = normalized_types(Args, Ts), + ArgTypes = concrete_types(Args, Ts), {RetType, _, _} = beam_call_types:types(Mod, Name, ArgTypes), RetType; type(call, [#b_remote{mod=Mod,name=Name} | _Args], _Anno, Ts, _Ds) -> @@ -2012,8 +2160,10 @@ type(get_tl, [Src], _Anno, Ts, _Ds) -> SrcType = #t_cons{} = normalized_type(Src, Ts), %Assertion. {RetType, _, _} = beam_call_types:types(erlang, tl, [SrcType]), RetType; -type(get_map_element, [_, _]=Args0, _Anno, Ts, _Ds) -> - [#t_map{}=Map, Key] = normalized_types(Args0, Ts), %Assertion. +type(get_map_element, [Map0, Key0], _Anno, Ts, _Ds) -> + Map = concrete_type(Map0, Ts), + Key = concrete_type(Key0, Ts), + %% Note the reversed argument order! {RetType, _, _} = beam_call_types:types(erlang, map_get, [Key, Map]), RetType; type(get_tuple_element, [Tuple, Offset], _Anno, _Ts, _Ds) -> @@ -2026,9 +2176,12 @@ type(get_tuple_element, [Tuple, Offset], _Anno, _Ts, _Ds) -> true = Index =< Size, %Assertion. beam_types:get_tuple_element(Index, Es) end; -type(has_map_field, [_, _]=Args0, _Anno, Ts, _Ds) -> - [#t_map{}=Map, Key] = normalized_types(Args0, Ts), %Assertion. +type(has_map_field, [Map0, Key0], _Anno, Ts, _Ds) -> + Map = concrete_type(Map0, Ts), + Key = concrete_type(Key0, Ts), + %% Note the reversed argument order! {RetType, _, _} = beam_call_types:types(erlang, is_map_key, [Key, Map]), + true = none =/= RetType, %Assertion. RetType; type(is_nonempty_list, [_], _Anno, _Ts, _Ds) -> beam_types:make_boolean(); @@ -2071,12 +2224,33 @@ type(raw_raise, [Class, _, _], _Anno, Ts, _Ds) -> end; type(resume, [_, _], _Anno, _Ts, _Ds) -> none; +type(update_record, [_Hint, _Size, Src | Updates], _Anno, Ts, _Ds) -> + update_tuple_type(Updates, Src, Ts); +type(update_tuple, [Src | Updates], _Anno, Ts, _Ds) -> + update_tuple_type(Updates, Src, Ts); type(wait_timeout, [#b_literal{val=infinity}], _Anno, _Ts, _Ds) -> %% Waits forever, never reaching the 'after' block. beam_types:make_atom(false); +type(bs_init_writable, [_Size], _, _, _) -> + beam_types:make_type_from_value(<<>>); type(_, _, _, _, _) -> any. +update_tuple_type([_|_]=Updates0, Src, Ts) -> + Updates = update_tuple_type_1(Updates0, Ts), + beam_types:update_tuple(concrete_type(Src, Ts), Updates). + +update_tuple_type_1([#b_literal{val=Index}, Value | Updates], Ts) -> + [{Index, concrete_type(Value, Ts)} | update_tuple_type_1(Updates, Ts)]; +update_tuple_type_1([], _Ts) -> + []. + +update_tuple_highest_index([#b_literal{val=Index}, _Value | Updates], Acc) -> + update_tuple_highest_index(Updates, max(Index, Acc)); +update_tuple_highest_index([], Acc) -> + true = Acc >= 1, %Assertion. + Acc. + join_tuple_elements(Tuple) -> join_tuple_elements(tuple_size(Tuple), Tuple, none). @@ -2088,16 +2262,69 @@ join_tuple_elements(I, Tuple, Type0) -> join_tuple_elements(I - 1, Tuple, Type). put_map_type(Map, Ss, Ts) -> - pmt_1(Ss, Ts, normalized_type(Map, Ts)). + pmt_1(Ss, Ts, concrete_type(Map, Ts)). pmt_1([Key0, Value0 | Ss], Ts, Acc0) -> - Key = normalized_type(Key0, Ts), - Value = normalized_type(Value0, Ts), + Key = concrete_type(Key0, Ts), + Value = concrete_type(Value0, Ts), {Acc, _, _} = beam_call_types:types(maps, put, [Key, Value, Acc0]), pmt_1(Ss, Ts, Acc); pmt_1([], _Ts, Acc) -> Acc. +bs_size_unit(Args, Ts) -> + #t_bitstring{size_unit=Unit0} = beam_types:make_type_from_value(<<>>), + Unit = bs_size_unit(Args, Ts, 0, Unit0), + safe_gcd(Unit0, Unit). + +bs_size_unit([#b_literal{val=Type},#b_literal{val=[U1|_]},Value,SizeTerm|Args], + Ts, Size0, U0) -> + case {Type,Value,SizeTerm} of + {_,_,#b_literal{val=all}} -> + case concrete_type(Value, Ts) of + #t_bitstring{size_unit=U2} -> + U = safe_gcd(U0, max(U1, U2)), + bs_size_unit(Args, Ts, none, U); + _ -> + U = safe_gcd(U0, U1), + bs_size_unit(Args, Ts, none, U) + end; + {utf8,_,_} -> + U = gcd(U0, 8), + bs_size_unit(Args, Ts, none, U); + {utf16,_,_} -> + U = gcd(U0, 16), + bs_size_unit(Args, Ts, none, U); + {utf32,_,_} -> + U = gcd(U0, 32), + bs_size_unit(Args, Ts, none, U); + {_,_,_} -> + case concrete_type(SizeTerm, Ts) of + #t_integer{elements={Size1, Size1}} + when is_integer(Size1), is_integer(U1), Size1 >= 0 -> + EffectiveSize = Size1 * U1, + U = safe_gcd(U0, EffectiveSize), + Size = safe_add(Size0, EffectiveSize), + bs_size_unit(Args, Ts, Size, U); + _ when is_integer(U1) -> + U = safe_gcd(U0, U1), + bs_size_unit(Args, Ts, none, U); + _ -> + bs_size_unit(Args, Ts, none, 1) + end + end; +bs_size_unit([], _Ts, none, Unit) -> + Unit; +bs_size_unit([], _Ts, Size, _Unit) when is_integer(Size) -> + Size. + +safe_gcd(0, Other) -> Other; +safe_gcd(Other, 0) -> Other; +safe_gcd(A, B) -> gcd(A, B). + +safe_add(none, _) -> none; +safe_add(A, B) -> A + B. + %% We seldom know how far a match operation may advance, but we can often tell %% which increment it will advance by. bs_match_stride([#b_literal{val=Type} | Args], Ts) -> @@ -2123,21 +2350,21 @@ bs_match_stride(_, _, _) -> -define(UNICODE_MAX, (16#10FFFF)). -bs_match_type([#b_literal{val=Type}|Args]) -> - bs_match_type(Type, Args). +bs_match_type([#b_literal{val=Type}|Args], Ts) -> + bs_match_type(Type, Args, Ts). -bs_match_type(binary, Args) -> +bs_match_type(binary, Args, _Ts) -> [_,_,_,#b_literal{val=U}] = Args, #t_bitstring{size_unit=U}; -bs_match_type(float, _) -> +bs_match_type(float, _, _Ts) -> #t_float{}; -bs_match_type(integer, Args) -> - case Args of - [_, - #b_literal{val=Flags}, - #b_literal{val=Size}, - #b_literal{val=Unit}] when Size * Unit < 64 -> - NumBits = Size * Unit, +bs_match_type(integer, Args, Ts) -> + [_,#b_literal{val=Flags},Size,#b_literal{val=Unit}] = Args, + SizeType = beam_types:meet(concrete_type(Size, Ts), #t_integer{}), + case SizeType of + #t_integer{elements={_,SizeMax}} + when is_integer(SizeMax), SizeMax >= 0, SizeMax * Unit < 64 -> + NumBits = SizeMax * Unit, Max = (1 bsl NumBits) - 1, case member(unsigned, Flags) of true -> @@ -2146,18 +2373,14 @@ bs_match_type(integer, Args) -> Min = -(Max + 1), beam_types:make_integer(Min, Max) end; - [_|_] -> + _ -> #t_integer{} end; -bs_match_type(skip, _) -> - any; -bs_match_type(string, _) -> - any; -bs_match_type(utf8, _) -> +bs_match_type(utf8, _, _) -> beam_types:make_integer(0, ?UNICODE_MAX); -bs_match_type(utf16, _) -> +bs_match_type(utf16, _, _) -> beam_types:make_integer(0, ?UNICODE_MAX); -bs_match_type(utf32, _) -> +bs_match_type(utf32, _, _) -> beam_types:make_integer(0, ?UNICODE_MAX). normalized_types(Values, Ts) -> @@ -2212,12 +2435,7 @@ concrete_type(#b_var{}=Var, Ts) -> %% subtraction for L will be added to FailTypes. infer_types_br(#b_var{}=V, Ts, IsTempVar, Ds) -> - #{V:=#b_set{op=Op,args=Args}} = Ds, - - {PosTypes, NegTypes} = infer_type(Op, Args, Ts, Ds), - - SuccTs0 = meet_types(PosTypes, Ts), - FailTs0 = subtract_types(NegTypes, Ts), + {SuccTs0, FailTs0} = infer_types_br_1(V, Ts, Ds), case IsTempVar of true -> @@ -2234,6 +2452,156 @@ infer_types_br(#b_var{}=V, Ts, IsTempVar, Ds) -> {SuccTs, FailTs} end. +infer_types_br_1(V, Ts, Ds) -> + #{V:=#b_set{op=Op0,args=Args}} = Ds, + + case inv_relop(Op0) of + none -> + %% Not a relational operator. + {PosTypes, NegTypes} = infer_type(Op0, Args, Ts, Ds), + SuccTs = meet_types(PosTypes, Ts), + FailTs = infer_subtract_types(NegTypes, Ts), + {SuccTs, FailTs}; + InvOp -> + %% This is an relational operator. + {bif,Op} = Op0, + + %% Infer the types for both sides of operator succceding. + Types = concrete_types(Args, Ts), + TrueTypes0 = infer_relop(Op, Args, Types, Ds), + TrueTypes = meet_types(TrueTypes0, Ts), + + %% Infer the types for both sides of operator failing. + FalseTypes0 = infer_relop(InvOp, Args, Types, Ds), + FalseTypes = meet_types(FalseTypes0, Ts), + + {TrueTypes, FalseTypes} + end. + +infer_relop('=:=', [LHS,RHS], [LType,RType], Ds) -> + EqTypes = infer_eq_type(map_get(LHS, Ds), RType), + + %% As an example, assume that L1 is known to be 'list', and L2 is + %% known to be 'cons'. Then if 'L1 =:= L2' evaluates to 'true', it + %% can be inferred that L1 is 'cons' (the meet of 'cons' and + %% 'list'). + Type = beam_types:meet(LType, RType), + Types = [{LHS,Type},{RHS,Type}], + [{V,T} || {#b_var{}=V,T} <- Types, T =/= any] ++ EqTypes; +infer_relop(Op, Args, Types, _Ds) -> + infer_relop(Op, Args, Types). + +infer_relop('=/=', [LHS,RHS], [LType,RType]) -> + %% We must be careful with types inferred from '=/='. + %% + %% For example, if we have L =/= [a], we must not subtract 'cons' + %% from the type of L. In general, subtraction is only safe if + %% the type being subtracted is singled-valued, for example if it + %% is [] or the the atom 'true'. + %% + %% Note that we subtract the left-hand type from the right-hand + %% value and vice versa. We must not subtract the meet of the two + %% as it may be too specific. See beam_type_SUITE:type_subtraction/1 + %% for details. + [{V,beam_types:subtract(ThisType, OtherType)} || + {#b_var{}=V, ThisType, OtherType} <- + [{RHS, RType, LType}, {LHS, LType, RType}], + beam_types:is_singleton_type(OtherType)]; +infer_relop(Op, [Arg1,Arg2], Types0) -> + case infer_relop(Op, Types0) of + any -> + %% Both operands lacked ranges. + []; + {NewType1,NewType2} -> + Types = [{Arg1,NewType1},{Arg2,NewType2}], + [{V,T} || {#b_var{}=V,T} <- Types, + T =/= any] + end. + +infer_relop(Op, [#t_integer{elements=R1}, + #t_integer{elements=R2}]) -> + case beam_bounds:infer_relop_types(Op, R1, R2) of + any -> + any; + {NewR1,NewR2} -> + {#t_integer{elements=NewR1}, + #t_integer{elements=NewR2}} + end; +infer_relop(Op0, [Type1,Type2]) -> + Op = case Op0 of + '<' -> '=<'; + '>' -> '>='; + _ -> Op0 + end, + case {infer_get_range(Type1),infer_get_range(Type2)} of + {unknown,unknown} -> + any; + {unknown,_}=R -> + infer_relop_any(Op, R); + {_,unknown}=R -> + infer_relop_any(Op, R); + {R1,R2} -> + %% Both operands are numeric types. + case beam_bounds:infer_relop_types(Op, R1, R2) of + any -> + any; + {NewR1,NewR2} -> + {#t_number{elements=NewR1}, + #t_number{elements=NewR2}} + end + end. + +infer_relop_any('=<', {unknown,any}) -> + %% Since numbers are smaller than any other terms, the LHS must be + %% a number if operator '=<' evaluates to true. + {#t_number{}, any}; +infer_relop_any('=<', {unknown,{_,Max}}) -> + {make_number({'-inf',Max}), any}; +infer_relop_any('>=', {any,unknown}) -> + {any, #t_number{}}; +infer_relop_any('>=', {{_,Max},unknown}) -> + {any, make_number({'-inf',Max})}; +infer_relop_any('>=', {unknown,{Min,_}}) when is_integer(Min) -> + %% LHS can be any term, including number, but we can figure out + %% the minimal possible number. + N = #t_number{elements={'-inf',Min}}, + {beam_types:subtract(any, N), any}; +infer_relop_any('=<', {{Min,_},unknown}) when is_integer(Min) -> + N = #t_number{elements={'-inf',Min}}, + {any, beam_types:subtract(any, N)}; +infer_relop_any(_Op, _R) -> + any. + +make_number({'-inf','+inf'}) -> + #t_number{}; +make_number({_,_}=R) -> + #t_number{elements=R}. + +inv_relop({bif,Op}) -> inv_relop_1(Op); +inv_relop(_) -> none. + +inv_relop_1('<') -> '>='; +inv_relop_1('=<') -> '>'; +inv_relop_1('>=') -> '<'; +inv_relop_1('>') -> '=<'; +inv_relop_1('=:=') -> '=/='; +inv_relop_1('=/=') -> '=:='; +inv_relop_1(_) -> none. + +infer_get_range(#t_integer{elements=R}) -> R; +infer_get_range(#t_number{elements=R}) -> R; +infer_get_range(_) -> unknown. + +infer_subtract_types([{V,T0}|Vs], Ts) -> + T1 = concrete_type(V, Ts), + case beam_types:subtract(T1, T0) of + none -> none; + T1 -> infer_subtract_types(Vs, Ts); + T -> infer_subtract_types(Vs, Ts#{V := T}) + end; +infer_subtract_types([], Ts) -> + Ts. + infer_br_value(_V, _Bool, none) -> none; infer_br_value(V, Bool, NewTs) -> @@ -2247,7 +2615,9 @@ infer_br_value(V, Bool, NewTs) -> end. infer_types_switch(V, Lit, Ts0, IsTempVar, Ds) -> - {PosTypes, _} = infer_type({bif,'=:='}, [V, Lit], Ts0, Ds), + Args = [V,Lit], + Types = concrete_types(Args, Ts0), + PosTypes = infer_relop('=:=', Args, Types, Ds), Ts = meet_types(PosTypes, Ts0), case IsTempVar of true -> ts_remove_var(V, Ts); @@ -2312,7 +2682,7 @@ infer_type({bif,is_map}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_map{}}, {[T], [T]}; infer_type({bif,is_number}, [#b_var{}=Arg], _Ts, _Ds) -> - T = {Arg, number}, + T = {Arg, #t_number{}}, {[T], [T]}; infer_type({bif,is_pid}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, pid}, @@ -2326,52 +2696,11 @@ infer_type({bif,is_reference}, [#b_var{}=Arg], _Ts, _Ds) -> infer_type({bif,is_tuple}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_tuple{}}, {[T], [T]}; -infer_type({bif,'=:='}, [#b_var{}=LHS,#b_var{}=RHS], Ts, _Ds) -> - %% As an example, assume that L1 is known to be 'list', and L2 is - %% known to be 'cons'. Then if 'L1 =:= L2' evaluates to 'true', it can - %% be inferred that L1 is 'cons' (the meet of 'cons' and 'list'). - LType = concrete_type(LHS, Ts), - RType = concrete_type(RHS, Ts), - Type = beam_types:meet(LType, RType), - - PosTypes = [{V,Type} || {V, OrigType} <- [{LHS, LType}, {RHS, RType}], - OrigType =/= Type], - - %% We must be careful with types inferred from '=:='. - %% - %% If we have seen L =:= [a], we know that L is 'cons' if the - %% comparison succeeds. However, if the comparison fails, L could - %% still be 'cons'. Therefore, we must not subtract 'cons' from the - %% previous type of L. - %% - %% However, it is safe to subtract a type inferred from '=:=' if - %% it is single-valued, e.g. if it is [] or the atom 'true'. - %% - %% Note that we subtract the left-hand type from the right-hand - %% value and vice versa. We must not subtract the meet of the two - %% as it may be too specific. See beam_type_SUITE:type_subtraction/1 - %% for details. - NegTypes = [T || {_, OtherType}=T <- [{RHS, LType}, {LHS, RType}], - beam_types:is_singleton_type(OtherType)], - - {PosTypes, NegTypes}; -infer_type({bif,'=:='}, [#b_var{}=Src,#b_literal{}=Lit], Ts, Ds) -> - Def = maps:get(Src, Ds), - LitType = concrete_type(Lit, Ts), - PosTypes = [{Src, LitType} | infer_eq_type(Def, LitType)], - - %% Subtraction is only safe if LitType is single-valued. - NegTypes = case beam_types:is_singleton_type(LitType) of - true -> PosTypes; - false -> [] - end, - - {PosTypes, NegTypes}; infer_type(_Op, _Args, _Ts, _Ds) -> {[], []}. infer_success_type({bif,Op}, Args, Ts, _Ds) -> - ArgTypes = normalized_types(Args, Ts), + ArgTypes = concrete_types(Args, Ts), {_, PosTypes0, CanSubtract} = beam_call_types:types(erlang, Op, ArgTypes), PosTypes = [T || {#b_var{},_}=T <- zip(Args, PosTypes0)], @@ -2458,16 +2787,6 @@ meet_types([{V,T0}|Vs], Ts) -> meet_types([], Ts) -> Ts. -subtract_types([{V,T0}|Vs], Ts) -> - T1 = concrete_type(V, Ts), - case beam_types:subtract(T1, T0) of - none -> none; - T1 -> subtract_types(Vs, Ts); - T -> subtract_types(Vs, Ts#{ V:= T }) - end; -subtract_types([], Ts) -> - Ts. - parallel_join([A | As], [B | Bs]) -> [beam_types:join(A, B) | parallel_join(As, Bs)]; parallel_join([], []) -> diff --git a/lib/compiler/src/beam_trim.erl b/lib/compiler/src/beam_trim.erl index 4fb141251d..b37830d44c 100644 --- a/lib/compiler/src/beam_trim.erl +++ b/lib/compiler/src/beam_trim.erl @@ -278,6 +278,12 @@ remap([{make_fun3,F,Index,OldUniq,Dst0,{list,Env0}}|Is], Remap) -> Dst = remap_arg(Dst0, Remap), I = {make_fun3,F,Index,OldUniq,Dst,{list,Env}}, [I|remap(Is, Remap)]; +remap([{update_record,Hint,Size,Src0,Dst0,{list,Updates0}}|Is], Remap) -> + Updates = remap_args(Updates0, Remap), + Src = remap_arg(Src0, Remap), + Dst = remap_arg(Dst0, Remap), + I = {update_record,Hint,Size,Src,Dst,{list,Updates}}, + [I|remap(Is, Remap)]; remap([{deallocate,N}|Is], {Trim,_}=Remap) -> I = {deallocate,N-Trim}, [I|remap(Is, Remap)]; @@ -527,6 +533,12 @@ do_usage([{make_fun3,_,_,_,Dst,{list,Ss}}|Is], Safe, Regs0, Ns0, Acc) -> Ns = ordsets:union(yregs([Dst]), Ns0), U = {Regs,Ns}, do_usage(Is, Safe, Regs, Ns, [U|Acc]); +do_usage([{update_record,_,_,Src,Dst,{list,Ss}}|Is], Safe, Regs0, Ns0, Acc) -> + Regs1 = ordsets:del_element(Dst, Regs0), + Regs = ordsets:union(Regs1, yregs([Src|Ss])), + Ns = ordsets:union(yregs([Dst]), Ns0), + U = {Regs,Ns}, + do_usage(Is, Safe, Regs, Ns, [U|Acc]); do_usage([{line,_}|Is], Safe, Regs, Ns, Acc) -> U = {Regs,Ns}, do_usage(Is, Safe, Regs, Ns, [U|Acc]); diff --git a/lib/compiler/src/beam_types.erl b/lib/compiler/src/beam_types.erl index 22b02038dc..532135c002 100644 --- a/lib/compiler/src/beam_types.erl +++ b/lib/compiler/src/beam_types.erl @@ -23,7 +23,7 @@ -define(BEAM_TYPES_INTERNAL, true). -include("beam_types.hrl"). --import(lists, [foldl/3, reverse/1, usort/1]). +-import(lists, [foldl/3, mapfoldl/3, reverse/1, usort/1]). -export([meet/1, meet/2, join/1, join/2, subtract/2]). @@ -35,7 +35,9 @@ is_singleton_type/1, normalize/1]). --export([get_tuple_element/2, set_tuple_element/3]). +-export([get_tuple_element/2, + set_tuple_element/3, + update_tuple/2]). -export([make_type_from_value/1]). @@ -61,7 +63,7 @@ N =:= nil). -define(IS_NUMBER_TYPE(N), - N =:= number orelse + is_record(N, t_number) orelse is_record(N, t_float) orelse is_record(N, t_integer)). @@ -277,19 +279,17 @@ join_unions(#t_union{atom=AtomA,list=ListA,number=NumberA, other=lub(OtherA, OtherB)}, shrink_union(Union); join_unions(#t_union{atom=AtomA}=A, #t_atom{}=B) -> - A#t_union{atom=lub(AtomA, B)}; + shrink_union(A#t_union{atom=lub(AtomA, B)}); join_unions(#t_union{list=ListA}=A, B) when ?IS_LIST_TYPE(B) -> - A#t_union{list=lub(ListA, B)}; + shrink_union(A#t_union{list=lub(ListA, B)}); join_unions(#t_union{number=NumberA}=A, B) when ?IS_NUMBER_TYPE(B) -> - A#t_union{number=lub(NumberA, B)}; + shrink_union(A#t_union{number=lub(NumberA, B)}); join_unions(#t_union{tuple_set=TSetA}=A, #t_tuple{}=B) -> Set = join_tuple_sets(TSetA, new_tuple_set(B)), shrink_union(A#t_union{tuple_set=Set}); join_unions(#t_union{other=OtherA}=A, B) -> - case lub(OtherA, B) of - any -> any; - T -> A#t_union{other=T} - end. + T = lub(OtherA, B), + shrink_union(A#t_union{other=T}). join_tuple_sets(A, none) -> A; @@ -334,6 +334,47 @@ jts_records([], [{KeyB, B} | RsB], N, Acc) -> -spec subtract(type(), type()) -> type(). +subtract(any, #t_number{elements={'-inf',Max}}) -> + %% We handle this case specially in order to represent the type + %% Var in Var =< Integer. + #t_union{atom=#t_atom{}, + list=#t_list{}, + number=#t_number{elements={Max,'+inf'}}, + tuple_set=#t_tuple{}, + other=other}; +subtract(any, nil) -> + %% + %% We handle this subtraction mainly for correctness. Consider: + %% + %% foobar(L) when L =/= [], is_list(L), L =/= [], hd(L) -> ok. + %% + %% The type-based optimizations would rewrite the hd/1 BIF call to + %% a get_hd instruction: + %% + %% foobar(L) when L =/= [], is_list(L), L =/= [], get_hd(L) -> ok. + %% + %% The beam_ssa_dead pass would later remove the redundant second + %% test of `L =/= []`: + %% + %% foobar(L) when L =/= [], is_list(L), get_hd(L) -> ok. + %% + %% With that test removed, the type for L in the get_hd instruction + %% would be #t_list{} instead of the required #t_cons{} and that + %% would trigger an assertion. (If the assertion were to be removed, + %% beam_validator would complain instead.) + %% + %% By letting the type subtraction of `any` by `nil` return a + %% union, the optimized code above becomes legal. As an added bonus, + %% the is_list/1 guard test can be replaced with the cheaper + %% is_nonempty_list instruction: + %% + %% foobar(L) when L =/= [], is_nonempty_list(L), get_hd(L) -> ok. + %% + #t_union{atom=#t_atom{}, + list=#t_cons{}, + number=#t_number{}, + tuple_set=#t_tuple{}, + other=other}; subtract(#t_atom{elements=[_|_]=Set0}, #t_atom{elements=[_|_]=Set1}) -> case ordsets:subtract(Set0, Set1) of [] -> none; @@ -358,8 +399,10 @@ subtract(#t_integer{elements={Min, Max}}, #t_integer{elements={N,N}}) -> Max =:= N -> #t_integer{elements={Min, Max - 1}} end; -subtract(number, #t_float{elements=any}) -> #t_integer{}; -subtract(number, #t_integer{elements=any}) -> #t_float{}; +subtract(#t_number{elements=R}, #t_float{elements=any}) -> + integer_from_range(R); +subtract(#t_number{elements=R}, #t_integer{elements=any}) -> + float_from_range(R); %% A list is essentially `#t_cons{} | nil`, so we're left with nil if we %% subtract a cons cell that is more general than the one in the list. @@ -475,7 +518,7 @@ is_boolean_type(_) -> -spec is_numerical_type(type()) -> boolean(). is_numerical_type(#t_integer{}) -> true; -is_numerical_type(number) -> true; +is_numerical_type(#t_number{}) -> true; is_numerical_type(_) -> false. -spec set_tuple_element(Index, Type, Elements) -> Elements when @@ -500,6 +543,58 @@ get_tuple_element(Index, Es) -> #{} -> any end. +%% Helper routine for `update_tuple` / `update_record` instructions, which copy +%% an existing type and updates a few fields. +-spec update_tuple(Type, Updates) -> Tuple when + Type :: type(), + Updates :: [{pos_integer(), type()}, ...], + Tuple :: type(). +update_tuple(#t_union{tuple_set=[_|_]=Set0}, [_|_]=Updates) -> + case Updates of + [{1, _} | _] -> + %% The update overwrites the tag, so we can no longer keep any + %% records apart. Normalize the set before trying again. + update_tuple(normalize_tuple_set(Set0, none), Updates); + [_|_] -> + case update_tuple_set(Set0, Updates) of + [] -> + none; + [_|_]=Set -> + verified_type(shrink_union(#t_union{tuple_set=Set})) + end + end; +update_tuple(#t_union{tuple_set=#t_tuple{}=Tuple}, [_|_]=Updates) -> + update_tuple(Tuple, Updates); +update_tuple(#t_tuple{exact=Exact, + size=Size, + elements=Es0}=Tuple, + [_|_]=Updates) -> + case update_tuple_1(Updates, Size, Es0) of + {MinSize, _Es} when Exact, MinSize > Size -> + none; + {MinSize, Es} -> + verified_normal_type(Tuple#t_tuple{size=MinSize,elements=Es}) + end; +update_tuple(Type, [_|_]=Updates) -> + case meet(Type, #t_tuple{size=1}) of + none -> none; + Tuple -> update_tuple(Tuple, Updates) + end. + +update_tuple_set([{Tag, Record0} | Set], Updates) -> + case update_tuple(Record0, Updates) of + none -> update_tuple_set(Set, Updates); + #t_tuple{}=Record -> [{Tag, Record} | update_tuple_set(Set, Updates)] + end; +update_tuple_set([], _Es) -> + []. + +update_tuple_1([{Index, Type} | Updates], MinSize, Es0) -> + Es = set_tuple_element(Index, Type, Es0), + update_tuple_1(Updates, max(Index, MinSize), Es); +update_tuple_1([], MinSize, Es) -> + {MinSize, Es}. + -spec normalize(type()) -> normal_type(). normalize(#t_union{atom=Atom,list=List,number=Number, tuple_set=Tuples,other=Other}) -> @@ -530,11 +625,9 @@ mtfv_1(A) when is_atom(A) -> mtfv_1(B) when is_bitstring(B) -> case bit_size(B) of 0 -> - %% This is a bit of a hack, but saying that empty binaries have a - %% unit of 8 helps us get rid of is_binary/1 checks. - #t_bitstring{size_unit=8}; + #t_bitstring{size_unit=256}; Size -> - #t_bitstring{size_unit=Size} + #t_bitstring{size_unit=gcd(Size, 256)} end; mtfv_1(F) when is_float(F) -> make_float(F); @@ -757,15 +850,8 @@ glb(#t_cons{type=TypeA,terminator=TermA}, {_, none} -> none; {Type, Term} -> #t_cons{type=Type,terminator=Term} end; -glb(#t_float{}=T, #t_float{elements=any}) -> - T; -glb(#t_float{elements=any}, #t_float{}=T) -> - T; -glb(#t_float{elements={MinA,MaxA}}, #t_float{elements={MinB,MaxB}}) - when MinA >= MinB, MinA =< MaxB; - MinB >= MinA, MinB =< MaxA -> - true = MinA =< MaxA andalso MinB =< MaxB, %Assertion. - #t_float{elements={max(MinA, MinB),min(MaxA, MaxB)}}; +glb(#t_float{elements=R1}, #t_float{elements=R2}) -> + float_from_range(glb_ranges(R1, R2)); glb(#t_fun{arity=SameArity,target=SameTarget,type=TypeA}, #t_fun{arity=SameArity,target=SameTarget,type=TypeB}=T) -> T#t_fun{type=meet(TypeA, TypeB)}; @@ -777,19 +863,12 @@ glb(#t_fun{arity=any}=A, #t_fun{arity=ArityB}=B) when ArityB =/= any-> glb(A#t_fun{arity=ArityB}, B); glb(#t_fun{arity=ArityA}=A, #t_fun{arity=any}=B) when ArityA =/= any -> glb(A, B#t_fun{arity=ArityA}); -glb(#t_integer{elements={_,_}}=T, #t_integer{elements=any}) -> - T; -glb(#t_integer{elements=any}, #t_integer{elements={_,_}}=T) -> - T; -glb(#t_integer{elements={MinA,MaxA}}, #t_integer{elements={MinB,MaxB}}) - when MinA >= MinB, MinA =< MaxB; - MinB >= MinA, MinB =< MaxA -> - true = MinA =< MaxA andalso MinB =< MaxB, %Assertion. - #t_integer{elements={max(MinA, MinB),min(MaxA, MaxB)}}; -glb(#t_integer{}=T, number) -> - T; -glb(#t_float{}=T, number) -> - T; +glb(#t_integer{elements=R1}, #t_integer{elements=R2}) -> + integer_from_range(glb_ranges(R1, R2)); +glb(#t_integer{elements=R1}, #t_number{elements=R2}) -> + integer_from_range(glb_ranges(R1, R2)); +glb(#t_float{elements=R1}, #t_number{elements=R2}) -> + float_from_range(glb_ranges(R1, R2)); glb(#t_list{type=TypeA,terminator=TermA}, #t_list{type=TypeB,terminator=TermB}) -> %% A list is a union of `[type() | _]` and `[]`, so we're left with @@ -805,10 +884,12 @@ glb(#t_list{}, nil) -> nil; glb(nil, #t_list{}) -> nil; -glb(number, #t_integer{}=T) -> - T; -glb(number, #t_float{}=T) -> - T; +glb(#t_number{elements=R1}, #t_number{elements=R2}) -> + number_from_range(glb_ranges(R1, R2)); +glb(#t_number{elements=R1}, #t_integer{elements=R2}) -> + integer_from_range(glb_ranges(R1, R2)); +glb(#t_number{elements=R1}, #t_float{elements=R2}) -> + float_from_range(glb_ranges(R1, R2)); glb(#t_map{super_key=SKeyA,super_value=SValueA}, #t_map{super_key=SKeyB,super_value=SValueB}) -> %% Note the use of meet/2; elements don't need to be normal types. @@ -817,10 +898,34 @@ glb(#t_map{super_key=SKeyA,super_value=SValueA}, #t_map{super_key=SKey,super_value=SValue}; glb(#t_tuple{}=T1, #t_tuple{}=T2) -> glb_tuples(T1, T2); +glb(other, T) -> + case is_other(T) of + true -> T; + false -> none + end; +glb(T, other) -> + glb(other, T); glb(_, _) -> %% Inconsistent types. There will be an exception at runtime. none. +glb_ranges({MinA,MaxA}, {MinB,MaxB}) -> + true = inf_le(MinA, MaxA) andalso inf_le(MinB, MaxB), %Assertion. + case (inf_ge(MinA, MinB) andalso inf_le(MinA, MaxB)) orelse + (inf_ge(MinB, MinA) andalso inf_le(MinB, MaxA)) of + true -> + true = inf_le(MinA, MaxA) andalso inf_le(MinB, MaxB), %Assertion. + {inf_max(MinA, MinB),inf_min(MaxA, MaxB)}; + false -> + none + end; +glb_ranges({MinA,MaxA}, any) -> + {MinA,MaxA}; +glb_ranges(any, {MinB,MaxB}) -> + {MinB,MaxB}; +glb_ranges(_, _) -> + any. + glb_tuples(#t_tuple{size=Sz1,exact=Ex1}, #t_tuple{size=Sz2,exact=Ex2}) when Ex1, Sz1 < Sz2; Ex2, Sz2 < Sz1 -> @@ -918,15 +1023,12 @@ lub(#t_cons{type=TypeA,terminator=TermA}, #t_list{type=join(TypeA,TypeB),terminator=join(TermA, TermB)}; lub(#t_cons{type=Type,terminator=Term}, nil) -> #t_list{type=Type,terminator=Term}; -lub(#t_float{elements={MinA,MaxA}}, - #t_float{elements={MinB,MaxB}}) -> - #t_float{elements={min(MinA,MinB),max(MaxA,MaxB)}}; -lub(#t_float{}, #t_float{}) -> - #t_float{}; -lub(#t_float{}, #t_integer{}) -> - number; -lub(#t_float{}, number) -> - number; +lub(#t_float{elements=R1}, #t_float{elements=R2}) -> + float_from_range(lub_ranges(R1, R2)); +lub(#t_float{elements=R1}, #t_integer{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); +lub(#t_float{elements=R1}, #t_number{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); lub(#t_fun{arity=SameArity,target=SameTarget,type=TypeA}, #t_fun{arity=SameArity,target=SameTarget,type=TypeB}) -> #t_fun{arity=SameArity,target=SameTarget,type=join(TypeA, TypeB)}; @@ -934,15 +1036,12 @@ lub(#t_fun{arity=SameArity,type=TypeA}, #t_fun{arity=SameArity,type=TypeB}) -> #t_fun{arity=SameArity,type=join(TypeA, TypeB)}; lub(#t_fun{type=TypeA}, #t_fun{type=TypeB}) -> #t_fun{type=join(TypeA, TypeB)}; -lub(#t_integer{elements={MinA,MaxA}}, - #t_integer{elements={MinB,MaxB}}) -> - #t_integer{elements={min(MinA,MinB),max(MaxA,MaxB)}}; -lub(#t_integer{}, #t_integer{}) -> - #t_integer{}; -lub(#t_integer{}, #t_float{}) -> - number; -lub(#t_integer{}, number) -> - number; +lub(#t_integer{elements=R1}, #t_integer{elements=R2}) -> + integer_from_range(lub_ranges(R1, R2)); +lub(#t_integer{elements=R1}, #t_float{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); +lub(#t_integer{elements=R1}, #t_number{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); lub(#t_list{type=TypeA,terminator=TermA}, #t_list{type=TypeB,terminator=TermB}) -> #t_list{type=join(TypeA, TypeB),terminator=join(TermA, TermB)}; @@ -954,10 +1053,12 @@ lub(nil, #t_list{}=T) -> T; lub(#t_list{}=T, nil) -> T; -lub(number, #t_integer{}) -> - number; -lub(number, #t_float{}) -> - number; +lub(#t_number{elements=R1}, #t_number{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); +lub(#t_number{elements=R1}, #t_integer{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); +lub(#t_number{elements=R1}, #t_float{elements=R2}) -> + number_from_range(lub_ranges(R1, R2)); lub(#t_map{super_key=SKeyA,super_value=SValueA}, #t_map{super_key=SKeyB,super_value=SValueB}) -> %% Note the use of join/2; elements don't need to be normal types. @@ -973,8 +1074,24 @@ lub(#t_tuple{size=SzA,elements=EsA}, #t_tuple{size=SzB,elements=EsB}) -> Sz = min(SzA, SzB), Es = lub_tuple_elements(Sz, EsA, EsB), #t_tuple{size=Sz,elements=Es}; -lub(_T1, _T2) -> - %%io:format("~p ~p\n", [_T1,_T2]), +lub(T1, T2) -> + %%io:format("~p ~p\n", [T1,T2]), + case is_other(T1) andalso is_other(T2) of + true -> other; + false -> any + end. + +is_other(Type) -> + AnyMinusOther = #t_union{atom=#t_atom{}, + list=#t_list{}, + number=#t_number{}, + tuple_set=#t_tuple{}, + other=none}, + meet(AnyMinusOther, Type) =:= none. + +lub_ranges({MinA,MaxA}, {MinB,MaxB}) -> + {inf_min(MinA, MinB), inf_max(MaxA, MaxB)}; +lub_ranges(_, _) -> any. lub_bs_matchable(UnitA, UnitB) -> @@ -1011,6 +1128,65 @@ gcd(A, B) -> X -> gcd(B, X) end. +%%% +%%% Handling of ranges. +%%% +%%% A range can begin with '-inf' OR end with '+inf'. +%%% +%%% Atoms are greater than all integers. Therefore, we don't +%%% need any special handling of '+inf'. +%%% + +float_from_range(none) -> + none; +float_from_range(any) -> + #t_float{}; +float_from_range({'-inf','+inf'}) -> + #t_float{}; +float_from_range({'-inf',Max}) -> + #t_float{elements={'-inf',float(Max)}}; +float_from_range({Min,'+inf'}) -> + #t_float{elements={float(Min),'+inf'}}; +float_from_range({Min,Max}) -> + #t_float{elements={float(Min),float(Max)}}. + +integer_from_range(none) -> + none; +integer_from_range(any) -> + #t_integer{}; +integer_from_range({'-inf','+inf'}) -> + #t_integer{}; +integer_from_range({'-inf',Max}) -> + #t_integer{elements={'-inf',ceil(Max)}}; +integer_from_range({Min,'+inf'}) -> + #t_integer{elements={floor(Min),'+inf'}}; +integer_from_range({Min,Max}) -> + #t_integer{elements={floor(Min),ceil(Max)}}. + +number_from_range(N) -> + case integer_from_range(N) of + #t_integer{elements=R} -> + #t_number{elements=R}; + none -> + none + end. + +inf_le('-inf', _) -> true; +inf_le(A, B) -> A =< B. + +inf_ge(_, '-inf') -> true; +inf_ge('-inf', _) -> false; +inf_ge(A, B) -> A >= B. + +inf_min(A, B) when A =:= '-inf'; B =:= '-inf' -> '-inf'; +inf_min(A, B) when A =< B -> A; +inf_min(A, B) when A > B -> B. + +inf_max('-inf', B) -> B; +inf_max(A, '-inf') -> A; +inf_max(A, B) when A >= B -> A; +inf_max(A, B) when A < B -> B. + %% record_key(#t_tuple{exact=true,size=Size,elements=#{ 1 := Tag }}) -> @@ -1029,8 +1205,6 @@ new_tuple_set(T) -> %% -shrink_union(#t_union{other=any}) -> - any; shrink_union(#t_union{atom=Atom,list=none,number=none, tuple_set=none,other=none}) -> Atom; @@ -1049,6 +1223,12 @@ shrink_union(#t_union{atom=none,list=none,number=none, shrink_union(#t_union{atom=none,list=none,number=none, tuple_set=none,other=Other}) -> Other; +shrink_union(#t_union{atom=#t_atom{elements=any}, + list=#t_list{type=any,terminator=any}, + number=#t_number{elements=any}, + tuple_set=#t_tuple{size=0,exact=false}, + other=other}) -> + any; shrink_union(#t_union{}=T) -> T. @@ -1124,6 +1304,12 @@ verified_normal_type(#t_fun{arity=Arity, T; verified_normal_type(#t_float{}=T) -> T; verified_normal_type(#t_integer{elements=any}=T) -> T; +verified_normal_type(#t_integer{elements={'-inf',Max}}=T) + when is_integer(Max) -> + T; +verified_normal_type(#t_integer{elements={Min,'+inf'}}=T) + when is_integer(Min) -> + T; verified_normal_type(#t_integer{elements={Min,Max}}=T) when is_integer(Min), is_integer(Max), Min =< Max -> T; @@ -1133,7 +1319,8 @@ verified_normal_type(#t_list{type=Type,terminator=Term}=T) -> T; verified_normal_type(#t_map{}=T) -> T; verified_normal_type(nil=T) -> T; -verified_normal_type(number=T) -> T; +verified_normal_type(#t_number{}=T) -> T; +verified_normal_type(other=T) -> T; verified_normal_type(pid=T) -> T; verified_normal_type(port=T) -> T; verified_normal_type(reference=T) -> T; @@ -1175,6 +1362,10 @@ verified_normal_type(#t_tuple{size=Size,elements=Es}=T) -> -define(BEAM_TYPE_REFERENCE, (1 bsl 11)). -define(BEAM_TYPE_TUPLE, (1 bsl 12)). +-define(BEAM_TYPE_HAS_LOWER_BOUND, (1 bsl 13)). +-define(BEAM_TYPE_HAS_UPPER_BOUND, (1 bsl 14)). +-define(BEAM_TYPE_HAS_UNIT, (1 bsl 15)). + ext_type_mapping() -> [{?BEAM_TYPE_ATOM, #t_atom{}}, {?BEAM_TYPE_BITSTRING, #t_bitstring{}}, @@ -1190,12 +1381,19 @@ ext_type_mapping() -> {?BEAM_TYPE_REFERENCE, reference}, {?BEAM_TYPE_TUPLE, #t_tuple{}}]. --spec decode_ext(binary()) -> type(). -decode_ext(<<TypeBits:16/big,Min:64,Max:64>>) -> +-spec decode_ext(binary()) -> {type(),binary()} | 'done'. +decode_ext(<<TypeBits:16/big,More/binary>>) -> Res = foldl(fun({Id, Type}, Acc) -> decode_ext_bits(TypeBits, Id, Type, Acc) end, none, ext_type_mapping()), - {Res,Min,Max}. + {[Min,Max,Unit],Extra} = decode_extra(TypeBits, More), + R = case {Min,Max} of + {'-inf','+inf'} -> any; + R0 -> R0 + end, + {decode_fix(Res, R, Unit),Extra}; +decode_ext(<<>>) -> + done. decode_ext_bits(Input, TypeBit, Type, Acc) -> case Input band TypeBit of @@ -1203,17 +1401,50 @@ decode_ext_bits(Input, TypeBit, Type, Acc) -> _ -> join(Type, Acc) end. +decode_extra(TypeBits, Extra) -> + L = [{?BEAM_TYPE_HAS_LOWER_BOUND, 64, '-inf'}, + {?BEAM_TYPE_HAS_UPPER_BOUND, 64, '+inf'}, + {?BEAM_TYPE_HAS_UNIT, 8, 1}], + mapfoldl(fun({Bit,Size,Default}, Acc0) -> + if + TypeBits band Bit =:= Bit -> + <<Value:Size,Acc/binary>> = Acc0, + {Value,Acc}; + true -> + {Default,Acc0} + end + end, Extra, L). + +decode_fix(#t_integer{}, Range, _Unit) -> + #t_integer{elements=Range}; +decode_fix(#t_number{}, Range, _Unit) -> + #t_number{elements=Range}; +decode_fix(#t_bitstring{}, _Range, Unit) -> + #t_bitstring{size_unit=Unit}; +decode_fix(#t_union{}=Type0, Range, Unit) -> + Type1 = case meet(Type0, #t_integer{}) of + #t_integer{} -> + Type0#t_union{number=#t_integer{elements=Range}}; + _ -> + Type0 + end, + case meet(Type1, #t_bitstring{}) of + #t_bitstring{} -> + Type1#t_union{other=#t_bitstring{size_unit=Unit}}; + _ -> + Type1 + end; +decode_fix(Type, _, _) -> + Type. + -spec encode_ext(type()) -> binary(). encode_ext(Input) -> - TypeBits = foldl(fun({Id, Type}, Acc) -> - encode_ext_bits(Input, Id, Type, Acc) - end, 0, ext_type_mapping()), - case get_integer_range(Input) of - none -> - <<TypeBits:16/big,0:64,-1:64>>; - {Min,Max} -> - <<TypeBits:16/big,Min:64,Max:64>> - end. + TypeBits0 = foldl(fun({Id, Type}, Acc) -> + encode_ext_bits(Input, Id, Type, Acc) + end, 0, ext_type_mapping()), + {TypeBits1,Extra} = encode_extra(Input), + TypeBits = TypeBits0 bor TypeBits1, + <<TypeBits:16,Extra/binary>>. encode_ext_bits(Input, TypeBit, Type, Acc) -> case meet(Input, Type) of @@ -1221,21 +1452,49 @@ encode_ext_bits(Input, TypeBit, Type, Acc) -> _ -> Acc bor TypeBit end. -get_integer_range(#t_integer{elements={Min,Max}}) -> - case is_small(Min) andalso is_small(Max) of +encode_extra(Input) -> + {TypeBits0,Extra0} = encode_range(Input), + {TypeBits1,Extra1} = encode_unit(Input), + {TypeBits0 bor TypeBits1,<<Extra0/binary,Extra1/binary>>}. + +encode_range(#t_integer{elements={Min,Max}}) -> + encode_range(Min, Max); +encode_range(#t_number{elements={Min,Max}}) -> + encode_range(Min, Max); +encode_range(#t_union{number=N}) -> + encode_range(N); +encode_range(_) -> + {0,<<>>}. + +encode_range(Min, Max) -> + case is_small(Min) of true -> - {Min,Max}; + encode_range(Max, ?BEAM_TYPE_HAS_LOWER_BOUND, <<Min:64>>); false -> - %% Not an integer with range, or at least one of the - %% endpoints is a bignum. - none - end; -get_integer_range(#t_union{number=N}) -> - get_integer_range(N); -get_integer_range(_) -> none. + encode_range(Max, 0, <<>>) + end. + +encode_range(Max, TypeBits, Extra) -> + case is_small(Max) of + true -> + {TypeBits bor ?BEAM_TYPE_HAS_UPPER_BOUND, + <<Extra/binary,Max:64>>}; + false -> + {TypeBits,Extra} + end. + +encode_unit(#t_bitstring{size_unit=Unit}) -> + true = is_integer(Unit) andalso 0 < Unit andalso Unit =< 256, %Assertion. + {?BEAM_TYPE_HAS_UNIT,<<(Unit-1):8>>}; +encode_unit(#t_union{other=Other}) -> + encode_unit(Other); +encode_unit(_) -> + {0,<<>>}. %% Test whether the number is a small on a 64-bit machine. %% (Normally the compiler doesn't know/doesn't care whether something is %% bignum, but because the type representation is versioned this is safe.) -is_small(N) -> - -(1 bsl 59) =< N andalso N =< (1 bsl 59) - 1. +is_small(N) when is_integer(N), -(1 bsl 59) =< N andalso N =< (1 bsl 59) - 1 -> + true; +is_small(_) -> + false. diff --git a/lib/compiler/src/beam_types.hrl b/lib/compiler/src/beam_types.hrl index 18f7e29074..9d6630673c 100644 --- a/lib/compiler/src/beam_types.hrl +++ b/lib/compiler/src/beam_types.hrl @@ -19,7 +19,7 @@ %% %% Type version, must be bumped whenever the external type format changes. --define(BEAM_TYPES_VERSION, 1). +-define(BEAM_TYPES_VERSION, 2). %% Common term types for passes operating on beam SSA and assembly. Helper %% functions for wrangling these can be found in beam_types.erl @@ -29,21 +29,22 @@ %% any Any Erlang term (top element). %% %% - #t_atom{} Atom, or a set thereof. -%% - #t_bs_matchable{} Binary-matchable types. -%% - #t_bitstring{} Bitstring. -%% - #t_bs_context{} Match context. -%% - #t_fun{} Fun. -%% - #t_map{} Map. -%% - number Any number. +%% - #t_number{} Any number. %% -- #t_float{} Floating point number. %% -- #t_integer{} Integer. %% - #t_list{} Any list. %% -- #t_cons{} Cons (nonempty list). %% -- nil The empty list. -%% - pid -%% - port -%% - reference %% - #t_tuple{} Tuple. +%% - other Other types. +%% -- #t_fun{} Fun. +%% -- #t_map{} Map. +%% -- pid +%% -- port +%% -- reference +%% -- #t_bs_matchable{} Binary-matchable types. +%% -- #t_bitstring{} Bitstring. +%% -- #t_bs_context{} Match context. %% %% none No type (bottom element). %% @@ -81,17 +82,23 @@ %% [1] https://en.wikipedia.org/wiki/Lattice_(order)#General_lattice -define(ATOM_SET_SIZE, 5). + +%% Documented limits -define(MAX_FUNC_ARGS, 255). +-define(MAX_TUPLE_SIZE, (1 bsl 24) - 1). + +-type float_range() :: 'any' | {'-inf',float()} | {float(),'+inf'}. -record(t_atom, {elements=any :: 'any' | ordsets:ordset(atom())}). -record(t_bitstring, {size_unit=1 :: pos_integer()}). -record(t_bs_context, {tail_unit=1 :: pos_integer()}). -record(t_bs_matchable, {tail_unit=1 :: pos_integer()}). --record(t_float, {elements=any :: 'any' | {float(),float()}}). +-record(t_float, {elements=any :: float_range()}). -record(t_fun, {arity=any :: arity() | 'any', target=any :: {atom(), non_neg_integer()} | 'any', type=any :: type() }). --record(t_integer, {elements=any :: 'any' | {integer(),integer()}}). +-record(t_integer, {elements=any :: 'any' | beam_bounds:range()}). +-record(t_number, {elements=any :: 'any' | beam_bounds:range()}). %% `super_key` and `super_value` are the join of all key and value types. %% @@ -132,27 +139,35 @@ -define(TUPLE_ELEMENT_LIMIT, 12). -type tuple_elements() :: #{ Key :: pos_integer() => type() }. --type normal_type() :: any | none | - number | #t_float{} | #t_integer{} | +-type normal_type() :: 'any' | 'none' | + #t_number{} | #t_float{} | #t_integer{} | #t_atom{} | #t_bitstring{} | #t_bs_context{} | #t_bs_matchable{} | #t_fun{} | - #t_list{} | #t_cons{} | nil | + #t_list{} | #t_cons{} | 'nil' | + 'other' | #t_map{} | - pid | - port | - reference | + 'pid' | + 'port' | + 'reference' | #t_tuple{}. +-type other_type() :: 'none' | #t_fun{} | #t_map{} | + 'pid' | 'port' | 'reference' | + #t_bitstring{} | #t_bs_context{} | + #t_bs_matchable{}. + -type record_key() :: {Arity :: integer(), Tag :: normal_type() }. -type record_set() :: ordsets:ordset({record_key(), #t_tuple{}}). -type tuple_set() :: #t_tuple{} | record_set(). --record(t_union, {atom=none :: none | #t_atom{}, - list=none :: none | #t_list{} | #t_cons{} | nil, - number=none :: none | number | #t_float{} | #t_integer{}, - tuple_set=none :: none | tuple_set(), - other=none :: normal_type()}). +%% The fields in the union must not overlap. In particular, that means +%% that the type `any` is not allowed in any field. +-record(t_union, {atom=none :: 'none' | #t_atom{}, + list=none :: 'none' | #t_list{} | #t_cons{} | nil, + number=none :: 'none' | #t_number{} | #t_float{} | #t_integer{}, + tuple_set=none :: 'none' | tuple_set(), + other=none :: 'other' | other_type()}). -type type() :: #t_union{} | normal_type(). diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 904e1f3e82..d407b6cb3e 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -115,6 +115,9 @@ replace_labels_1([{get_map_elements=I,{f,Lbl},Src,List}|Is], Acc, D, Fb) when Lb replace_labels_1([{bs_start_match4,{f,Lbl},Live,Src,Dst}|Is], Acc, D, Fb) -> I = {bs_start_match4,{f,label(Lbl, D, Fb)},Live,Src,Dst}, replace_labels_1(Is, [I | Acc], D, Fb); +replace_labels_1([{bs_match,{f,Lbl},Ctx,List}|Is], Acc, D, Fb) -> + I = {bs_match,{f,label(Lbl, D, Fb)},Ctx,List}, + replace_labels_1(Is, [I | Acc], D, Fb); replace_labels_1([I|Is], Acc, D, Fb) -> replace_labels_1(Is, [I|Acc], D, Fb); replace_labels_1([], Acc, _, _) -> Acc. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 9b3f60e377..f98bbe894f 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -427,7 +427,8 @@ vi({jump,{f,Lbl}}, Vst) -> %% The next instruction is never executed. branch(Lbl, Vst, fun kill_state/1); -vi({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> +vi({select_val,Src0,{f,Fail},{list,Choices}}, Vst) -> + Src = unpack_typed_arg(Src0), assert_term(Src, Vst), assert_choices(Choices), validate_select_val(Fail, Choices, Src, Vst); @@ -474,7 +475,7 @@ vi({test,is_integer,{f,Lbl},[Src]}, Vst) -> vi({test,is_nonempty_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_cons{}, Src, Vst); vi({test,is_number,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, number, Src, Vst); + type_test(Lbl, #t_number{}, Src, Vst); vi({test,is_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_list{}, Src, Vst); vi({test,is_map,{f,Lbl},[Src]}, Vst) -> @@ -666,9 +667,12 @@ vi({set_tuple_element,Src,Tuple,N}, Vst) -> %% helpers as we must support overwriting (rather than just widening or %% narrowing) known elements, and we can't use extract_term either since %% the source tuple may be aliased. - #t_tuple{elements=Es0}=Type = normalize(get_term_type(Tuple, Vst)), - Es = beam_types:set_tuple_element(I, get_term_type(Src, Vst), Es0), - override_type(Type#t_tuple{elements=Es}, Tuple, Vst); + TupleType0 = get_term_type(Tuple, Vst), + ArgType = get_term_type(Src, Vst), + TupleType = beam_types:update_tuple(TupleType0, [{I, ArgType}]), + override_type(TupleType, Tuple, Vst); +vi({update_record,_Hint,Size,Src,Dst,{list,Ss}}, Vst) -> + verify_update_record(Size, Src, Dst, Ss, Vst); %% %% Calls @@ -917,6 +921,20 @@ vi({put_map_exact=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> %% Bit syntax matching %% +vi({bs_match,{f,Fail},Ctx0,{commands,List}}, Vst) -> + Ctx = unpack_typed_arg(Ctx0), + + assert_no_exception(Fail), + assert_type(#t_bs_context{}, Ctx, Vst), + verify_y_init(Vst), + + branch(Fail, Vst, + fun(FailVst) -> + validate_failed_bs_match(List, Ctx, FailVst) + end, + fun(SuccVst) -> + validate_bs_match(List, Ctx, 1, SuccVst) + end); vi({bs_get_tail,Ctx,Dst,Live}, Vst0) -> assert_type(#t_bs_context{}, Ctx, Vst0), verify_live(Live, Vst0), @@ -934,7 +952,8 @@ vi({test,bs_start_match3,{f,_}=Fail,Live,[Src],Dst}, Vst) -> vi({test,bs_match_string,{f,Fail},[Ctx,Stride,{string,String}]}, Vst) -> true = is_bitstring(String), %Assertion. validate_bs_skip(Fail, Ctx, Stride, Vst); -vi({test,bs_skip_bits2,{f,Fail},[Ctx,Size,Unit,_Flags]}, Vst) -> +vi({test,bs_skip_bits2,{f,Fail},[Ctx,Size0,Unit,_Flags]}, Vst) -> + Size = unpack_typed_arg(Size0), assert_term(Size, Vst), Stride = case get_concrete_type(Size, Vst) of @@ -975,28 +994,20 @@ vi({test,bs_get_binary2=Op,{f,Fail},Live,[Ctx,Size,Unit,_],Dst}, Vst) -> validate_bs_get(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst); vi({test,bs_get_integer2=Op,{f,Fail},Live, [Ctx,{integer,Sz},Unit,{field_flags,Flags}],Dst},Vst) -> - NumBits = Unit * Sz, Stride = NumBits, - - Type = if - 0 =< NumBits, NumBits =< 64 -> - Max = (1 bsl NumBits) - 1, - case member(unsigned, Flags) of - true -> - beam_types:make_integer(0, Max); - false -> - Min = -(Max + 1), - beam_types:make_integer(Min, Max) - end; - true -> - %% Way too large or negative size. + Type = bs_integer_type(NumBits, Flags), + validate_bs_get(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst); +vi({test,bs_get_integer2=Op,{f,Fail},Live,[Ctx,Sz0,Unit,{field_flags,Flags}],Dst},Vst) -> + Sz = unpack_typed_arg(Sz0), + Type = case meet(get_term_type(Sz, Vst), #t_integer{}) of + #t_integer{elements={_,SizeMax}} when SizeMax * Unit < 64 -> + NumBits = SizeMax * Unit, + bs_integer_type(NumBits, Flags); + _ -> #t_integer{} end, - - validate_bs_get(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst); -vi({test,bs_get_integer2=Op,{f,Fail},Live,[Ctx,_Sz,Unit,_Flags],Dst},Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, Unit, #t_integer{}, Dst, Vst); + validate_bs_get(Op, Fail, Ctx, Live, Unit, Type, Dst, Vst); vi({test,bs_get_float2=Op,{f,Fail},Live,[Ctx,Size,Unit,_],Dst},Vst) -> Stride = bsm_stride(Size, Unit), validate_bs_get(Op, Fail, Ctx, Live, Stride, #t_float{}, Dst, Vst); @@ -1009,8 +1020,29 @@ vi({test,bs_get_utf16=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> vi({test,bs_get_utf32=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> Type = beam_types:make_integer(0, ?UNICODE_MAX), validate_bs_get(Op, Fail, Ctx, Live, 32, Type, Dst, Vst); +vi({test,is_lt,{f,Fail},Args0}, Vst) -> + Args = [unpack_typed_arg(Arg) || Arg <- Args0], + validate_src(Args, Vst), + Types = [get_term_type(Arg, Vst) || Arg <- Args], + branch(Fail, Vst, + fun(FailVst) -> + infer_relop_types('>=', Args, Types, FailVst) + end, + fun(SuccVst) -> + infer_relop_types('<', Args, Types, SuccVst) + end); +vi({test,is_ge,{f,Fail},Args0}, Vst) -> + Args = [unpack_typed_arg(Arg) || Arg <- Args0], + validate_src(Args, Vst), + Types = [get_term_type(Arg, Vst) || Arg <- Args], + branch(Fail, Vst, + fun(FailVst) -> + infer_relop_types('<', Args, Types, FailVst) + end, + fun(SuccVst) -> + infer_relop_types('>=', Args, Types, SuccVst) + end); vi({test,_Op,{f,Lbl},Ss}, Vst) -> - %% is_lt, is_gt, et cetera. validate_src([unpack_typed_arg(Arg) || Arg <- Ss], Vst), branch(Lbl, Vst); @@ -1049,7 +1081,7 @@ vi({fconv,Src0,{fr,_}=Dst}, Vst) -> assert_term(Src, Vst), branch(?EXCEPTION_LABEL, Vst, fun(SuccVst0) -> - SuccVst = update_type(fun meet/2, number, Src, SuccVst0), + SuccVst = update_type(fun meet/2, #t_number{}, Src, SuccVst0), set_freg(Dst, SuccVst) end); @@ -1218,6 +1250,81 @@ vi({bs_put_utf32,{f,Fail},_,Src}, Vst) -> vi(_, _) -> error(unknown_instruction). +infer_relop_types(Op, Args, Types, Vst) -> + case infer_relop_types(Op, Types) of + [] -> + Vst; + Infer -> + Zipped = zip(Args, Infer), + foldl(fun({V,T}, Acc) -> + update_type(fun meet/2, T, V, Acc) + end, Vst, Zipped) + end. + +infer_relop_types(Op, [#t_integer{elements=R1}, + #t_integer{elements=R2}]) -> + case beam_bounds:infer_relop_types(Op, R1, R2) of + any -> + []; + {NewR1,NewR2} -> + NewType1 = #t_integer{elements=NewR1}, + NewType2 = #t_integer{elements=NewR2}, + [NewType1,NewType2] + end; +infer_relop_types(Op0, [Type1,Type2]) -> + Op = case Op0 of + '<' -> '=<'; + '>' -> '>='; + _ -> Op0 + end, + case {infer_get_range(Type1),infer_get_range(Type2)} of + {none,_}=R -> + [infer_relop_any(Op, R, Type1),Type2]; + {_,none}=R -> + [Type1,infer_relop_any(Op, R, Type2)]; + {R1,R2} -> + case beam_bounds:infer_relop_types(Op, R1, R2) of + any -> + []; + {NewR1,NewR2} -> + NewType1 = meet(#t_number{elements=NewR1}, Type1), + NewType2 = meet(#t_number{elements=NewR2}, Type2), + [NewType1,NewType2] + end + end; +infer_relop_types(_, _) -> + []. + +infer_relop_any('=<', {none,any}, Type) -> + N = #t_number{}, + meet(N, Type); +infer_relop_any('=<', {none,{_,Max}}, Type) -> + N = infer_make_number({'-inf',Max}), + meet(N, Type); +infer_relop_any('>=', {any,none}, Type) -> + N = #t_number{}, + meet(N, Type); +infer_relop_any('>=', {{_,Max},none}, Type) -> + N = infer_make_number({'-inf',Max}), + meet(N, Type); +infer_relop_any('>=', {none,{Min,_}}, Type) when is_integer(Min) -> + N = #t_number{elements={'-inf',Min}}, + meet(subtract(any, N), Type); +infer_relop_any('=<', {{Min,_},none}, Type) when is_integer(Min) -> + N = #t_number{elements={'-inf',Min}}, + meet(subtract(any, N), Type); +infer_relop_any(_, _, Type) -> + Type. + +infer_make_number({'-inf','+inf'}) -> + #t_number{}; +infer_make_number({_,_}=R) -> + #t_number{elements=R}. + +infer_get_range(#t_integer{elements=R}) -> R; +infer_get_range(#t_number{elements=R}) -> R; +infer_get_range(_) -> none. + validate_var_info([{fun_type, Type} | Info], Reg, Vst0) -> %% Explicit type information inserted after make_fun2 instructions to mark %% the return type of the created fun. @@ -1422,17 +1529,48 @@ verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> end. put_map_type(Map0, List, Vst) -> - Map = normalize(get_term_type(Map0, Vst)), + Map = get_term_type(Map0, Vst), pmt_1(List, Vst, Map). pmt_1([Key0, Value0 | List], Vst, Acc0) -> - Key = normalize(get_term_type(Key0, Vst)), - Value = normalize(get_term_type(Value0, Vst)), + Key = get_term_type(Key0, Vst), + Value = get_term_type(Value0, Vst), {Acc, _, _} = beam_call_types:types(maps, put, [Key, Value, Acc0]), pmt_1(List, Vst, Acc); pmt_1([], _Vst, Acc) -> Acc. +verify_update_record(Size, Src, Dst, List, Vst0) -> + assert_type(#t_tuple{exact=true,size=Size}, Src, Vst0), + verify_y_init(Vst0), + + Vst = eat_heap(Size + 1, Vst0), + + case update_tuple_type(List, Src, Vst) of + none -> error(invalid_index); + Type -> create_term(Type, update_record, [], Dst, Vst) + end. + +update_tuple_type([_|_]=Updates0, Src, Vst) -> + Filter = #t_tuple{size=update_tuple_highest_index(Updates0, -1)}, + case meet(get_term_type(Src, Vst), Filter) of + none -> + none; + TupleType -> + Updates = update_tuple_type_1(Updates0, Vst), + beam_types:update_tuple(TupleType, Updates) + end. + +update_tuple_type_1([Index, Value | Updates], Vst) -> + [{Index, get_term_type(Value, Vst)} | update_tuple_type_1(Updates, Vst)]; +update_tuple_type_1([], _Vst) -> + []. + +update_tuple_highest_index([Index, _Val | List], Acc) when is_integer(Index) -> + update_tuple_highest_index(List, max(Index, Acc)); +update_tuple_highest_index([], Acc) when Acc >= 1 -> + Acc. + verify_create_bin_list([{atom,string},_Seg,Unit,Flags,Val,Size|Args], Vst) -> assert_bs_unit({atom,string}, Unit), assert_term(Flags, Vst), @@ -1591,8 +1729,81 @@ validate_bs_start_match({f,Fail}, Live, Src, Dst, Vst) -> end). %% +%% Validate the bs_match instruction. +%% + +validate_bs_match([{get_tail,Live,_,Dst}], Ctx, _, Vst0) -> + validate_ctx_live(Ctx, Live), + verify_live(Live, Vst0), + Vst = prune_x_regs(Live, Vst0), + #t_bs_context{tail_unit=Unit} = get_concrete_type(Ctx, Vst0), + Type = #t_bitstring{size_unit=Unit}, + extract_term(Type, get_tail, [Ctx], Dst, Vst, Vst0); +validate_bs_match([I|Is], Ctx, Unit0, Vst0) -> + case I of + {ensure_at_least,_Size,Unit} -> + Type = #t_bs_context{tail_unit=Unit}, + Vst1 = update_bs_unit(Ctx, Unit, Vst0), + Vst = update_type(fun meet/2, Type, Ctx, Vst1), + validate_bs_match(Is, Ctx, Unit, Vst); + {ensure_exactly,Stride} -> + Vst = advance_bs_context(Ctx, Stride, Vst0), + validate_bs_match(Is, Ctx, Unit0, Vst); + {'=:=',nil,Bits,Value} when Bits =< 64, is_integer(Value) -> + validate_bs_match(Is, Ctx, Unit0, Vst0); + {Type0,Live,{literal,Flags},Size,Unit,Dst} when Type0 =:= binary; + Type0 =:= integer -> + validate_ctx_live(Ctx, Live), + verify_live(Live, Vst0), + Vst1 = prune_x_regs(Live, Vst0), + Stride = Size * Unit, + Type = case Type0 of + integer -> + bs_integer_type(Stride, Flags); + binary -> + #t_bitstring{size_unit=bsm_size_unit({integer,Size}, Unit)} + end, + Vst = extract_term(Type, bs_match, [Ctx], Dst, Vst1, Vst0), + validate_bs_match(Is, Ctx, Unit0, Vst); + {skip,_Stride} -> + validate_bs_match(Is, Ctx, Unit0, Vst0) + end; +validate_bs_match([], _Ctx, _Unit, Vst) -> + Vst. + +validate_ctx_live({x,X}=Ctx, Live) when X >= Live -> + error({live_does_not_preserve_context,Live,Ctx}); +validate_ctx_live(_, _) -> + ok. + +validate_failed_bs_match([{ensure_at_least,_Size,Unit}|_], Ctx, Vst) -> + Type = #t_bs_context{tail_unit=Unit}, + update_type(fun subtract/2, Type, Ctx, Vst); +validate_failed_bs_match([_|Is], Ctx, Vst) -> + validate_failed_bs_match(Is, Ctx, Vst); +validate_failed_bs_match([], _Ctx, Vst) -> + Vst. + +bs_integer_type(NumBits, Flags) -> + if + 0 =< NumBits, NumBits =< 64 -> + Max = (1 bsl NumBits) - 1, + case member(signed, Flags) of + false -> + beam_types:make_integer(0, Max); + true -> + Min = -(Max + 1), + beam_types:make_integer(Min, Max) + end; + true -> + %% Way too large or negative size. + #t_integer{} + end. + +%% %% Common code for validating bs_get* instructions. %% + validate_bs_get(Op, Fail, Ctx0, Live, Stride, Type, Dst, Vst) -> Ctx = unpack_typed_arg(Ctx0), @@ -2106,6 +2317,18 @@ infer_types_1(#value{op={bif,'=/='},args=[LHS,RHS]}, Val, Op, Vst) -> _ -> Vst end; +infer_types_1(#value{op={bif,RelOp},args=[_,_]=Args}, Val, Op, Vst) + when RelOp =:= '<'; RelOp =:= '=<'; RelOp =:= '>='; RelOp =:= '>' -> + case Val of + {atom, Bool} when Op =:= eq_exact, Bool; Op =:= ne_exact, not Bool -> + Types = [get_term_type(Arg, Vst) || Arg <- Args], + infer_relop_types(RelOp, Args, Types, Vst); + {atom, Bool} when Op =:= ne_exact, Bool; Op =:= eq_exact, not Bool -> + Types = [get_term_type(Arg, Vst) || Arg <- Args], + infer_relop_types(invert_relop(RelOp), Args, Types, Vst); + _ -> + Vst + end; infer_types_1(#value{op={bif,is_atom},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_atom{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_boolean},args=[Src]}, Val, Op, Vst) -> @@ -2139,7 +2362,7 @@ infer_types_1(#value{op={bif,is_list},args=[Src]}, Val, Op, Vst) -> infer_types_1(#value{op={bif,is_map},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_map{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_number},args=[Src]}, Val, Op, Vst) -> - infer_type_test_bif(number, Src, Val, Op, Vst); + infer_type_test_bif(#t_number{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_pid},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(pid, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_port},args=[Src]}, Val, Op, Vst) -> @@ -2195,6 +2418,11 @@ infer_type_test_bif(Type, Src, Val, Op, Vst) -> Vst end. +invert_relop('<') -> '>='; +invert_relop('=<') -> '>'; +invert_relop('>=') -> '<'; +invert_relop('>') -> '=<'. + %%% %%% Keeping track of types. %%% @@ -2346,7 +2574,7 @@ update_ne_types_1(LHS, RHS, Vst0) -> %% is a bit trickier since all we know is that the *value* of LHS differs %% from RHS, so we can't blindly subtract their types. %% - %% Consider `number =/= #t_integer{}`; all we know is that LHS isn't equal + %% Consider `#number{} =/= #t_integer{}`; all we know is that LHS isn't equal %% to some *specific integer* of unknown value, and if we were to subtract %% #t_integer{} we would erroneously infer that the new type is float. %% @@ -3105,7 +3333,7 @@ assert_not_fragile(Lit, #vst{}) -> %%% bif_types(Op, Ss, Vst) -> - Args = [normalize(get_term_type(Arg, Vst)) || Arg <- Ss], + Args = [get_term_type(Arg, Vst) || Arg <- Ss], case {Op,Ss} of {element,[_,{literal,Tuple}]} when tuple_size(Tuple) > 0 -> case beam_call_types:types(erlang, Op, Args) of @@ -3132,6 +3360,9 @@ join_tuple_elements(I, Tuple, Type0) -> call_types({extfunc,M,F,A}, A, Vst) -> Args = get_call_args(A, Vst), beam_call_types:types(M, F, Args); +call_types(bs_init_writable, A, Vst) -> + T = beam_types:make_type_from_value(<<>>), + {T, get_call_args(A, Vst), false}; call_types(_, A, Vst) -> {any, get_call_args(A, Vst), false}. @@ -3144,7 +3375,7 @@ will_bif_succeed(Op, Ss, Vst) -> true -> 'maybe'; false -> - Args = [normalize(get_term_type(Arg, Vst)) || Arg <- Ss], + Args = [get_term_type(Arg, Vst) || Arg <- Ss], beam_call_types:will_succeed(erlang, Op, Args) end. @@ -3179,7 +3410,7 @@ get_call_args(Arity, Vst) -> get_call_args_1(Arity, Arity, _) -> []; get_call_args_1(N, Arity, Vst) when N < Arity -> - ArgType = normalize(get_movable_term_type({x,N}, Vst)), + ArgType = get_movable_term_type({x,N}, Vst), [ArgType | get_call_args_1(N + 1, Arity, Vst)]. check_limit({x,X}=Src) when is_integer(X) -> diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index bc28f58712..99651ebc2f 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -1647,34 +1647,41 @@ ann_c_map(As, Es) -> -spec ann_c_map([term()], c_map() | c_literal(), [c_map_pair()]) -> c_map() | c_literal(). -ann_c_map(As,#c_literal{val=M},Es) when is_map(M) -> - fold_map_pairs(As,Es,M); -ann_c_map(As,M,Es) -> - #c_map{arg=M, es=Es, anno=As }. - -fold_map_pairs(As,[],M) -> #c_literal{anno=As,val=M}; -%% M#{ K => V} -fold_map_pairs(As,[#c_map_pair{op=#c_literal{val=assoc},key=Ck,val=Cv}=E|Es],M) -> +ann_c_map(As, #c_literal{val=M0}=Lit, Es) when is_map(M0) -> + case update_map_literal(Es, M0) of + none -> + #c_map{arg=Lit, es=Es, anno=As}; + M1 -> + #c_literal{anno=As, val=M1} + end; +ann_c_map(As, M, Es) -> + #c_map{arg=M, es=Es, anno=As}. + +update_map_literal([#c_map_pair{op=#c_literal{val=assoc},key=Ck,val=Cv}|Es], M) -> + %% M#{K => V} case is_lit_list([Ck,Cv]) of true -> [K,V] = lit_list_vals([Ck,Cv]), - fold_map_pairs(As,Es,maps:put(K,V,M)); + update_map_literal(Es, M#{K => V}); false -> - #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } + none end; -%% M#{ K := V} -fold_map_pairs(As,[#c_map_pair{op=#c_literal{val=exact},key=Ck,val=Cv}=E|Es],M) -> +update_map_literal([#c_map_pair{op=#c_literal{val=exact},key=Ck,val=Cv}|Es], M) -> + %% M#{K := V} case is_lit_list([Ck,Cv]) of true -> [K,V] = lit_list_vals([Ck,Cv]), - case maps:is_key(K,M) of - true -> fold_map_pairs(As,Es,maps:put(K,V,M)); + case is_map_key(K, M) of + true -> + update_map_literal(Es, M#{K => V}); false -> - #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } + none end; false -> - #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } - end. + none + end; +update_map_literal([], M) -> + M. -spec update_c_map(c_map(), cerl(), [cerl()]) -> c_map() | c_literal(). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 7ef37a6082..dc530d0e4a 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -274,7 +274,10 @@ expand_opt(r23, Os) -> no_recv_opt, no_init_yregs | expand_opt(r24, Os)]); expand_opt(r24, Os) -> - expand_opt(no_type_opt, [no_bs_create_bin, no_ssa_opt_ranges | Os]); + expand_opt(no_type_opt, [no_bs_create_bin, no_ssa_opt_ranges | + expand_opt(r25, Os)]); +expand_opt(r25, Os) -> + [no_ssa_opt_update_tuple, no_bs_match | Os]; expand_opt(no_make_fun3, Os) -> [no_make_fun3, no_fun_opt | Os]; expand_opt({debug_info_key,_}=O, Os) -> diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl index a50a2ffa8d..a7f06b8da4 100644 --- a/lib/compiler/src/core_scan.erl +++ b/lib/compiler/src/core_scan.erl @@ -58,6 +58,12 @@ -type error_description() :: term(). -type error_info() :: {erl_anno:location(), module(), error_description()}. +-define(IS_UNICODE(C), + (is_integer(C) andalso + (C >= 0 andalso C < 16#D800 orelse + C > 16#DFFF andalso C < 16#FFFE orelse + C > 16#FFFF andalso C =< 16#10FFFF))). + %% string([Char]) -> %% string([Char], StartPos) -> %% {ok, [Tok], EndPos} | @@ -256,6 +262,8 @@ scan(Cs, Pos) -> %% scan1(Characters, TokenStack, Position) %% Scan a list of characters into tokens. +scan1([C|_s], _Toks, _Pos) when not ?IS_UNICODE(C) -> + error({badchar,C}); scan1([$\n|Cs], Toks, Pos) -> %Skip newline scan1(Cs, Toks, Pos+1); scan1([C|Cs], Toks, Pos) when C >= $\000, C =< $\s -> %Skip control chars @@ -272,9 +280,9 @@ scan1([C|Cs], Toks, Pos) when C >= $À, C =< $Þ, C /= $× -> scan_variable(C, Cs, Toks, Pos); scan1([C|Cs], Toks, Pos) when C >= $0, C =< $9 -> %Numbers scan_number(C, Cs, Toks, Pos); -scan1([$-,C|Cs], Toks, Pos) when C >= $0, C =< $9 -> %Signed numbers +scan1([$-,C|Cs], Toks, Pos) when is_integer(C), C >= $0, C =< $9 -> %Signed numbers scan_signed_number($-, C, Cs, Toks, Pos); -scan1([$+,C|Cs], Toks, Pos) when C >= $0, C =< $9 -> %Signed numbers +scan1([$+,C|Cs], Toks, Pos) when is_integer(C), C >= $0, C =< $9 -> %Signed numbers scan_signed_number($+, C, Cs, Toks, Pos); scan1([$_|Cs], Toks, Pos) -> %_ variables scan_variable($_, Cs, Toks, Pos); @@ -338,6 +346,8 @@ scan_name([C|Cs], Ncs) -> scan_name([], Ncs) -> {Ncs,[]}. +name_char(C) when not ?IS_UNICODE(C) -> + error({badchar,C}); name_char(C) when C >= $a, C =< $z -> true; name_char(C) when C >= $ß, C =< $ÿ, C /= $÷ -> true; name_char(C) when C >= $A, C =< $Z -> true; @@ -374,15 +384,18 @@ scan_char([C|Cs], Pos) -> {C,Cs,Pos}. scan_escape([O1,O2,O3|Cs], Pos) when %\<1-3> octal digits - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> + is_integer(O1), O1 >= $0, O1 =< $7, + is_integer(O2), O2 >= $0, O2 =< $7, + is_integer(O3), O3 >= $0, O3 =< $7 -> Val = (O1*8 + O2)*8 + O3 - 73*$0, {Val,Cs,Pos}; scan_escape([O1,O2|Cs], Pos) when - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7 -> + is_integer(O1), O1 >= $0, O1 =< $7, + is_integer(O2), O2 >= $0, O2 =< $7 -> Val = (O1*8 + O2) - 9*$0, {Val,Cs,Pos}; scan_escape([O1|Cs], Pos) when - O1 >= $0, O1 =< $7 -> + is_integer(O1), O1 >= $0, O1 =< $7 -> {O1 - $0,Cs,Pos}; scan_escape([$^,C|Cs], Pos) -> %\^X -> CTL-X Val = C band 31, @@ -422,7 +435,8 @@ escape_char(C) -> C. %% SPos == Start position %% CPos == Current position -scan_number(C, Cs0, Toks, Pos) -> +scan_number(C, Cs0, Toks, Pos) when + is_integer(C), C >= $0, C =< $9 -> {Ncs,Cs,Pos1} = scan_integer(Cs0, [C], Pos), scan_after_int(Cs, Ncs, Toks, Pos, Pos1). @@ -430,17 +444,19 @@ scan_signed_number(S, C, Cs0, Toks, Pos) -> {Ncs,Cs,Pos1} = scan_integer(Cs0, [C,S], Pos), scan_after_int(Cs, Ncs, Toks, Pos, Pos1). -scan_integer([C|Cs], Stack, Pos) when C >= $0, C =< $9 -> +scan_integer([C|Cs], Stack, Pos) when + is_integer(C), C >= $0, C =< $9 -> scan_integer(Cs, [C|Stack], Pos); scan_integer(Cs, Stack, Pos) -> {Stack,Cs,Pos}. -scan_after_int([$.,C|Cs0], Ncs0, Toks, SPos, CPos) when C >= $0, C =< $9 -> +scan_after_int([$.,C|Cs0], Ncs0, Toks, SPos, CPos) when + is_integer(C), C >= $0, C =< $9 -> {Ncs,Cs,CPos1} = scan_integer(Cs0, [C,$.|Ncs0], CPos), - scan_after_fraction(Cs, Ncs, Toks, SPos, CPos1); + scan_after_fraction(Cs, Ncs, Toks, SPos, CPos1); scan_after_int([$#|Cs], Ncs, Toks, SPos, CPos) -> case list_to_integer(reverse(Ncs)) of - Base when Base >= 2, Base =< 16 -> + Base when is_integer(Base), Base >= 2, Base =< 16 -> scan_based_int(Cs, 0, Base, Toks, SPos, CPos); Base -> scan_error({base,Base}, CPos) @@ -450,15 +466,15 @@ scan_after_int(Cs, Ncs, Toks, SPos, CPos) -> scan1(Cs, [{integer,SPos,N}|Toks], CPos). scan_based_int([C|Cs], SoFar, Base, Toks, SPos, CPos) when - C >= $0, C =< $9, C < Base + $0 -> + is_integer(C), C >= $0, C =< $9, C < Base + $0 -> Next = SoFar * Base + (C - $0), scan_based_int(Cs, Next, Base, Toks, SPos, CPos); scan_based_int([C|Cs], SoFar, Base, Toks, SPos, CPos) when - C >= $a, C =< $f, C < Base + $a - 10 -> + is_integer(C), C >= $a, C =< $f, C < Base + $a - 10 -> Next = SoFar * Base + (C - $a + 10), scan_based_int(Cs, Next, Base, Toks, SPos, CPos); scan_based_int([C|Cs], SoFar, Base, Toks, SPos, CPos) when - C >= $A, C =< $F, C < Base + $A - 10 -> + is_integer(C), C >= $A, C =< $F, C < Base + $A - 10 -> Next = SoFar * Base + (C - $A + 10), scan_based_int(Cs, Next, Base, Toks, SPos, CPos); scan_based_int(Cs, SoFar, _, Toks, SPos, CPos) -> @@ -485,7 +501,8 @@ scan_exponent([$-|Cs], Ncs, Toks, SPos, CPos) -> scan_exponent(Cs, Ncs, Toks, SPos, CPos) -> scan_exponent1(Cs, Ncs, Toks, SPos, CPos). -scan_exponent1([C|Cs0], Ncs0, Toks, SPos, CPos) when C >= $0, C =< $9 -> +scan_exponent1([C|Cs0], Ncs0, Toks, SPos, CPos) when + is_integer(C), C >= $0, C =< $9 -> {Ncs,Cs,CPos1} = scan_integer(Cs0, [C|Ncs0], CPos), case catch list_to_float(reverse(Ncs)) of N when is_float(N) -> diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index e1e36fa4dc..3363f5873f 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -670,3 +670,31 @@ BEAM_FORMAT_NUMBER=0 ## @spec badrecord Value ## @doc Raises a {badrecord,Value} error exception. 180: badrecord/1 + +# OTP 26 + +## @spec update_record Hint Size Src Dst Updates=[Index, Value] +## @doc Sets Dst to a copy of Src with the update list applied. Hint can be +## one of: +## +## * {atom,copy} - The result will always differ from Src, so +## don't bother checking if it can be reused. +## * {atom,reuse} - Reuse Src if a runtime check deduces that it's +## equal to the result. +## +## Note that these are just hints and the implementation is free to +## ignore them. More hints may be added in the future. +181: update_record/5 + +## @spec bs_match Fail Ctx {commands,Commands} +## @doc Match one or more binary segments of fixed size. Commands +## can be one of the following: +## +## * {ensure_at_least,Stride,Unit} +## * {ensure_exactly,Stride} +## * {binary,Live,Flags,Size,Unit,Dst} +## * {integer,Live,Flags,Size,Unit,Dst} +## * {skip,Stride} +## * {get_tail,Live,Unit,Dst} +## * {'=:=',Live,Size,Value}. +182: bs_match/3 diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index d79753068f..9021d9eb5c 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -93,6 +93,9 @@ -define(ASSERT(E), ignore). -endif. +-define(MAX_FUNC_ARGS, 255). +-define(IS_FUNC_ARITY(A), is_integer(A) andalso 0 =< A andalso A =< ?MAX_FUNC_ARGS). + %% Variable value info. -record(sub, {v=[], %Variable substitutions s=sets:new([{version, 2}]) :: sets:set(), %Variables in scope @@ -1866,6 +1869,7 @@ case_opt_data_2(P, TypeSig, Bs0) -> {[V|Vs],none} -> {Type,Arity} = TypeSig, Ann = [compiler_generated], + true = ?IS_FUNC_ARITY(Arity), Vars = make_vars(Ann, Arity), Data = cerl:ann_make_data(Ann, Type, Vars), Bs = [{V,Data} | [{Var,V} || Var <- Vs] ++ Bs0], @@ -1923,7 +1927,7 @@ pat_to_expr(P) -> pat_to_expr_list(Ps) -> [pat_to_expr(P) || P <- Ps]. -make_vars(A, Max) -> +make_vars(A, Max) when ?IS_FUNC_ARITY(Max) -> make_vars(A, 1, Max). make_vars(A, I, Max) when I =< Max -> @@ -2443,6 +2447,7 @@ delay_build_1(Core0, TypeSig) -> Core -> {Type,Arity} = TypeSig, Ann = [compiler_generated], + true = ?IS_FUNC_ARITY(Arity), Vars = make_vars(Ann, Arity), Data = cerl:ann_make_data(Ann, Type, Vars), {yes,Vars,Core,Data} diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 974d3413b1..16b3ac340f 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -160,7 +160,8 @@ opts=[] :: [compile:option()], %Options. dialyzer=false :: boolean(), %Help dialyzer or not. ws=[] :: [warning()], %Warnings. - file=[{file,""}] %File. + file=[{file,""}], %File. + load_nif=false :: boolean() %true if calls erlang:load_nif/2 }). %% XXX: The following type declarations do not belong in this module @@ -171,12 +172,16 @@ -record(imodule, {name = [], exports = ordsets:new(), - nifs = sets:new([{version, 2}]), + nifs = none :: + 'none' | sets:set(), % Is a set if the attribute is + % present in the module. attrs = [], defs = [], file = [], opts = [], - ws = []}). + ws = [], + load_nif=false :: boolean() %true if calls erlang:load_nif/2 + }). -spec module([form()], [compile:option()]) -> {'ok',cerl:c_module(),[warning()]}. @@ -186,19 +191,28 @@ module(Forms0, Opts) -> Module = foldl(fun (F, Acc) -> form(F, Acc, Opts) end, #imodule{}, Forms), - #imodule{name=Mod,exports=Exp0,attrs=As0,defs=Kfs0,ws=Ws} = Module, + #imodule{name=Mod,exports=Exp0,attrs=As0, + defs=Kfs0,ws=Ws,load_nif=LoadNif,nifs=Nifs} = Module, Exp = case member(export_all, Opts) of true -> defined_functions(Forms); false -> Exp0 end, Cexp = [#c_var{name=FA} || {_,_}=FA <- Exp], + Kfs1 = reverse(Kfs0), + Kfs = if LoadNif and (Nifs =:= none) -> + insert_nif_start(Kfs1); + true -> + Kfs1 + end, As = reverse(As0), - Kfs = reverse(Kfs0), + {ok,#c_module{name=#c_literal{val=Mod},exports=Cexp,attrs=As,defs=Kfs},Ws}. -form({function,_,_,_,_}=F0, #imodule{defs=Defs}=Module, Opts) -> - {F,Ws} = function(F0, Module, Opts), - Module#imodule{defs=[F|Defs],ws=Ws}; +form({function,_,_,_,_}=F0, + #imodule{defs=Defs,load_nif=LoadNif0}=Module, + Opts) -> + {F,Ws,LoadNif} = function(F0, Module, Opts), + Module#imodule{defs=[F|Defs],ws=Ws,load_nif=LoadNif or LoadNif0}; form({attribute,_,module,Mod}, Module, _Opts) -> true = is_atom(Mod), Module#imodule{name=Mod}; @@ -211,7 +225,13 @@ form({attribute,_,export,Es}, #imodule{exports=Exp0}=Module, _Opts) -> Exp = ordsets:union(ordsets:from_list(Es), Exp0), Module#imodule{exports=Exp}; form({attribute,_,nifs,Ns}, #imodule{nifs=Nifs0}=Module, _Opts) -> - Nifs = sets:union(sets:from_list(Ns, [{version,2}]), Nifs0), + Nifs1 = case Nifs0 of + none -> + sets:new([{version, 2}]); + _ -> + Nifs0 + end, + Nifs = sets:union(sets:from_list(Ns, [{version,2}]), Nifs1), Module#imodule{nifs=Nifs}; form({attribute,_,_,_}=F, #imodule{attrs=As}=Module, _Opts) -> Module#imodule{attrs=[attribute(F)|As]}; @@ -236,7 +256,8 @@ defined_functions(Forms) -> %% io:format("~w/~w " ++ Format,[Name,Arity]++Terms), %% ok. -function({function,_,Name,Arity,Cs0}, Module, Opts) -> +function({function,_,Name,Arity,Cs0}, Module, Opts) + when is_integer(Arity), 0 =< Arity, Arity =< 255 -> #imodule{file=File, ws=Ws0, nifs=Nifs} = Module, try St0 = #core{vcount=0,function={Name,Arity},opts=Opts, @@ -248,9 +269,9 @@ function({function,_,Name,Arity,Cs0}, Module, Opts) -> %% ok = function_dump(Name, Arity, "ubody:~n~p~n",[B1]), {B2,St3} = cbody(B1, Nifs, St2), %% ok = function_dump(Name, Arity, "cbody:~n~p~n",[B2]), - {B3,#core{ws=Ws}} = lbody(B2, St3), + {B3,#core{ws=Ws,load_nif=LoadNif}} = lbody(B2, St3), %% ok = function_dump(Name, Arity, "lbody:~n~p~n",[B3]), - {{#c_var{name={Name,Arity}},B3},Ws} + {{#c_var{name={Name,Arity}},B3},Ws,LoadNif} catch Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), @@ -859,6 +880,9 @@ expr({call,L,{remote,_,M0,F0},As0}, St0) -> name=#c_literal{val=match_fail}, args=[Tuple]}, {Fail,Aps,St1}; + {#c_literal{val=erlang},#c_literal{val=load_nif},[_,_]} -> + {#icall{anno=#a{anno=Anno},module=M1,name=F1,args=As1}, + Aps,St1#core{load_nif=true}}; {_,_,_} -> {#icall{anno=#a{anno=Anno},module=M1,name=F1,args=As1},Aps,St1} end; @@ -1516,7 +1540,7 @@ verify_suitable_fields([]) -> ok. %% Count the number of bits approximately needed to store Int. %% (We don't need an exact result for this purpose.) -count_bits(Int) -> +count_bits(Int) when is_integer(Int) -> count_bits_1(abs(Int), 64). count_bits_1(0, Bits) -> Bits; @@ -2215,19 +2239,19 @@ string_to_conses(Line, Cs, Tail) -> make_vars(Vs) -> [ #c_var{name=V} || V <- Vs ]. -new_fun_name(#core{function={F,A},fcount=I}=St) -> +new_fun_name(#core{function={F,A},fcount=I}=St) when is_integer(I) -> Name = "-" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A) ++ "-fun-" ++ integer_to_list(I) ++ "-", {list_to_atom(Name),St#core{fcount=I+1}}. %% new_fun_name(Type, State) -> {FunName,State}. -new_fun_name(Type, #core{fcount=C}=St) -> +new_fun_name(Type, #core{fcount=C}=St) when is_integer(C) -> {list_to_atom(Type ++ "$^" ++ integer_to_list(C)),St#core{fcount=C+1}}. %% new_var_name(State) -> {VarName,State}. -new_var_name(#core{vcount=C}=St) -> +new_var_name(#core{vcount=C}=St) when is_integer(C) -> {C,St#core{vcount=C + 1}}. %% new_var(State) -> {{var,Name},State}. @@ -3030,6 +3054,9 @@ ren_is_subst(_V, []) -> no. %% from case/receive. In subblocks/clauses the AfterVars of the block %% are just the exported variables. +cbody(B0, none, St0) -> + {B1,_,_,St1} = cexpr(B0, [], St0), + {B1,St1}; cbody(B0, Nifs, St0) -> {B1,_,_,St1} = cexpr(B0, [], St0), B2 = case sets:is_element(St1#core.function,Nifs) of @@ -3878,6 +3905,18 @@ is_simple(_) -> false. is_simple_list(Es) -> lists:all(fun is_simple/1, Es). +insert_nif_start([VF={V,F=#c_fun{body=Body}}|Funs]) -> + case Body of + #c_seq{arg=#c_primop{name=#c_literal{val=nif_start}}} -> + [VF|insert_nif_start(Funs)]; + #c_case{} -> + NifStart = #c_primop{name=#c_literal{val=nif_start},args=[]}, + [{V,F#c_fun{body=#c_seq{arg=NifStart,body=Body}}} + |insert_nif_start(Funs)] + end; +insert_nif_start([]) -> + []. + %%% %%% Handling of warnings. %%% diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index 8e145ae136..4c4ec14e3e 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -115,6 +115,12 @@ R24= \ bs_utf \ bs_bincomp +R25= \ + bs_construct \ + bs_match \ + bs_utf \ + bs_bincomp + DIALYZER = bs_match CORE_MODULES = \ @@ -133,12 +139,16 @@ POST_OPT_MODULES= $(NO_OPT:%=%_post_opt_SUITE) POST_OPT_ERL_FILES= $(POST_OPT_MODULES:%=%.erl) NO_CORE_OPT_MODULES= $(NO_OPT:%=%_no_copt_SUITE) NO_CORE_OPT_ERL_FILES= $(NO_CORE_OPT_MODULES:%=%.erl) +NO_CORE_SSA_OPT_MODULES= $(NO_OPT:%=%_no_copt_ssa_SUITE) +NO_CORE_SSA_OPT_ERL_FILES= $(NO_CORE_SSA_OPT_MODULES:%=%.erl) INLINE_MODULES= $(INLINE:%=%_inline_SUITE) INLINE_ERL_FILES= $(INLINE_MODULES:%=%.erl) R23_MODULES= $(R23:%=%_r23_SUITE) R23_ERL_FILES= $(R23_MODULES:%=%.erl) R24_MODULES= $(R24:%=%_r24_SUITE) R24_ERL_FILES= $(R24_MODULES:%=%.erl) +R25_MODULES= $(R25:%=%_r25_SUITE) +R25_ERL_FILES= $(R25_MODULES:%=%.erl) NO_MOD_OPT_MODULES= $(NO_MOD_OPT:%=%_no_module_opt_SUITE) NO_MOD_OPT_ERL_FILES= $(NO_MOD_OPT_MODULES:%=%.erl) NO_SSA_OPT_MODULES= $(NO_SSA_OPT:%=%_no_ssa_opt_SUITE) @@ -179,9 +189,10 @@ EBIN = . DISABLE_SSA_OPT = +no_bool_opt +no_share_opt +no_bsm_opt +no_fun_opt +no_ssa_opt +no_recv_opt make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(NO_SSA_OPT_ERL_FILES) \ - $(NO_CORE_OPT_ERL_FILES) $(INLINE_ERL_FILES) $(R23_ERL_FILES) \ + $(NO_CORE_OPT_ERL_FILES) $(NO_CORE_SSA_OPT_ERL_FILES) \ + $(INLINE_ERL_FILES) $(R23_ERL_FILES) \ $(NO_MOD_OPT_ERL_FILES) $(NO_TYPE_OPT_ERL_FILES) \ - $(DIALYZER_ERL_FILES) $(R24_ERL_FILES) + $(DIALYZER_ERL_FILES) $(R24_ERL_FILES) $(R25_ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ > $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt $(DISABLE_SSA_OPT) +no_postopt \ @@ -192,12 +203,16 @@ make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(NO_SSA_OPT_ERL_FILES -o$(EBIN) $(POST_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(NO_CORE_OPT_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +no_copt +no_ssa_opt $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(NO_CORE_SSA_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +inline $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(INLINE_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +r23 $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(R23_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +r24 $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(R24_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +r25 $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(R25_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_module_opt $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(NO_MOD_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +from_core $(ERL_COMPILE_FLAGS) \ @@ -233,6 +248,9 @@ docs: %_no_copt_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_no_copt_ssa_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + %_inline_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ @@ -242,6 +260,9 @@ docs: %_r24_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_r25_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + %_no_module_opt_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ @@ -266,7 +287,9 @@ release_tests_spec: make_emakefile $(INLINE_ERL_FILES) \ $(R23_ERL_FILES) \ $(R24_ERL_FILES) \ + $(R25_ERL_FILES) \ $(NO_CORE_OPT_ERL_FILES) \ + $(NO_CORE_SSA_OPT_ERL_FILES) \ $(NO_MOD_OPT_ERL_FILES) \ $(NO_SSA_OPT_ERL_FILES) \ $(NO_TYPE_OPT_ERL_FILES) \ diff --git a/lib/compiler/test/beam_bounds_SUITE.erl b/lib/compiler/test/beam_bounds_SUITE.erl index 88deb704f3..cf8ac9673a 100644 --- a/lib/compiler/test/beam_bounds_SUITE.erl +++ b/lib/compiler/test/beam_bounds_SUITE.erl @@ -25,7 +25,12 @@ multiplication_bounds/1, division_bounds/1, rem_bounds/1, band_bounds/1, bor_bounds/1, bxor_bounds/1, bsr_bounds/1, bsl_bounds/1, - lt_bounds/1, le_bounds/1, gt_bounds/1, ge_bounds/1]). + bnot_bounds/1, + lt_bounds/1, le_bounds/1, gt_bounds/1, ge_bounds/1, + min_bounds/1, max_bounds/1, + abs_bounds/1, + infer_lt_gt_bounds/1, + redundant_masking/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -42,12 +47,18 @@ groups() -> band_bounds, bor_bounds, bxor_bounds, + bnot_bounds, bsr_bounds, bsl_bounds, lt_bounds, le_bounds, gt_bounds, - ge_bounds]}]. + ge_bounds, + min_bounds, + max_bounds, + abs_bounds, + infer_lt_gt_bounds, + redundant_masking]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -63,22 +74,75 @@ end_per_group(_GroupName, Config) -> Config. addition_bounds(_Config) -> - test_commutative('+', {-12,12}). + test_commutative('+', {-12,12}), + + {'-inf',-15} = beam_bounds:bounds('+', {'-inf',-20}, {2,5}), + {'-inf',55} = beam_bounds:bounds('+', {'-inf',50}, {'-inf',5}), + {'-inf',110} = beam_bounds:bounds('+', {1,10}, {'-inf',100}), + any = beam_bounds:bounds('+', {1,'+inf'}, {'-inf',100}), + + {-8,'+inf'} = beam_bounds:bounds('+', {2,'+inf'}, {-10,20}), + {6,'+inf'} = beam_bounds:bounds('+', {1,10}, {5,'+inf'}), + {9,'+inf'} = beam_bounds:bounds('+', {2,'+inf'}, {7,'+inf'}), + + ok. subtraction_bounds(_Config) -> - test_noncommutative('-', {-12,12}). + test_noncommutative('-', {-12,12}), + + {'-inf',18} = beam_bounds:bounds('-', {'-inf',20}, {2,9}), + any = beam_bounds:bounds('-', {'-inf',20}, {'-inf',17}), + {-99,'+inf'} = beam_bounds:bounds('-', {1,10}, {'-inf',100}), + {-93,'+inf'} = beam_bounds:bounds('-', {7,'+inf'}, {'-inf',100}), + + {-18,'+inf'} = beam_bounds:bounds('-', {2,'+inf'}, {-10,20}), + {'-inf',6} = beam_bounds:bounds('-', {1,11}, {5,'+inf'}), + any = beam_bounds:bounds('-', {2,'+inf'}, {7,'+inf'}), + + ok. multiplication_bounds(_Config) -> - test_commutative('*', {-12,12}). + test_commutative('*', {-12,12}), + + {'-inf',-40} = beam_bounds:bounds('*', {'-inf',-20}, {2,5}), + {'-inf',1000} = beam_bounds:bounds('*', {'-inf',100}, {1,10}), + any = beam_bounds:bounds('*', {'-inf',100}, {-10,10}), + + {-100,'+inf'} = beam_bounds:bounds('*', {-10,'+inf'}, {1,10}), + {7,'+inf'} = beam_bounds:bounds('*', {7,'+inf'}, {1,10}), + any = beam_bounds:bounds('*', {-10,'+inf'}, {-5,5}), + + {'-inf',1000} = beam_bounds:bounds('*', {1,10}, {'-inf',100}), + {-100,'+inf'} = beam_bounds:bounds('*', {1,10}, {-10,'+inf'}), + + ok. division_bounds(_Config) -> - test_noncommutative('div', {-12,12}). + test_noncommutative('div', {-12,12}), + + {'-inf',-5} = beam_bounds:bounds('div', {'-inf',-20}, {2,4}), + {'-inf',50} = beam_bounds:bounds('div', {'-inf',100}, {2,4}), + + {-5,'+inf'} = beam_bounds:bounds('div', {-10,'+inf'}, {2,4}), + {2,'+inf'} = beam_bounds:bounds('div', {10,'+inf'}, {2,4}), + + + ok. rem_bounds(_Config) -> test_noncommutative('rem', {-12,12}), - {-7,7} = beam_bounds:'rem'(any, {1,8}), - {-11,11} = beam_bounds:'rem'(any, {-12,8}), + {-7,7} = beam_bounds:bounds('rem', any, {1,8}), + {-11,11} = beam_bounds:bounds('rem', any, {-12,8}), + + {-7,7} = beam_bounds:bounds('rem', {'-inf',10}, {1,8}), + {0,7} = beam_bounds:bounds('rem', {10,'+inf'}, {1,8}), + + any = beam_bounds:bounds('rem', {1,10}, {'-inf',10}), + any = beam_bounds:bounds('rem', {1,10}, {10,'+inf'}), + + any = beam_bounds:bounds('rem', {-10,10}, {'-inf',10}), + any = beam_bounds:bounds('rem', {-10,10}, {10,'+inf'}), ok. @@ -86,36 +150,93 @@ band_bounds(_Config) -> test_commutative('band'), %% Coverage. - {0,17} = beam_bounds:'band'(any, {7,17}), - {0,42} = beam_bounds:'band'({0,42}, any), - any = beam_bounds:'band'({-1,1}, any), - any = beam_bounds:'band'(any, {-10,0}), - any = beam_bounds:'band'({-10,0},{-1,10}), - any = beam_bounds:'band'({-20,-10},{-1,10}), + {0,17} = beam_bounds:bounds('band', any, {7,17}), + {0,42} = beam_bounds:bounds('band', {0,42}, any), + any = beam_bounds:bounds('band', {-1,1}, any), + any = beam_bounds:bounds('band', any, {-10,0}), + any = beam_bounds:bounds('band', {-10,0}, {-1,10}), + any = beam_bounds:bounds('band', {-20,-10}, {-1,10}), ok. bor_bounds(_Config) -> test_commutative('bor'), - any = beam_bounds:'bor'({-10,0},{-1,10}), - any = beam_bounds:'bor'({-20,-10},{-1,10}), + any = beam_bounds:bounds('bor', {-10,0},{-1,10}), + any = beam_bounds:bounds('bor', {-20,-10}, {-1,10}), ok. bxor_bounds(_Config) -> test_commutative('bxor'), - any = beam_bounds:'bxor'({-10,0},{-1,10}), - any = beam_bounds:'bxor'({-20,-10},{-1,10}), + any = beam_bounds:bounds('bxor', {-10,0}, {-1,10}), + any = beam_bounds:bounds('bxor', {-20,-10}, {-1,10}), + + ok. + +bnot_bounds(_Config) -> + Min = -7, + Max = 7, + Seq = lists:seq(Min, Max), + _ = [bnot_bounds_1({A,B}) || + A <- Seq, + B <- lists:nthtail(A-Min, Seq)], + + {-43,'+inf'} = beam_bounds:bounds('bnot', {'-inf',42}), + {99,'+inf'} = beam_bounds:bounds('bnot', {'-inf',-100}), + {'-inf',-8} = beam_bounds:bounds('bnot', {7,'+inf'}), + {'-inf',9} = beam_bounds:bounds('bnot', {-10,'+inf'}), ok. +bnot_bounds_1(R) -> + {HighestMin,LowestMax} = min_max_unary_op('bnot', R), + {Min,Max} = beam_bounds:bounds('bnot', R), + if + Min =< HighestMin, LowestMax =< Max -> + ok; + true -> + io:format("bnot(~p) evaluates to ~p; should be ~p\n", + [R,{Min,Max},{HighestMin,LowestMax}]), + ct:fail(bad_min_or_max) + end. + bsr_bounds(_Config) -> - test_noncommutative('bsr', {-12,12}, {0,7}). + test_noncommutative('bsr', {-12,12}, {0,7}), + + {0,10} = beam_bounds:bounds('bsr', {0,10}, {0,'+inf'}), + {0,2} = beam_bounds:bounds('bsr', {0,10}, {2,'+inf'}), + + {-1,10} = beam_bounds:bounds('bsr', {-1,10}, {0,'+inf'}), + {-100,900} = beam_bounds:bounds('bsr', {-100,900}, {0,'+inf'}), + {-50,450} = beam_bounds:bounds('bsr', {-100,900}, {1,'+inf'}), + + {'-inf',16} = beam_bounds:bounds('bsr', {'-inf',32}, {1,10}), + {-5,'+inf'} = beam_bounds:bounds('bsr', {-10,'+inf'}, {1,10}), + + ok. bsl_bounds(_Config) -> - test_noncommutative('bsl', {-12,12}, {0,7}). + test_noncommutative('bsl', {-12,12}, {-7,7}), + + {2,'+inf'} = beam_bounds:bounds('bsl', {1,10}, {1,10_000}), + {0,'+inf'} = beam_bounds:bounds('bsl', {1,10}, {-10,10_000}), + any = beam_bounds:bounds('bsl', {-7,10}, {1,10_000}), + + any = beam_bounds:bounds('bsl', {-10,100}, {0,'+inf'}), + any = beam_bounds:bounds('bsl', {-10,100}, {1,'+inf'}), + any = beam_bounds:bounds('bsl', {-10,100}, {-1,'+inf'}), + + {0,10} = beam_bounds:bounds('bsl', {1,10}, {'-inf',0}), + {0,20} = beam_bounds:bounds('bsl', {1,10}, {'-inf',1}), + {-7,10} = beam_bounds:bounds('bsl', {-7,10}, {'-inf',0}), + {-28,40} = beam_bounds:bounds('bsl', {-7,10}, {'-inf',2}), + + {'-inf',-1} = beam_bounds:bounds('bsl', {-10,-1}, {500,1024}), + {0,'+inf'} = beam_bounds:bounds('bsl', {1,10}, {500,1024}), + + ok. lt_bounds(_Config) -> test_relop('<'). @@ -129,8 +250,98 @@ gt_bounds(_Config) -> ge_bounds(_Config) -> test_relop('>='). +min_bounds(_Config) -> + test_commutative(min, {-12,12}), + + {'-inf',-10} = min_bounds({'-inf',-10}, {1,100}), + {'-inf',1} = min_bounds({'-inf',1}, {1,100}), + {'-inf',50} = min_bounds({'-inf',50}, {1,100}), + {'-inf',100} = min_bounds({'-inf',500}, {1,100}), + + {'-inf',-10} = min_bounds({'-inf',-10}, {1,'+inf'}), + {'-inf',1} = min_bounds({'-inf',1}, {1,'+inf'}), + {'-inf',700} = min_bounds({'-inf',700}, {1,'+inf'}), + + {1,99} = min_bounds({1,99}, {100,'+inf'}), + {1,100} = min_bounds({1,100}, {100,'+inf'}), + {100,200} = min_bounds({150,200}, {100,'+inf'}), + + ok. + +min_bounds(R1, R2) -> + Result = beam_bounds:bounds(min, R1, R2), + Result = beam_bounds:bounds(min, R2, R1). + +max_bounds(_Config) -> + test_commutative(max, {-12,12}), + + {1,100} = max_bounds({'-inf',-10}, {1,100}), + {1,100} = max_bounds({'-inf',1}, {1,100}), + {1,100} = max_bounds({'-inf',50}, {1,100}), + {1,500} = max_bounds({'-inf',500}, {1,100}), + + {1,'+inf'} = max_bounds({'-inf',-10}, {1,'+inf'}), + {1,'+inf'} = max_bounds({'-inf',1}, {1,'+inf'}), + {1,'+inf'} = max_bounds({'-inf',700}, {1,'+inf'}), + + {100,'+inf'} = max_bounds({1,99}, {100,'+inf'}), + {100,'+inf'} = max_bounds({1,100}, {100,'+inf'}), + {150,'+inf'} = max_bounds({150,200}, {100,'+inf'}), + + ok. + +max_bounds(R1, R2) -> + Result = beam_bounds:bounds(max, R1, R2), + Result = beam_bounds:bounds(max, R2, R1). + +abs_bounds(_Config) -> + Min = -7, + Max = 7, + Seq = lists:seq(Min, Max), + _ = [abs_bounds_1({A,B}) || + A <- Seq, + B <- lists:nthtail(A-Min, Seq)], + ok. + +abs_bounds_1(R) -> + {HighestMin,LowestMax} = min_max_unary_op('abs', R), + {Min,Max} = beam_bounds:bounds(abs, R), + if + Min =< HighestMin, LowestMax =< Max -> + ok; + true -> + io:format("~p(~p) evaluates to ~p; should be ~p\n", + [bif_abs,R,{Min,Max},{HighestMin,LowestMax}]), + ct:fail(bad_min_or_max) + end. + +infer_lt_gt_bounds(_Config) -> + {{'-inf',-1}, {'-inf',0}} = infer_lt_gt({'-inf',0}, {'-inf',0}), + {{'-inf',1}, {'-inf',2}} = infer_lt_gt({'-inf',1}, {'-inf',2}), + {{'-inf',-2}, {'-inf',-1}} = infer_lt_gt({'-inf',1}, {'-inf',-1}), + {{'-inf',2}, {1,3}} = infer_lt_gt({'-inf',2}, {1,3}), + + any = infer_lt_gt({'-inf',2}, {3,10}), + any = infer_lt_gt({'-inf',2}, {3,'+inf'}), + + {{0,10}, {1,84}} = infer_lt_gt({0,10}, {'-inf',84}), + {{0,83}, {1,84}} = infer_lt_gt({0,'+inf'}, {'-inf',84}), + + {{0,'+inf'}, {42, '+inf'}} = infer_lt_gt({0,'+inf'}, {42, '+inf'}), + {{100,'+inf'}, {101, '+inf'}} = infer_lt_gt({100,'+inf'}, {42, '+inf'}), + + ok. + %%% Utilities +infer_lt_gt(R1, R2) -> + case beam_bounds:infer_relop_types('>', R2, R1) of + {Rb,Ra} -> + {Ra,Rb} = beam_bounds:infer_relop_types('<', R1, R2); + any -> + any = beam_bounds:infer_relop_types('<', R1, R2) + end. + test_commutative(Op) -> test_commutative(Op, {0,32}). @@ -146,8 +357,8 @@ test_commutative(Op, {Min,Max}) -> test_commutative_1(Op, R1, R2) -> {HighestMin,LowestMax} = min_max_op(Op, R1, R2), - {Min,Max} = beam_bounds:Op(R1, R2), - {Min,Max} = beam_bounds:Op(R2, R1), + {Min,Max} = beam_bounds:bounds(Op, R1, R2), + {Min,Max} = beam_bounds:bounds(Op, R2, R1), if Min =< HighestMin, LowestMax =< Max -> ok; @@ -156,6 +367,7 @@ test_commutative_1(Op, R1, R2) -> [Op,R1,R2,{Min,Max},{HighestMin,LowestMax}]), ct:fail(bad_min_or_max) end. + test_noncommutative(Op, Range) -> test_noncommutative(Op, Range, Range). @@ -171,7 +383,7 @@ test_noncommutative(Op, {Min1,Max1}, {Min2,Max2}) -> test_noncommutative_1(Op, R1, R2) -> {HighestMin,LowestMax} = min_max_op(Op, R1, R2), - case beam_bounds:Op(R1, R2) of + case beam_bounds:bounds(Op, R1, R2) of any -> case {Op,R2} of {'rem',{0,0}} -> ok @@ -206,6 +418,20 @@ min_max_op_2(Op, A, C, D, MinMax) when C =< D -> min_max_op_2(_Op, _, _, _, MinMax) -> MinMax. +min_max_unary_op(Op, {A,B}) -> + min_max_unary_op_1(Op, A, B, {infinity,-(1 bsl 24)}). + +min_max_unary_op_1(Op, A, B, {Min,Max}) when A =< B -> + Val = erlang:Op(A), + if + Min =< Val, Val =< Max -> + min_max_unary_op_1(Op, A + 1, B, {Min,Max}); + true -> + min_max_unary_op_1(Op, A + 1, B, {min(Min, Val),max(Max, Val)}) + end; +min_max_unary_op_1(_Op, _, _, MinMax) -> + MinMax. + test_relop(Op) -> Max = 15, Seq = lists:seq(0, Max), @@ -220,13 +446,43 @@ test_relop_1(Op, R1, R2) -> Bool = rel_op(Op, R1, R2), case beam_bounds:relop(Op, R1, R2) of Bool -> - ok; + test_infer_relop(Bool, Op, R1, R2); Wrong -> io:format("~p(~p, ~p) evaluates to ~p; should be ~p\n", [Op,R1,R2,Wrong,Bool]), ct:fail(bad_bool_result) end. +test_infer_relop(Bool, Op, R1, R2) when is_boolean(Bool) -> + any = beam_bounds:infer_relop_types(Op, R1, R2); +test_infer_relop('maybe', Op, {A0,B0}=R1, {C0,D0}=R2) -> + {{A,B},{C,D}} = beam_bounds:infer_relop_types(Op, R1, R2), + if + A =< B, C =< D, A0 =< A, B0 >= B, C0 =< C, D0 >= D -> + ok; + true -> + io:format("~p ~p infers as ~p ~p\n", + [R1,R2,{A,B},{C,D}]), + ct:fail(ranges_grew) + end, + _ = [begin + case in_range(X, {A,B}) andalso in_range(Y, {C,D}) of + true -> + ok; + false -> + io:format("X = ~p; Y = ~p\n", [X,Y]), + io:format("~p ~p infers as ~p ~p\n", + [R1,R2,{A,B},{C,D}]), + ct:fail(bad_inference) + end + end || X <- lists:seq(A0, B0), + Y <- lists:seq(C0, D0), + erlang:Op(X, Y)], + ok. + +in_range(Int, {A,B}) -> + A =< Int andalso Int =< B. + rel_op(Op, {A,B}, {C,D}) -> rel_op_1(Op, A, B, C, D, none). @@ -246,3 +502,37 @@ rel_op_2(Op, A, C, D, BoolResult0) when C =< D -> rel_op_2(Op, A, C + 1, D, BoolResult); rel_op_2(_Op, _, _, _, BoolResult) -> BoolResult. + +redundant_masking(_Config) -> + Min = -7, + Max = 15, + Seq = lists:seq(Min, Max), + _ = [test_redundant_masking({A,B}, M) || + A <- Seq, + B <- lists:nthtail(A-Min, Seq), + M <- Seq], + + false = beam_bounds:is_masking_redundant({'-inf',10}, 16#ff), + false = beam_bounds:is_masking_redundant({0,'+inf'}, 16#ff), + ok. + +test_redundant_masking({A,B}=R, M) -> + ShouldBe = test_redundant_masking(A, B, M), + case beam_bounds:is_masking_redundant(R, M) of + ShouldBe -> + ok; + false when M band (M + 1) =/= 0 -> + %% M + 1 is not a power of two. + ok; + false when A =:= B -> + ok; + Unexpected -> + io:format("beam_bounds:is_masking_redundant(~p, ~p) " + "evaluates to ~p; should be ~p\n", + [R,M,Unexpected,ShouldBe]), + ct:fail(bad_boolean) + end. + +test_redundant_masking(A, B, M) when A =< B -> + A band M =:= A andalso test_redundant_masking(A + 1, B, M); +test_redundant_masking(_, _, _) -> true. diff --git a/lib/compiler/test/beam_ssa_SUITE.erl b/lib/compiler/test/beam_ssa_SUITE.erl index d1330b78bf..5606f80ed2 100644 --- a/lib/compiler/test/beam_ssa_SUITE.erl +++ b/lib/compiler/test/beam_ssa_SUITE.erl @@ -886,6 +886,10 @@ grab_bag(_Config) -> {'EXIT',{{try_clause,[]},[_|_]}} = catch grab_bag_18(), + {'EXIT',{{badmatch,[whatever]},[_|_]}} = catch grab_bag_19(), + + {'EXIT',{if_clause,[_|_]}} = catch grab_bag_20(), + ok. grab_bag_1() -> @@ -1112,6 +1116,42 @@ grab_bag_18() -> end end. +grab_bag_19() -> + ([<<bad/utf8>>] = + %% beam_ssa_pre_codegen would produce single-valued phi + %% nodes, which in turn would cause the constant propagation + %% in beam_ssa_codegen:prefer_xregs/2 to produce get_hd and + %% get_tl instructions with literal operands. + try + [whatever] + catch + _:_ when false -> + ok + end) ! (some_atom ++ <<>>). + +grab_bag_20() -> + %% Similarly to grab_bag_19, beam_ssa_pre_codegen would produce + %% single-valued phi nodes. The fix for grab_bag_19 would not + %% suffice because several phi nodes were involved. + {[_ | _] = + receive + list -> + "list"; + 1 when day -> + [] + after + 0 -> + if + false -> + error + end + end, + try + ok + catch + error:_ -> + error + end}. redundant_br(_Config) -> {false,{x,y,z}} = redundant_br_1(id({x,y,z})), diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index ceb7ff07b3..8bdce54543 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -27,7 +27,9 @@ test_size/1,cover_lists_functions/1,list_append/1,bad_binary_unit/1, none_argument/1,success_type_oscillation/1,type_subtraction/1, container_subtraction/1,is_list_opt/1,connected_tuple_elements/1, - switch_fail_inference/1,failures/1]). + switch_fail_inference/1,failures/1, + cover_maps_functions/1,min_max_mixed_types/1, + not_equal/1]). %% Force id/1 to return 'any'. -export([id/1]). @@ -63,7 +65,10 @@ groups() -> is_list_opt, connected_tuple_elements, switch_fail_inference, - failures + failures, + cover_maps_functions, + min_max_mixed_types, + not_equal ]}]. init_per_suite(Config) -> @@ -105,6 +110,20 @@ integers(_Config) -> -693 = do_integers_9(id(7), id(1)), + 3 = do_integers_10(1, 2), + 10 = do_integers_10(-2, -5), + + {'EXIT',{badarith,_}} = catch do_integers_11(42), + {'EXIT',{badarith,_}} = catch do_integers_11({a,b}), + + {'EXIT',{system_limit,_}} = catch do_integers_12(42), + {'EXIT',{system_limit,_}} = catch do_integers_12([]), + + {'EXIT',{{badmatch,42},_}} = catch do_integers_13(-43), + {'EXIT',{{badmatch,0},_}} = catch do_integers_13(-1), + {'EXIT',{{badmatch,-1},_}} = catch do_integers_13(0), + {'EXIT',{{badmatch,-18},_}} = catch do_integers_13(17), + ok. do_integers_1(B0) -> @@ -176,6 +195,47 @@ do_integers_8() -> do_integers_9(X, Y) -> X * (-100 bor (Y band 1)). +do_integers_10(A, B) when is_integer(A), is_integer(B), A < 2, B < 5 -> + if + A < B -> A + B; + true -> A * B + end. + +do_integers_11(V) -> + true - V bsl []. + +do_integers_12(X) -> + (1 bsl (1 bsl 100)) + X. + +%% GH-6427. +do_integers_13(X) -> + try do_integers_13_1(<<X>>) of + _ -> error(should_fail) + catch + C:R:_ -> + try do_integers_13_2(X) of + _ -> error(should_fail) + catch + C:R:_ -> + try do_integers_13_3(X) of + _ -> error(should_fail) + catch + C:R:Stk -> + erlang:raise(C, R, Stk) + end + end + end. + +do_integers_13_1(<<X>>) -> + <<(X = bnot X)>>. + +do_integers_13_2(X) when is_integer(X), -64 < X, X < 64 -> + (X = bnot X) + 1. + +do_integers_13_3(X) when is_integer(X), -64 < X, X < 64 -> + X = bnot X, + X + 1. + numbers(_Config) -> Int = id(42), true = is_integer(Int), @@ -227,8 +287,16 @@ numbers(_Config) -> Meet1 = id(0) + -10.0, %Float. 10.0 = abs(Meet1), %Number. + %% Cover code in beam_call_types:beam_bounds_type/3. + ok = fcmp(0.0, 1.0), + error = fcmp(1.0, 0.0), + ok. +fcmp(0.0, 0.0) -> ok; +fcmp(F1, F2) when (F1 - F2) / F2 < 0.0000001 -> ok; +fcmp(_, _) -> error. + coverage(Config) -> {'EXIT',{badarith,_}} = (catch id(1) bsl 0.5), {'EXIT',{badarith,_}} = (catch id(2.0) bsl 2), @@ -276,6 +344,18 @@ coverage(Config) -> {'EXIT',{function_clause,_}} = catch coverage_3("a"), {'EXIT',{function_clause,_}} = catch coverage_3("b"), + Number = id(1), + if + 0 =< Number, Number < 10 -> + 0 = coverage_4(-1, Number), + 10 = coverage_4(0, Number), + 20 = coverage_4(1, Number), + 30 = coverage_4(2, Number) + end, + + {'EXIT',{badarg,_}} = catch false ++ true, + {'EXIT',{badarg,_}} = catch false -- true, + ok. coverage_1() -> @@ -295,6 +375,9 @@ coverage_2() -> coverage_3("a" = V) when is_function(V, false) -> 0. +coverage_4(X, Y) -> + 10 * (X + Y). + booleans(_Config) -> {'EXIT',{{case_clause,_},_}} = (catch do_booleans_1(42)), @@ -326,6 +409,11 @@ booleans(_Config) -> end, false = is_boolean(NotBool), + {'EXIT',{{case_clause,false},_}} = catch do_booleans_4(42), + {'EXIT',{{case_clause,true},_}} = catch do_booleans_4(a), + {'EXIT',{{case_clause,true},_}} = catch do_booleans_4(false), + {'EXIT',{{badmatch,true},_}} = catch do_booleans_4(true), + ok. do_booleans_1(B) -> @@ -354,6 +442,15 @@ do_booleans_3(NewContent, IsAnchor) -> error end. +do_booleans_4(X) -> + case is_atom(X) of + Y when X -> + false = Y, + 0 + end. + +-record(update_tuple_a, {a,b}). +-record(update_tuple_b, {a,b,c}). setelement(_Config) -> T0 = id({a,42}), @@ -375,8 +472,41 @@ setelement(_Config) -> end, {'EXIT',{badarg,_}} = catch setelement(Index1, {a,b,c}, y), + %% Cover some edge cases in beam_call_types:will_succeed/3 and + %% beam_call_types:types/3 + {y} = setelement(1, tuple_or_integer(0), y), + {y} = setelement(1, record_or_integer(0), y), + {'EXIT',{badarg,_}} = catch setelement(2, tuple_or_integer(id(0)), y), + {'EXIT',{badarg,_}} = catch setelement(2, tuple_or_integer(id(1)), y), + {'EXIT',{badarg,_}} = catch setelement(2, record_or_integer(id(0)), y), + {'EXIT',{badarg,_}} = catch setelement(2, record_or_integer(id(1)), y), + {'EXIT',{badarg,_}} = catch setelement(id(2), not_a_tuple, y), + + %% Cover some edge cases in beam_types:update_tuple/2 + {'EXIT',{badarg,_}} = catch setelement(2, not_a_tuple, y), + {'EXIT',{badarg,_}} = catch setelement(not_an_index, {a,b,c}, y), + {'EXIT',{badarg,_}} = catch setelement(8, {out_of_range}, y), + {y,_,_} = update_tuple_1(#update_tuple_a{}, y), + {y,_,_,_} = update_tuple_1(#update_tuple_b{}, y), + #update_tuple_a{a=y} = update_tuple_2(#update_tuple_a{}, y), + #update_tuple_b{a=y} = update_tuple_2(#update_tuple_b{}, y), + {'EXIT',{badarg,_}} = catch update_tuple_3(id(#update_tuple_a{}), y), + {'EXIT',{badarg,_}} = catch update_tuple_3(id(#update_tuple_b{}), y), + {'EXIT',{badarg,_}} = catch update_tuple_4(id(#update_tuple_a{}), y), + #update_tuple_b{c=y} = update_tuple_4(id(#update_tuple_b{}), y), + ok. +record_or_integer(0) -> + {tuple}; +record_or_integer(N) when is_integer(N) -> + N. + +tuple_or_integer(0) -> + {id(tuple)}; +tuple_or_integer(N) when is_integer(N) -> + N. + do_setelement_1(<<N:32>>, Tuple, NewValue) -> _ = element(N, Tuple), %% While updating the type for Tuple, beam_ssa_type would do: @@ -389,6 +519,36 @@ do_setelement_2(<<N:1>>, Tuple, NewValue) -> two = element(2, Tuple), setelement(N, Tuple, NewValue). +update_tuple_1(Tuple, Value0) -> + Value = case Tuple of + #update_tuple_a{} -> Value0; + #update_tuple_b{} -> Value0 + end, + setelement(1, Tuple, Value). + +update_tuple_2(Tuple, Value0) -> + Value = case Tuple of + #update_tuple_a{} -> Value0; + #update_tuple_b{} -> Value0 + end, + setelement(2, Tuple, Value). + +update_tuple_3(Tuple, Value0) -> + Value = case Tuple of + #update_tuple_a{} -> Value0; + #update_tuple_b{} -> Value0 + end, + setelement(47, Tuple, Value). + +update_tuple_4(Tuple, Value0) -> + Value = case Tuple of + #update_tuple_a{} -> Value0; + #update_tuple_b{} -> Value0 + end, + %% #update_tuple_a{} is three elements long, so this should only work for + %% #update_tuple_b{}. + setelement(4, Tuple, Value). + cons(_Config) -> [did] = cons(assigned, did), @@ -623,8 +783,21 @@ do_test_size(Term) when is_binary(Term) -> size(Term). cover_lists_functions(Config) -> + foo = lists:foldl(id(fun(_, _) -> foo end), foo, Config), + foo = lists:foldl(fun(_, _) -> foo end, foo, Config), + {'EXIT',_} = catch lists:foldl(not_a_fun, foo, Config), + + foo = lists:foldr(id(fun(_, _) -> foo end), foo, Config), + foo = lists:foldr(fun(_, _) -> foo end, foo, Config), + {'EXIT',_} = catch lists:foldr(not_a_fun, foo, Config), + {data_dir,_DataDir} = lists:keyfind(data_dir, id(1), Config), + {'EXIT',_} = catch lists:keyfind(data_dir, not_a_position, Config), + {'EXIT',_} = catch lists:keyfind(data_dir, 1, not_a_list), + {'EXIT',_} = catch lists:map(not_a_fun, Config), + {'EXIT',_} = catch lists:map(not_a_fun, []), + {'EXIT',_} = catch lists:map(fun id/1, not_a_list), Config = lists:map(id(fun id/1), Config), case lists:suffix([no|Config], Config) of @@ -634,6 +807,9 @@ cover_lists_functions(Config) -> ok end, + [] = lists:zip([], []), + {'EXIT',_} = (catch lists:zip(not_list, [b])), + Zipper = fun(A, B) -> {A,B} end, [] = lists:zipwith(Zipper, [], []), @@ -648,11 +824,17 @@ cover_lists_functions(Config) -> Zipped), [{zip_zip,{zip,_}}|_] = DoubleZip, + {'EXIT',_} = (catch lists:zipwith(not_a_fun, [a], [b])), + {'EXIT',{bad,_}} = (catch lists:zipwith(fun(_A, _B) -> error(bad) end, + [a], [b])), + {'EXIT',_} = (catch lists:zipwith(fun(_A, _B) -> error(bad) end, + not_list, [b])), {'EXIT',{bad,_}} = (catch lists:zipwith(fun(_A, _B) -> error(bad) end, lists:duplicate(length(Zipped), zip_zip), Zipped)), - [{zip_zip,{zip,_}}|_] = DoubleZip, + {'EXIT',_} = catch lists:unzip(not_a_list), + {'EXIT',_} = catch lists:unzip([not_a_tuple]), {[_|_],[_|_]} = lists:unzip(Zipped), ok. @@ -927,5 +1109,100 @@ failures_1([] = V1, V2, V3) -> %% beam_clean to crash. {V1 - V3, (V1 = V2) - V3}. +%% Covers various edge cases in beam_call_types:types/3 relating to maps +cover_maps_functions(_Config) -> + {'EXIT',_} = catch maps:filter(fun(_, _) -> true end, not_a_map), + {'EXIT',_} = catch maps:filter(not_a_predicate, #{}), + + error = maps:find(key_not_present, #{}), + + {'EXIT',_} = catch maps:fold(fun(_, _, _) -> true end, init, not_a_map), + {'EXIT',_} = catch maps:fold(not_a_fun, init, #{}), + + #{} = maps:from_keys([], gurka), + #{ hello := gurka } = maps:from_keys([hello], gurka), + {'EXIT',_} = catch maps:from_keys(not_a_list, gurka), + + #{} = catch maps:from_list([]), + {'EXIT',_} = catch maps:from_list([not_a_tuple]), + + default = maps:get(key_not_present, #{}, default), + {'EXIT',_} = catch maps:get(key_not_present, #{}), + + [] = maps:keys(#{}), + {'EXIT',_} = catch maps:keys(not_a_map), + + #{ a := ok } = catch maps:map(fun(_, _) -> ok end, #{ a => a }), + {'EXIT',_} = catch maps:map(fun(_, _) -> error(crash) end, #{ a => a }), + {'EXIT',_} = catch maps:map(not_a_fun, #{}), + {'EXIT',_} = catch maps:map(fun(_, _) -> ok end, not_a_map), + + {'EXIT',_} = catch maps:merge(not_a_map, #{}), + + #{} = maps:new(), + + {'EXIT',_} = catch maps:put(key, value, not_a_map), + + #{} = maps:remove(a, #{ a => a }), + {'EXIT',_} = catch maps:remove(gurka, not_a_map), + + error = maps:take(key_not_present, #{}), + {'EXIT',_} = catch maps:take(key, not_a_map), + + {'EXIT',_} = catch maps:to_list(not_a_map), + + #{ a := ok } = maps:update_with(a, fun(_) -> ok end, #{ a => a }), + {'EXIT',_} = catch maps:update_with(a, fun(_) -> error(a) end, #{ a => a }), + {'EXIT',_} = catch maps:update_with(key_not_present, fun(_) -> ok end, #{}), + {'EXIT',_} = catch maps:update_with(key, not_a_fun, not_a_map), + + [] = maps:values(#{}), + {'EXIT',_} = catch maps:values(not_a_map), + + #{} = maps:with([key_not_present], #{}), + {'EXIT',_} = catch maps:with(not_a_list, #{}), + {'EXIT',_} = catch maps:with([], not_a_map), + {'EXIT',_} = catch maps:with([foobar], not_a_map), + + {'EXIT',_} = catch maps:without(not_a_list, #{}), + {'EXIT',_} = catch maps:without([], not_a_map), + + ok. + +%% The types for erlang:min/2 and erlang:max/2 were wrong, assuming that the +%% result was a float if either argument was a float. +min_max_mixed_types(_Config) -> + NotFloatA = min(id(12), 100.0), + id(NotFloatA * 0.5), + + NotFloatB = max(id(12.0), 100), + id(NotFloatB * 0.5), + + %% Cover more of the type analysis code. + 1 = id(min(id(0)+1, 42)), + -10 = id(min(id(0)+1, -10)), + 43 = id(max(3, id(42)+1)), + 42 = id(max(-99, id(41)+1)), + + ok. + +%% GH-6183. beam_validator had a stronger type analysis for '=/=' and +%% is_ne_exact than the beam_ssa_type pass. It would figure out that +%% at the time the comparison 'a /= V' was evaluated, V must be equal +%% to 'true' and the comparison would therefore always return 'false'. +%% beam_validator would report that as a type conflict. + +not_equal(_Config) -> + true = do_not_equal(true), + {'EXIT',{function_clause,_}} = catch do_not_equal(false), + {'EXIT',{function_clause,_}} = catch do_not_equal(0), + {'EXIT',{function_clause,_}} = catch do_not_equal(42), + {'EXIT',{function_clause,_}} = catch do_not_equal(self()), + + ok. + +do_not_equal(V) when (V / V < (V orelse true)); V; V -> + (V = (a /= V)) orelse 0. + id(I) -> I. diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index a6dd65a540..ee43c81a2f 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -40,7 +40,8 @@ receive_marker/1,safe_instructions/1, missing_return_type/1,will_succeed/1, bs_saved_position_units/1,parent_container/1, - container_performance/1]). + container_performance/1, + infer_relops/1]). -include_lib("common_test/include/ct.hrl"). @@ -75,7 +76,7 @@ groups() -> receive_marker,safe_instructions, missing_return_type,will_succeed, bs_saved_position_units,parent_container, - container_performance]}]. + container_performance,infer_relops]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -1035,5 +1036,24 @@ container_performance(Config) -> _ -> ok end. +%% Type inference was half-broken for relational operators, being implemented +%% for is_lt/is_ge instructions but not the {bif,RelOp} form. +infer_relops(_Config) -> + [lt = infer_relops_1(N) || N <- lists:seq(0,3)], + [ge = infer_relops_1(N) || N <- lists:seq(4,7)], + ok. + +infer_relops_1(N) -> + true = N >= 0, + Below4 = N < 4, + id(N), %% Force Below4 to use the {bif,'<'} form instead of is_lt + case Below4 of + true -> infer_relops_true(Below4, N); + false -> infer_relops_false(Below4, N) + end. + +infer_relops_true(_, _) -> lt. +infer_relops_false(_, _) -> ge. + id(I) -> I. diff --git a/lib/compiler/test/bs_bincomp_SUITE.erl b/lib/compiler/test/bs_bincomp_SUITE.erl index c3324b64dc..488cebf661 100644 --- a/lib/compiler/test/bs_bincomp_SUITE.erl +++ b/lib/compiler/test/bs_bincomp_SUITE.erl @@ -24,6 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, + verify_highest_opcode/1, byte_aligned/1,bit_aligned/1,extended_byte_aligned/1, extended_bit_aligned/1,mixed/1,filters/1,trim_coverage/1, nomatch/1,sizes/1,general_expressions/1, @@ -33,13 +34,14 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [byte_aligned, bit_aligned, extended_byte_aligned, +all() -> + [verify_highest_opcode, + byte_aligned, bit_aligned, extended_byte_aligned, extended_bit_aligned, mixed, filters, trim_coverage, nomatch, sizes, general_expressions, no_generator, zero_pattern, multiple_segments]. -groups() -> +groups() -> []. init_per_suite(Config) -> @@ -55,6 +57,28 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_construct_r24_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 176 -> + ok; + TooHigh -> + ct:fail({too_high_opcode,TooHigh}) + end; + bs_construct_r25_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 180 -> + ok; + TooHigh -> + ct:fail({too_high_opcode,TooHigh}) + end; + _ -> + ok + end. + byte_aligned(Config) when is_list(Config) -> cs_init(), <<"abcdefg">> = cs(<< <<(X+32)>> || <<X>> <= <<"ABCDEFG">> >>), @@ -603,6 +627,8 @@ cs(Bin) -> ok; bs_bincomp_no_ssa_opt_SUITE -> ok; + bs_bincomp_no_copt_ssa_SUITE -> + ok; bs_bincomp_post_opt_SUITE -> ok; _ -> diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index 201b411656..cecc3660d4 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -78,7 +78,15 @@ verify_highest_opcode(_Config) -> Highest when Highest =< 176 -> ok; TooHigh -> - ct:fail({too_high_opcode_for_21,TooHigh}) + ct:fail({too_high_opcode,TooHigh}) + end; + bs_construct_r25_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 180 -> + ok; + TooHigh -> + ct:fail({too_high_opcode,TooHigh}) end; _ -> ok diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index b477109f45..7fe6d89883 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -49,7 +49,8 @@ bad_phi_paths/1,many_clauses/1, combine_empty_segments/1,hangs_forever/1, bs_saved_position_units/1,empty_matches/1, - trim_bs_start_match_resume/1]). + trim_bs_start_match_resume/1, + gh_6410/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -89,7 +90,8 @@ groups() -> exceptions_after_match_failure,bad_phi_paths, many_clauses,combine_empty_segments,hangs_forever, bs_saved_position_units,empty_matches, - trim_bs_start_match_resume]}]. + trim_bs_start_match_resume, + gh_6410]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -113,10 +115,10 @@ end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> verify_highest_opcode(_Config) -> case ?MODULE of - bs_match_r21_SUITE -> + bs_match_r25_SUITE -> {ok,Beam} = file:read_file(code:which(?MODULE)), case test_lib:highest_opcode(Beam) of - Highest when Highest =< 163 -> + Highest when Highest =< 180 -> ok; TooHigh -> ct:fail({too_high_opcode_for_21,TooHigh}) @@ -1406,6 +1408,13 @@ bad_size(Config) when is_list(Config) -> true = bad_size_1(<<0>>), error = bad_size_1(<<0,1>>), + [] = bad_size_2([a]), + [] = bad_size_2([<<1,2,3>>]), + {'EXIT',{{bad_generator,no_list},_}} = catch bad_size_2(no_list), + + error = bad_size_3(<<>>), + error = bad_size_3(<<0>>), + ok. bad_all_size(Bin) -> @@ -1465,6 +1474,19 @@ bad_size_1(<<0>>) -> true; bad_size_1(<<0:[]>>) -> false; bad_size_1(_) -> error. +-record(rec_bad_size_2, {a}). + +bad_size_2(L) -> + [ + ok || + <<_:(bad#rec_bad_size_2.a)/float>> <- L + ]. + +bad_size_3(<<0:((bnot (1 div 0)))>>) -> + ok; +bad_size_3(_) -> + error. + haystack(Config) when is_list(Config) -> <<0:10/unit:8>> = haystack_1(<<0:10/unit:8>>), [<<0:10/unit:8>>, @@ -2491,143 +2513,95 @@ trim_bs_start_match_resume_1(<<Context/binary>>) -> _ = id(Context), Context. -id(I) -> I. - expand_and_squeeze(Config) when is_list(Config) -> %% UTF8 literals are expanded and then squeezed into integer16 - [ - {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<$á/utf8,_/binary>>"), - ?Q("<<$é/utf8,_/binary>>") - ]), + ensure_squeezed(16, [?Q("<<$á/utf8,_/binary>>"), + ?Q("<<$é/utf8,_/binary>>")]), %% Sized integers are expanded and then squeezed into integer16 - [ - {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<0:32,_/binary>>"), - ?Q("<<\"bbbb\",_/binary>>") - ]), + ensure_squeezed(16, [?Q("<<0:32,_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>")]), %% Groups of 8 bits are squeezed into integer16 - [ - {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<\"aaaa\",_/binary>>"), - ?Q("<<\"bbbb\",_/binary>>") - ]), + ensure_squeezed(16, [?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>")]), %% Groups of 8 bits with empty binary are also squeezed - [ - {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<\"aaaa\",_/binary>>"), - ?Q("<<\"bbbb\",_/binary>>"), - ?Q("<<>>") - ]), + ensure_squeezed(16, [?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>"), + ?Q("<<>>")]), %% Groups of 8 bits with float lookup are not squeezed - [ - {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<\"aaaa\",_/binary>>"), - ?Q("<<\"bbbb\",_/binary>>"), - ?Q("<<_/float>>") - ]), + ensure_squeezed(8, [?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>"), + ?Q("<<_/float>>")]), %% Groups of diverse bits go with minimum possible - [ - {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<\"aa\",_/binary>>"), - ?Q("<<\"bb\",_/binary>>"), - ?Q("<<\"c\",_/binary>>") - ]), + ensure_squeezed(8, [?Q("<<\"aa\",_/binary>>"), + ?Q("<<\"bb\",_/binary>>"), + ?Q("<<\"c\",_/binary>>")]), %% Groups of diverse bits go with minimum possible but are recursive... - [ - {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} - | RestDiverse - ] = binary_match_to_asm([ - ?Q("<<\"aaa\",_/binary>>"), - ?Q("<<\"abb\",_/binary>>"), - ?Q("<<\"c\",_/binary>>") - ]), - - %% so we still perform a 16 bits lookup for the remaining - true = lists:any(fun({test,bs_get_integer2,_,_,[_,{integer,16}|_],_}) -> true; - (_) -> false end, RestDiverse), + [{bs_match,{f,_},_Ctx, + {commands,[{ensure_at_least,Size,1}, + {integer,_Live,_Flags,Size,1,_Dst}]}} | RestDiverse] = + binary_match_to_asm([?Q("<<\"aaa\",_/binary>>"), + ?Q("<<\"abb\",_/binary>>"), + ?Q("<<\"c\",_/binary>>")]), + + %% ... so we still perform a 16 bits lookup for the remaining + F = fun({bs_match,{f,_},_, + {commands,[{ensure_at_least,16,1}, + {integer,_Live,_Flags,16,1,_Dst}]}}) -> + true; + (_) -> false + end, + true = lists:any(F, RestDiverse), %% Large match is kept as is if there is a sized match later - [ - {test,bs_get_integer2,_,_,[_,{integer,64}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<255,255,255,255,255,255,255,255>>"), - ?Q("<<_:64>>") - ]), + ensure_squeezed(64, [?Q("<<255,255,255,255,255,255,255,255>>"), + ?Q("<<_:64>>")]), %% Large match is kept as is with large matches before and after - [ - {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<A:32,_:A>>"), - ?Q("<<0:32>>"), - ?Q("<<_:32>>") - ]), + ensure_squeezed(32, [?Q("<<A:32,_:A>>"), + ?Q("<<0:32>>"), + ?Q("<<_:32>>")]), %% Large match is kept as is with large matches before and after - [ - {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<A:32,_:A>>"), - ?Q("<<0,0,0,0>>"), - ?Q("<<_:32>>") - ]), + ensure_squeezed(32, [?Q("<<A:32,_:A>>"), + ?Q("<<0,0,0,0>>"), + ?Q("<<_:32>>")]), %% Large match is kept as is with smaller but still large matches before and after - [ - {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<A:32, _:A>>"), - ?Q("<<0:64>>"), - ?Q("<<_:32>>") - ]), + ensure_squeezed(32, [?Q("<<A:32, _:A>>"), + ?Q("<<0:64>>"), + ?Q("<<_:32>>")]), %% There is no squeezing for groups with more than 16 matches - [ - {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} - | _ - ] = binary_match_to_asm([ - ?Q("<<\"aa\", _/binary>>"), - ?Q("<<\"bb\", _/binary>>"), - ?Q("<<\"cc\", _/binary>>"), - ?Q("<<\"dd\", _/binary>>"), - ?Q("<<\"ee\", _/binary>>"), - ?Q("<<\"ff\", _/binary>>"), - ?Q("<<\"gg\", _/binary>>"), - ?Q("<<\"hh\", _/binary>>"), - ?Q("<<\"ii\", _/binary>>"), - ?Q("<<\"jj\", _/binary>>"), - ?Q("<<\"kk\", _/binary>>"), - ?Q("<<\"ll\", _/binary>>"), - ?Q("<<\"mm\", _/binary>>"), - ?Q("<<\"nn\", _/binary>>"), - ?Q("<<\"oo\", _/binary>>"), - ?Q("<<\"pp\", _/binary>>") - ]), - - ok. + ensure_squeezed(8, [?Q("<<\"aa\", _/binary>>"), + ?Q("<<\"bb\", _/binary>>"), + ?Q("<<\"cc\", _/binary>>"), + ?Q("<<\"dd\", _/binary>>"), + ?Q("<<\"ee\", _/binary>>"), + ?Q("<<\"ff\", _/binary>>"), + ?Q("<<\"gg\", _/binary>>"), + ?Q("<<\"hh\", _/binary>>"), + ?Q("<<\"ii\", _/binary>>"), + ?Q("<<\"jj\", _/binary>>"), + ?Q("<<\"kk\", _/binary>>"), + ?Q("<<\"ll\", _/binary>>"), + ?Q("<<\"mm\", _/binary>>"), + ?Q("<<\"nn\", _/binary>>"), + ?Q("<<\"oo\", _/binary>>"), + ?Q("<<\"pp\", _/binary>>")]), + + ok. + +ensure_squeezed(ExpectedSize, Fields) -> + [{bs_match,{f,_},_, + {commands,[{ensure_at_least,ExpectedSize,1}, + {integer,_Live,_Flags,ExpectedSize,1,_Dst}]}} | _] = + binary_match_to_asm(Fields). binary_match_to_asm(Matches) -> Clauses = [ @@ -2676,3 +2650,26 @@ many_clauses(_Config) -> one_clause(I) -> ?Q(<<"{_@I@,<<L:8,Val:L>>} -> _@I@ + Val">>). + +%% GH-6410: Fix crash in beam_ssa_bsm. +gh_6410(_Config) -> + 0 = do_gh_6410(<<42>>), + {'EXIT',{{case_clause,<<>>},[_|_]}} = catch do_gh_6410(<<>>), + {'EXIT',{{case_clause,a},[_|_]}} = catch do_gh_6410(a), + {'EXIT',{badarith,[_|_]}} = catch do_gh_6410([]), + + ok. + +do_gh_6410(<<_>>) -> + 0; +do_gh_6410(X) -> + +(case X of + <<_>> -> + X; + [] -> + X + end). + +%%% Utilities. +id(I) -> I. + diff --git a/lib/compiler/test/bs_utf_SUITE.erl b/lib/compiler/test/bs_utf_SUITE.erl index 10954dd833..427499b995 100644 --- a/lib/compiler/test/bs_utf_SUITE.erl +++ b/lib/compiler/test/bs_utf_SUITE.erl @@ -20,8 +20,9 @@ -module(bs_utf_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, + verify_highest_opcode/1, utf8_roundtrip/1,unused_utf_char/1,utf16_roundtrip/1, utf32_roundtrip/1,guard/1,extreme_tripping/1, literals/1,coverage/1]). @@ -30,12 +31,13 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [utf8_roundtrip, unused_utf_char, utf16_roundtrip, +all() -> + [verify_highest_opcode, + utf8_roundtrip, unused_utf_char, utf16_roundtrip, utf32_roundtrip, guard, extreme_tripping, literals, coverage]. -groups() -> +groups() -> []. init_per_suite(Config) -> @@ -51,6 +53,27 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_construct_r24_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 176 -> + ok; + TooHigh -> + ct:fail({too_high_opcode,TooHigh}) + end; + bs_construct_r25_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 180 -> + ok; + TooHigh -> + ct:fail({too_high_opcode,TooHigh}) + end; + _ -> + ok + end. utf8_roundtrip(Config) when is_list(Config) -> [utf8_roundtrip_1(P) || P <- utf_data()], diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl index 81f4e9b35c..63bfdec2be 100644 --- a/lib/compiler/test/compilation_SUITE.erl +++ b/lib/compiler/test/compilation_SUITE.erl @@ -402,7 +402,7 @@ string_table(Config) when is_list(Config) -> File = filename:join(DataDir, "string_table.erl"), {ok,string_table,Beam,[]} = compile:file(File, [return, binary]), {ok,{string_table,[StringTableChunk]}} = beam_lib:chunks(Beam, ["StrT"]), - {"StrT", <<"stringtable">>} = StringTableChunk, + {"StrT", <<"abcdefghiABCDEFGHI">>} = StringTableChunk, ok. otp_8949_a(Config) when is_list(Config) -> diff --git a/lib/compiler/test/compilation_SUITE_data/string_table.erl b/lib/compiler/test/compilation_SUITE_data/string_table.erl index 1da1d015dd..57d0fc579c 100644 --- a/lib/compiler/test/compilation_SUITE_data/string_table.erl +++ b/lib/compiler/test/compilation_SUITE_data/string_table.erl @@ -1,8 +1,8 @@ -module(string_table). -export([f/1, g/1]). -f(<<"string">>) -> string; -f(<<"stringtable">>) -> stringtable. +f(<<"abcdefghi">>) -> string; +f(<<"abcdefghiABCDEFGHI">>) -> stringtable. -g(<<"stringtable">>) -> stringtable; -g(<<"table">>) -> table. +g(<<"abcdefghiABCDEFGHI">>) -> stringtable; +g(<<"ABCDEFGHI">>) -> table. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index cbe7179e0a..79bd7b9a91 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -38,7 +38,7 @@ warnings/1, pre_load_check/1, env_compiler_options/1, bc_options/1, deterministic_include/1, deterministic_paths/1, compile_attribute/1, message_printing/1, other_options/1, - transforms/1, erl_compile_api/1, types_pp/1 + transforms/1, erl_compile_api/1, types_pp/1, bs_init_writable/1 ]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -58,7 +58,7 @@ all() -> env_compiler_options, custom_debug_info, bc_options, custom_compile_info, deterministic_include, deterministic_paths, compile_attribute, message_printing, other_options, transforms, - erl_compile_api, types_pp]. + erl_compile_api, types_pp, bs_init_writable]. groups() -> []. @@ -1719,7 +1719,8 @@ bc_options(Config) -> {168, small, [r22]}, {168, small, [no_init_yregs,no_shared_fun_wrappers, no_ssa_opt_record,no_make_fun3, - no_ssa_opt_float,no_line_info,no_type_opt]}, + no_ssa_opt_float,no_line_info,no_type_opt, + no_bs_match]}, {169, small, [r23]}, {169, big, [no_init_yregs,no_shared_fun_wrappers, @@ -2055,9 +2056,9 @@ types_pp(Config) when is_list(Config) -> "{any(), any(), any(), any(), any()}"}, {make_inexact_tuple, "{any(), any(), any(), ...}"}, {make_union, - "'foo' | nonempty_list(1..3) | number() |" - " {'tag0', 1, 2} | {'tag1', 3, 4} | bitstring(24)"}, - {make_bitstring, "bitstring(24)"}, + "'foo' | nonempty_list(1..3) | number(3, 7) |" + " {'tag0', 1, 2} | {'tag1', 3, 4} | bitstring(8)"}, + {make_bitstring, "bitstring(8)"}, {make_none, "none()"}], lists:foreach(fun({FunName, Expected}) -> Actual = map_get(atom_to_list(FunName), ResultTypes), @@ -2091,6 +2092,22 @@ get_result_types([CallLine|Lines], TypeLine, Acc) -> [_,Callee,_] = string:split(CallLine, "`", all), get_result_types(Lines, Acc#{ Callee => TypeLine }). +%% Check that the beam_ssa_type pass knows about bs_init_writable. +bs_init_writable(Config) -> + DataDir = proplists:get_value(data_dir, Config), + PrivDir = proplists:get_value(priv_dir, Config), + InFile = filename:join(DataDir, "bs_init_writable.erl"), + OutDir = filename:join(PrivDir, "bs_init_writable"), + OutFile = filename:join(OutDir, "bs_init_writable.S"), + ok = file:make_dir(OutDir), + {ok,bs_init_writable} = compile:file(InFile, ['S',{outdir,OutDir}]), + {ok,Listing} = file:read_file(OutFile), + Os = [global,multiline,{capture,all_but_first,list}], + %% The is_bitstr test should be optimized away. + nomatch = re:run(Listing, "({test,is_bitstr,.+})", Os), + %% The is_bitstr test should be optimized away. + nomatch = re:run(Listing, "({test,is_binary,.+})", Os), + ok = file:del_dir_r(OutDir). %%% diff --git a/lib/compiler/test/compile_SUITE_data/bs_init_writable.erl b/lib/compiler/test/compile_SUITE_data/bs_init_writable.erl new file mode 100644 index 0000000000..a9e061645f --- /dev/null +++ b/lib/compiler/test/compile_SUITE_data/bs_init_writable.erl @@ -0,0 +1,45 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2022. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(bs_init_writable). + +-export([do/0]). + +do() -> + Val = ex:foo(), + X = << <<B:1>> || B <- Val >>, + should_not_have_bitstring_test(X), + Y = << <<B:8>> || B <- Val >>, + should_not_have_binary_test(Y). + + +%% If the beam_ssa_type pass does its job, +%% should_not_have_bitstring_test/1 should not contain a is_bitstr test. +should_not_have_bitstring_test(X) when is_bitstring(X) -> + bitstring; +should_not_have_bitstring_test(_) -> + something_else. + +%% If the beam_ssa_type pass does its job, +%% should_not_have_binary_test/1 should not contain a is_binary test. +should_not_have_binary_test(X) when is_binary(X) -> + binary; +should_not_have_binary_test(_) -> + something_else. diff --git a/lib/compiler/test/core_SUITE.erl b/lib/compiler/test/core_SUITE.erl index a17ad9c6ad..6bc7b1442d 100644 --- a/lib/compiler/test/core_SUITE.erl +++ b/lib/compiler/test/core_SUITE.erl @@ -31,7 +31,7 @@ cover_v3_kernel_4/1,cover_v3_kernel_5/1, non_variable_apply/1,name_capture/1,fun_letrec_effect/1, get_map_element/1,receive_tests/1, - core_lint/1]). + core_lint/1,nif/1,no_nif/1,no_load_nif/1]). -include_lib("common_test/include/ct.hrl"). @@ -61,7 +61,7 @@ groups() -> cover_v3_kernel_4,cover_v3_kernel_5, non_variable_apply,name_capture,fun_letrec_effect, get_map_element,receive_tests, - core_lint + core_lint,nif,no_nif,no_load_nif ]}]. @@ -170,3 +170,53 @@ core_lint_function(Exports, Attributes, Body) -> (_) -> true end, Errors), error = compile:forms(Mod, [from_core,clint0,report]). + +nif(Conf) -> + %% Check that only the function in the nif attribute starts with nif_start + Funs = + nif_compile_to_cerl(Conf, [{d,'WITH_ATTRIBUTE'},{d,'WITH_LOAD_NIF'}]), + false = nif_first_instruction_is_nif_start(init, 1, Funs), + true = nif_first_instruction_is_nif_start(start, 1, Funs), + false = nif_first_instruction_is_nif_start(module_info, 0, Funs), + false = nif_first_instruction_is_nif_start(module_info, 1, Funs), + ok. + +no_nif(Conf) -> + %% Check that all functions start with nif_start + Funs = nif_compile_to_cerl(Conf, [{d,'WITH_LOAD_NIF'}]), + true = nif_first_instruction_is_nif_start(init, 1, Funs), + true = nif_first_instruction_is_nif_start(start, 1, Funs), + true = nif_first_instruction_is_nif_start(module_info, 0, Funs), + true = nif_first_instruction_is_nif_start(module_info, 1, Funs), + ok. + +no_load_nif(Conf) -> + %% Check that no functions start with nif_start + Funs = nif_compile_to_cerl(Conf, []), + false = nif_first_instruction_is_nif_start(init, 1, Funs), + false = nif_first_instruction_is_nif_start(start, 1, Funs), + false = nif_first_instruction_is_nif_start(module_info, 0, Funs), + false = nif_first_instruction_is_nif_start(module_info, 1, Funs), + ok. + +nif_compile_to_cerl(Conf, Flags) -> + Src = filename:join(proplists:get_value(data_dir, Conf), "nif.erl"), + {ok, _, F} = compile:file(Src, [to_core, binary, deterministic]++Flags), + Defs = cerl:module_defs(F), + [ {cerl:var_name(V),cerl:fun_body(Def)} || {V,Def} <- Defs]. + +nif_first_instruction_is_nif_start(F, A, [{{F,A},Body}|_]) -> + try + Primop = cerl:seq_arg(Body), + Name = cerl:primop_name(Primop), + 0 = cerl:primop_arity(Primop), + nif_start = cerl:atom_val(Name), + true + catch + error:_ -> + false + end; +nif_first_instruction_is_nif_start(F, A, [_|Rest]) -> + nif_first_instruction_is_nif_start(F, A, Rest); +nif_first_instruction_is_nif_start(_, _, []) -> + not_found. diff --git a/lib/compiler/test/core_SUITE_data/nif.erl b/lib/compiler/test/core_SUITE_data/nif.erl new file mode 100644 index 0000000000..873e20252b --- /dev/null +++ b/lib/compiler/test/core_SUITE_data/nif.erl @@ -0,0 +1,17 @@ +-module(nif). + +-export([init/1, start/1]). + +-ifdef(WITH_ATTRIBUTE). +-nifs([start/1]). +-endif. + +-ifdef(WITH_LOAD_NIF). +init(File) -> + ok = erlang:load_nif(File, 0). +-else. +init(_File) -> + ok. +-endif. + +start(_) -> erlang:nif_error(not_loaded). diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index 1a96fa4b6c..f03dcf6225 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -1360,6 +1360,23 @@ rb(_, _, _) -> false. rel_ops(Config) when is_list(Config) -> + TupleUnion = case id(2) of + 2 -> {a,id(b)}; + 3 -> {b,id(b),c} + end, + Float = float(id(42)), + Int = trunc(id(42.0)), + + IntFunFloat = make_fun(Float), + IntFunInt = make_fun(Int), + + FloatFun = make_fun(Float, Float), + IntFun = make_fun(Int, Int), + MixedFun = make_fun(42, 42.0), + + MixedFun14 = make_fun(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13.0, 14.0), + IntFun14 = make_fun(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14), + ?T(=/=, 1, 1.0), ?F(=/=, 2, 2), ?F(=/=, {a}, {a}), @@ -1368,9 +1385,50 @@ rel_ops(Config) when is_list(Config) -> ?F(/=, 0, 0.0), ?T(/=, 0, 1), ?F(/=, {a}, {a}), + ?F(/=, {a,b}, TupleUnion), + ?T(/=, {x,y}, TupleUnion), + ?T(/=, TupleUnion, {x,y}), + ?F(/=, #{key => Int}, #{key => Float}), + ?F(/=, #{key => Int}, #{key => Float}), + ?F(/=, #{40 => Int}, #{40 => Int}), + ?F(/=, #{42 => Float}, #{42 => Int}), + ?T(/=, #{100.0 => Float}, #{100 => Float}), + + ?F(/=, FloatFun, FloatFun), + ?T(/=, FloatFun, MixedFun14), + + ?T(==, Int, 42.0), + ?T(==, Float, 42), ?T(==, 1, 1.0), + ?T(==, 1.0, 1), + ?F(==, Float, a), + ?T(==, Float, Float), ?F(==, a, {}), + ?T(==, TupleUnion, {a,b}), + ?F(==, {x,y}, TupleUnion), + ?F(==, {a,Float}, TupleUnion), + + ?T(==, #{key => Float}, #{key => Int}), + ?T(==, #{40 => Int}, #{40 => Int}), + ?T(==, #{42 => Int}, #{42 => Float}), + ?F(==, #{100 => Float}, #{100.0 => Float}), + + case ?MODULE of + guard_inline_SUITE -> + %% Inlining will inline the fun environment into the fun bodies, + %% creating funs having no enviroment and different bodies. + ok; + _ -> + ?T(==, IntFunInt, IntFunFloat), + ?T(==, FloatFun, FloatFun), + ?T(==, FloatFun, IntFun), + ?T(==, MixedFun, IntFun), + ?T(==, MixedFun, FloatFun), + ?T(==, IntFun14, MixedFun14), + ?T(==, MixedFun14, IntFun14), + ?F(==, IntFun14, IntFun) + end, ?F(=:=, 1, 1.0), ?T(=:=, 42.0, 42.0), @@ -1427,6 +1485,17 @@ rel_ops(Config) when is_list(Config) -> ok. +make_fun(N) -> + fun() -> round(N + 0.5) end. + +make_fun(A, B) -> + fun() -> {A,B} end. + +make_fun(A, B, C, D, E, F, G, H, I, J, K, L, M, N) -> + fun() -> + {A, B, C, D, E, F, G, H, I, J, K, L, M, N} + end. + -undef(TestOp). rel_op_combinations(Config) when is_list(Config) -> diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index 4db5b01109..8716d85477 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -87,7 +87,8 @@ %% miscellaneous t_conflicting_destinations/1, - t_cse_assoc/1 + t_cse_assoc/1, + shared_key_tuples/1 ]). -define(badmap(V, F, Args), {'EXIT', {{badmap,V}, [{maps,F,Args,_}|_]}}). @@ -161,7 +162,8 @@ all() -> %% miscellaneous t_conflicting_destinations, - t_cse_assoc + t_cse_assoc, + shared_key_tuples ]. groups() -> []. @@ -2546,6 +2548,25 @@ do_cse_assoc(M, V) -> Assoc end. +shared_key_tuples(_Config) -> + A = decimal(0), + B = decimal(1), + + case ?MODULE of + map_inline_SUITE -> + %% With inlining, two separate map literals will be created. They + %% will not share keys. + ok; + _ -> + %% The two instances should share the key tuple. + true = erts_debug:same(erts_internal:map_to_tuple_keys(A), + erts_internal:map_to_tuple_keys(B)) + end, + ok. + +decimal(Int) -> + #{type => decimal, int => Int, exp => 0}. + %% aux rand_terms(0) -> []; diff --git a/lib/compiler/test/property_test/beam_types_prop.erl b/lib/compiler/test/property_test/beam_types_prop.erl index b70de8626b..1d8e3694c0 100644 --- a/lib/compiler/test/property_test/beam_types_prop.erl +++ b/lib/compiler/test/property_test/beam_types_prop.erl @@ -184,10 +184,11 @@ term_type(Depth) -> term_types(Depth) -> nested_generators(Depth) ++ numerical_generators() ++ - [gen_atom(), gen_bs_matchable()]. + [gen_atom(), gen_bs_matchable(), + pid, port, reference, other]. numerical_generators() -> - [gen_integer(), gen_float(), number]. + [gen_integer(), gen_float(), gen_number()]. nested_generators(Depth) when Depth =< 0 -> [nil]; @@ -221,8 +222,18 @@ gen_bs_matchable() -> gen_float() -> oneof([?LET({A, B}, {integer(), integer()}, begin - Min = float(min(A,B)), - Max = float(max(A,B)), + Min = float(min(A, B)), + Max = float(max(A, B)), + #t_float{elements={Min,Max}} + end), + ?LET({A, B, AExp, BExp}, + {integer(0, 1_000_000), integer(0, 10_00_000), + integer(-300, 300), integer(-300, 300)}, + begin + F1 = A * math:pow(10, AExp), + F2 = B * math:pow(10, BExp), + Min = min(F1, F2), + Max = max(F1, F2), #t_float{elements={Min,Max}} end), #t_float{}]). @@ -235,6 +246,10 @@ gen_fun(Depth) -> gen_integer() -> oneof([?LET({A, B}, {integer(), integer()}, #t_integer{elements={min(A,B), max(A,B)}}), + ?LET(Min, integer(), + #t_integer{elements={Min, '+inf'}}), + ?LET(Max, integer(), + #t_integer{elements={'-inf', Max}}), #t_integer{}]). gen_list(Depth) -> @@ -250,6 +265,15 @@ gen_map(Depth) -> #t_map{super_key=SKey,super_value=SValue}), [#t_map{}]). +gen_number() -> + oneof([?LET({A, B}, {integer(), integer()}, + #t_number{elements={min(A,B), max(A,B)}}), + ?LET(Min, integer(), + #t_number{elements={Min, '+inf'}}), + ?LET(Max, integer(), + #t_number{elements={'-inf', Max}}), + #t_number{}]). + gen_tuple(Depth) -> ?SHRINK(oneof([gen_tuple_plain(Depth), gen_tuple_record(Depth)]), [#t_tuple{}]). diff --git a/lib/compiler/test/record_SUITE.erl b/lib/compiler/test/record_SUITE.erl index cd60525691..f6c6c27965 100644 --- a/lib/compiler/test/record_SUITE.erl +++ b/lib/compiler/test/record_SUITE.erl @@ -699,6 +699,34 @@ grab_bag(_Config) -> error = T5(#gb_bar{}), error = T5(atom), + %% With type optimizations disabled, beam_ssa_pre_codegen would insert + %% set_tuple_element instructions between the call to setelement/3 and + %% its succeeded instruction. + T6 = fun(R) -> + try + %% The succeeded instruction should immediately follow its instruction. + %% Not like this: + %% + %% x0/_212 = call (`erlang`:`setelement`/3), `5`, y0/_87:37, `4` + %% z0/@ssa_dummy:34 = set_tuple_element `3`, x0/_212, `3` + %% z0/@ssa_dummy:35 = set_tuple_element `2`, x0/_212, `2` + %% z0/@ssa_dummy:36 = set_tuple_element `1`, x0/_212, `1` + %% z0/@ssa_bool:13 = succeeded x0/_212 + %% br z0/@ssa_bool:13, ^14, ^4 + R#foo{a=1,b=2,c=3,d=4} + of + 42 -> + ok + catch + _:_ -> + error + end + end, + error = catch T6(100), + error = catch T6([a,b,c]), + error = catch T6(#bar{}), + {'EXIT',{{try_clause,#foo{}},_}} = catch T6(#foo{}), + ok. %% ERIERL-436; the following code used to be very slow to compile. diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index c4d377e4f1..a6e75f232c 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -92,6 +92,7 @@ opt_opts(Mod) -> (inline) -> true; (no_bs_create_bin) -> true; (no_bsm_opt) -> true; + (no_bs_match) -> true; (no_copt) -> true; (no_fun_opt) -> true; (no_init_yregs) -> true; @@ -119,6 +120,7 @@ get_data_dir(Config) -> Opts = [{return,list}], Suffixes = ["_no_opt_SUITE", "_no_copt_SUITE", + "_no_copt_ssa_SUITE", "_post_opt_SUITE", "_inline_SUITE", "_no_module_opt_SUITE", @@ -138,6 +140,7 @@ is_cloned_mod(Mod) -> is_cloned_mod_1("_no_opt_SUITE") -> true; is_cloned_mod_1("_no_copt_SUITE") -> true; +is_cloned_mod_1("_no_copt_ssa_SUITE") -> true; is_cloned_mod_1("_no_ssa_opt_SUITE") -> true; is_cloned_mod_1("_no_type_opt_SUITE") -> true; is_cloned_mod_1("_post_opt_SUITE") -> true; diff --git a/lib/crypto/c_src/algorithms.c b/lib/crypto/c_src/algorithms.c index 53b574af3a..be19286509 100644 --- a/lib/crypto/c_src/algorithms.c +++ b/lib/crypto/c_src/algorithms.c @@ -29,7 +29,7 @@ #ifdef HAS_3_0_API #else static unsigned int algo_hash_cnt, algo_hash_fips_cnt; -static ERL_NIF_TERM algo_hash[14]; /* increase when extending the list */ +static ERL_NIF_TERM algo_hash[16]; /* increase when extending the list */ void init_hash_types(ErlNifEnv* env); #endif @@ -113,6 +113,12 @@ void init_hash_types(ErlNifEnv* env) { #ifdef HAVE_SHA3_512 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_512"); #endif +#ifdef HAVE_SHAKE128 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "shake128"); +#endif +#ifdef HAVE_SHAKE256 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "shake256"); +#endif #ifdef HAVE_BLAKE2 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "blake2b"); algo_hash[algo_hash_cnt++] = enif_make_atom(env, "blake2s"); diff --git a/lib/crypto/c_src/check_openssl.cocci b/lib/crypto/c_src/check_openssl.cocci index 75d1a6e44b..69259f7638 100644 --- a/lib/crypto/c_src/check_openssl.cocci +++ b/lib/crypto/c_src/check_openssl.cocci @@ -85,6 +85,8 @@ // EVP_sha3_256 // EVP_sha3_384 // EVP_sha3_512 +// EVP_shake128 +// EVP_shake256 // EVP_sha512 // OpenSSL_version // OpenSSL_version_num @@ -183,7 +185,7 @@ position p; @openssl_check_1@ expression X; identifier L; -identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; +identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestFinalXOF|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; position p; @@ @@ -252,7 +254,7 @@ position pnull != openssl_check_null.p; identifier FUNCNOT =~ "^(BN_add|BN_div|BN_exp|BN_from_montgomery|BN_gcd|BN_generate_prime_ex|BN_mod|BN_mod_add|BN_mod_exp|BN_mod_mul|BN_mod_mul_montgomery|BN_mod_sqr|BN_mod_sub|BN_mul|BN_nnmod|BN_priv_rand|BN_priv_rand_range|BN_pseudo_rand|BN_pseudo_rand_range|BN_rand|BN_rand_range|BN_set_bit|BN_set_word|BN_sqr|BN_sub|BN_to_montgomery|CMAC_Final|CMAC_Init|CMAC_Update|CRYPTO_set_mem_debug|CRYPTO_set_mem_functions|DH_check|DH_check_ex|DH_check_params|DH_check_pub_key_ex|DH_generate_key|DH_generate_parameters_ex|DH_set0_key|DH_set0_pqg|DH_set_length|DSA_set0_key|DSA_set0_pqg|EC_GROUP_check|EC_GROUP_check_discriminant|EC_GROUP_copy|EC_GROUP_get_curve_name|EC_GROUP_get_pentanomial_basis|EC_GROUP_get_trinomial_basis|EC_GROUP_precompute_mult|EC_GROUP_set_generator|EC_GROUP_set_seed|EC_KEY_check_key|EC_KEY_generate_key|EC_KEY_key2buf|EC_KEY_oct2key|EC_KEY_oct2priv|EC_KEY_precompute_mult|EC_KEY_priv2buf|EC_KEY_priv2oct|EC_KEY_set_group|EC_KEY_set_private_key|EC_KEY_set_public_key|EC_KEY_set_public_key_affine_coordinates|EC_KEY_up_ref|EC_POINT_add|EC_POINT_copy|EC_POINT_dbl|EC_POINT_get_Jprojective_coordinates_GFp|EC_POINT_get_affine_coordinates_GF2m|EC_POINT_get_affine_coordinates_GFp|EC_POINT_invert|EC_POINT_make_affine|EC_POINT_mul|EC_POINT_oct2point|EC_POINT_point2oct|EC_POINT_set_Jprojective_coordinates_GFp|EC_POINT_set_affine_coordinates_GF2m|EC_POINT_set_affine_coordinates_GFp|EC_POINT_set_compressed_coordinates_GF2m|EC_POINT_set_compressed_coordinates_GFp|EC_POINT_set_to_infinity|EC_POINTs_make_affine|EC_POINTs_mul|ENGINE_add|ENGINE_ctrl_cmd|ENGINE_ctrl_cmd_string|ENGINE_finish|ENGINE_free|ENGINE_init|ENGINE_register_DH|ENGINE_register_DSA|ENGINE_register_EC|ENGINE_register_RAND|ENGINE_register_RSA|ENGINE_register_all_complete|ENGINE_register_ciphers|ENGINE_register_complete|ENGINE_register_digests|ENGINE_register_pkey_asn1_meths|ENGINE_register_pkey_meths|ENGINE_remove|ENGINE_set_RSA|ENGINE_set_default|ENGINE_set_default_DH|ENGINE_set_default_DSA|ENGINE_set_default_EC|ENGINE_set_default_RAND|ENGINE_set_default_RSA|ENGINE_set_digests|ENGINE_set_id|ENGINE_set_init_function|ENGINE_set_load_privkey_function|ENGINE_set_load_pubkey_function|ENGINE_set_name|ENGINE_up_ref|HMAC_CTX_copy|HMAC_CTX_reset|HMAC_Final|HMAC_Init_ex|HMAC_Update|MD2_Init|MD2_Update|MD2_Final|MD4_Init|MD4_Update|MD4_Final|MD5_Init|MD5_Update|MD5_Final|OPENSSL_init_crypto|OPENSSL_mem_debug_pop|OPENSSL_mem_debug_push|RSA_generate_key_ex|RSA_generate_multi_prime_key|RSA_meth_set_finish|RSA_meth_set_sign|RSA_meth_set_verify|RSA_padding_add_SSLv23|RSA_set0_crt_params|RSA_set0_factors|RSA_set0_key|RSA_set0_multi_prime_params)$"; position pnot != openssl_check_not.p; -identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; +identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestFinalXOF|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; position p1 != openssl_check_1.p; identifier FUNCVOID =~ "^(AES_cfb128_encrypt|AES_cfb8_encrypt|AES_ige_encrypt|BN_GENCB_set|DSA_get0_key|DSA_get0_pqg|EC_GROUP_set_asn1_flag|EC_GROUP_set_point_conversion_form|ENGINE_get_static_state|ENGINE_unregister_DH|ENGINE_unregister_DSA|ENGINE_unregister_EC|ENGINE_unregister_RAND|ENGINE_unregister_RSA|ENGINE_unregister_ciphers|ENGINE_unregister_digests|ENGINE_unregister_pkey_asn1_meths|ENGINE_unregister_pkey_meths|OpenSSL_add_all_ciphers|OpenSSL_add_all_digests|RAND_seed|RC4|RC4_set_key|RSA_get0_crt_params|RSA_get0_factors|RSA_get0_key)$"; diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index ef6d0bdc13..12abf8aca2 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -76,6 +76,7 @@ static ErlNifFunc nif_funcs[] = { {"hash_init_nif", 1, hash_init_nif, 0}, {"hash_update_nif", 2, hash_update_nif, 0}, {"hash_final_nif", 1, hash_final_nif, 0}, + {"hash_final_xof_nif", 2, hash_final_xof_nif, 0}, {"mac_nif", 4, mac_nif, 0}, {"mac_init_nif", 3, mac_init_nif, 0}, {"mac_update_nif", 2, mac_update_nif, 0}, diff --git a/lib/crypto/c_src/digest.c b/lib/crypto/c_src/digest.c index d13304de49..2d211593d4 100644 --- a/lib/crypto/c_src/digest.c +++ b/lib/crypto/c_src/digest.c @@ -114,6 +114,22 @@ static struct digest_type_t digest_types[] = #endif }, + {"shake128", "SHAKE-128", 0, 0, +#ifdef HAVE_SHAKE128 + {&EVP_shake128, NULL} +#else + {NULL,NULL} +#endif + }, + + {"shake256", "SHAKE-256", 0, 0, +#ifdef HAVE_SHAKE256 + {&EVP_shake256, NULL} +#else + {NULL,NULL} +#endif + }, + {"blake2b", "BLAKE2b512", 0, 0, #ifdef HAVE_BLAKE2 {&EVP_blake2b512,NULL} diff --git a/lib/crypto/c_src/engine.c b/lib/crypto/c_src/engine.c index 16de4b8a40..6fb195b813 100644 --- a/lib/crypto/c_src/engine.c +++ b/lib/crypto/c_src/engine.c @@ -470,7 +470,11 @@ ERL_NIF_TERM engine_load_dynamic_nif(ErlNifEnv* env, int argc, const ERL_NIF_TER #ifdef HAS_ENGINE_SUPPORT ASSERT(argc == 0); +# if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) ENGINE_load_dynamic(); +# else + OPENSSL_init_crypto(OPENSSL_INIT_ENGINE_DYNAMIC, NULL); +# endif return atom_ok; #else return atom_notsup; diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index ba454f5062..029dffd44b 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -508,3 +508,40 @@ ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } #endif /* OPENSSL_VERSION_NUMBER < 1.0 */ + +#if defined(HAVE_SHAKE128) || defined(HAVE_SHAKE256) +ERL_NIF_TERM hash_final_xof_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context) */ + struct evp_md_ctx *ctx; + EVP_MD_CTX *new_ctx; + ERL_NIF_TERM ret; + unsigned char *outp; + unsigned int len; + + ASSERT(argc == 2); + if (!enif_get_resource(env, argv[0], evp_md_ctx_rtype, (void**)&ctx)) + return EXCP_BADARG_N(env, 0, "Bad state"); + if (!enif_get_uint(env, argv[1], &len)) + return EXCP_BADARG_N(env, 1, "Bad len"); + ASSERT(0 < len); + + if ((new_ctx = EVP_MD_CTX_new()) == NULL) + assign_goto(ret, done, EXCP_ERROR(env, "Low-level call EVP_MD_CTX_new failed")); + if (EVP_MD_CTX_copy(new_ctx, ctx->ctx) != 1) + assign_goto(ret, done, EXCP_ERROR(env, "Low-level call EVP_MD_CTX_copy failed")); + if ((outp = enif_make_new_binary(env, len>>3, &ret)) == NULL) + assign_goto(ret, done, EXCP_ERROR(env, "Can't make a new binary")); + if (EVP_DigestFinalXOF(new_ctx, outp, len>>3) != 1) + assign_goto(ret, done, EXCP_ERROR(env, "Low-level call EVP_DigestFinalXOF failed")); + + done: + if (new_ctx) + EVP_MD_CTX_free(new_ctx); + return ret; +} +#else +ERL_NIF_TERM hash_final_xof_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return EXCP_NOTSUP(env, "Low-level EVP_DigestFinalXOF function is not supported in this cryptolib"); +} +#endif /* defined(HAVE_SHAKE128) || defined(HAVE_SHAKE256) */ diff --git a/lib/crypto/c_src/hash.h b/lib/crypto/c_src/hash.h index df3a43bb4b..36fec0d7be 100644 --- a/lib/crypto/c_src/hash.h +++ b/lib/crypto/c_src/hash.h @@ -30,5 +30,6 @@ ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hash_final_xof_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); #endif /* E_HASH_H__ */ diff --git a/lib/crypto/c_src/info.c b/lib/crypto/c_src/info.c index 3021c3d71b..b05dbcca0c 100644 --- a/lib/crypto/c_src/info.c +++ b/lib/crypto/c_src/info.c @@ -46,6 +46,11 @@ #endif +#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) +#define OPENSSL_VERSION SSLEAY_VERSION +#define OpenSSL_version SSLeay_version +#endif + #ifdef HAVE_DYNAMIC_CRYPTO_LIB char *crypto_callback_name = CB_NAME; @@ -111,7 +116,7 @@ ERL_NIF_TERM info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) enif_make_map_put(env, ret, enif_make_atom(env, "cryptolib_version_linked"), - enif_make_string(env, SSLeay_version(SSLEAY_VERSION), ERL_NIF_LATIN1), + enif_make_string(env, OpenSSL_version(OPENSSL_VERSION), ERL_NIF_LATIN1), &ret); #ifdef HAS_3_0_API @@ -140,7 +145,7 @@ ERL_NIF_TERM info_lib(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) ASSERT(argc == 0); name_sz = strlen(libname); - ver = SSLeay_version(SSLEAY_VERSION); + ver = OpenSSL_version(OPENSSL_VERSION); ver_sz = strlen(ver); ver_num = OPENSSL_VERSION_NUMBER; diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index 647ab25ad4..9c7b05660e 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -181,7 +181,14 @@ # ifdef NID_sha3_256 # define HAVE_SHA3_256 # endif +# ifdef NID_shake128 +# define HAVE_SHAKE128 +# endif +# ifdef NID_shake256 +# define HAVE_SHAKE256 +# endif #endif + # ifdef NID_sha3_384 # define HAVE_SHA3_384 # endif diff --git a/lib/crypto/c_src/otp_test_engine.c b/lib/crypto/c_src/otp_test_engine.c index d3520de26e..f827c4438d 100644 --- a/lib/crypto/c_src/otp_test_engine.c +++ b/lib/crypto/c_src/otp_test_engine.c @@ -101,9 +101,11 @@ static int test_init(ENGINE *e) { goto err; #endif /* if defined(FAKE_RSA_IMPL) */ +#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) /* Load all digest and cipher algorithms. Needed for password protected private keys */ OpenSSL_add_all_ciphers(); OpenSSL_add_all_digests(); +#endif return 111; diff --git a/lib/crypto/doc/src/algorithm_details.xml b/lib/crypto/doc/src/algorithm_details.xml index 767fa3cd6f..603f7ebe4f 100644 --- a/lib/crypto/doc/src/algorithm_details.xml +++ b/lib/crypto/doc/src/algorithm_details.xml @@ -224,6 +224,8 @@ <row><cell><c>sha3_256</c></cell> <cell>32</cell></row> <row><cell><c>sha3_384</c></cell> <cell>48</cell></row> <row><cell><c>sha3_512</c></cell> <cell>64</cell></row> + <row><cell><c>shake128</c></cell> <cell>64</cell></row> + <row><cell><c>shake256</c></cell> <cell>64</cell></row> <row><cell><c>blake2b</c></cell> <cell>64</cell></row> <row><cell><c>blake2s</c></cell> <cell>32</cell></row> <row><cell><c>md4</c></cell> <cell>16</cell></row> @@ -258,11 +260,11 @@ <table> <row><cell><strong>Type</strong></cell> <cell><strong>Names</strong></cell> - <cell><strong>Limitated to</strong><br/><strong>OpenSSL versions</strong></cell> + <cell><strong>Limited to</strong><br/><strong>OpenSSL versions</strong></cell> </row> <row><cell>SHA1</cell><cell>sha</cell><cell></cell></row> <row><cell>SHA2</cell><cell>sha224, sha256, sha384, sha512</cell><cell></cell></row> - <row><cell>SHA3</cell><cell>sha3_224, sha3_256, sha3_384, sha3_512</cell><cell>≥1.1.1</cell></row> + <row><cell>SHA3</cell><cell>sha3_224, sha3_256, sha3_384, sha3_512, shake128, shake256</cell><cell>≥1.1.1</cell></row> <row><cell>MD4</cell><cell>md4</cell><cell></cell></row> <row><cell>MD5</cell><cell>md5</cell><cell></cell></row> <row><cell>RIPEMD</cell><cell>ripemd160</cell><cell></cell></row> diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 0d0ba07d8f..3a65824a67 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -257,6 +257,12 @@ </datatype> <datatype> + <name name="hash_xof_algorithm"/> + <desc> + </desc> + </datatype> + + <datatype> <name name="hmac_hash_algorithm"/> <desc> </desc> @@ -290,6 +296,7 @@ <name name="sha1"/> <name name="sha2"/> <name name="sha3"/> + <name name="sha3_xof"/> <name name="blake2"/> <desc> </desc> @@ -1108,6 +1115,18 @@ end </func> <func> + <name name="hash_xof" arity="3" since="OTP 26.0"/> + <fsummary></fsummary> + <desc> + <p>Uses the <seeerl marker="#error_3tup">3-tuple style</seeerl> for error handling.</p> + <p>Computes a message digest of type <c>Type</c> from <c>Data</c> of <c>Length</c> + for the chosen <c>xof_algorithm</c>.</p> + <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> + is not supported by the underlying libcrypto implementation.</p> + </desc> + </func> + + <func> <name name="hash_init" arity="1" since="OTP R15B02"/> <fsummary></fsummary> <desc> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 53cdc76830..7c1e1d188b 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -25,7 +25,7 @@ -export([start/0, stop/0, info/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). -export([cipher_info/1, hash_info/1]). --export([hash/2, hash_init/1, hash_update/2, hash_final/1]). +-export([hash/2, hash_xof/3, hash_init/1, hash_update/2, hash_final/1, hash_final_xof/2]). -export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). -export([exor/2, strong_rand_bytes/1, mod_pow/3]). @@ -139,7 +139,7 @@ hash_algorithms/0, pubkey_algorithms/0, cipher_algorithms/0, mac_algorithms/0, curve_algorithms/0, rsa_opts_algorithms/0, hash_info/1, hash_nif/2, hash_init_nif/1, hash_update_nif/2, - hash_final_nif/1, mac_nif/4, mac_init_nif/3, mac_update_nif/2, + hash_final_nif/1, hash_final_xof_nif/2, mac_nif/4, mac_init_nif/3, mac_update_nif/2, mac_final_nif/1, cipher_info_nif/1, ng_crypto_init_nif/4, ng_crypto_update_nif/2, ng_crypto_update_nif/3, ng_crypto_final_nif/1, ng_crypto_get_data_nif/1, ng_crypto_one_time_nif/5, @@ -424,6 +424,7 @@ -type sha1() :: sha . -type sha2() :: sha224 | sha256 | sha384 | sha512 . -type sha3() :: sha3_224 | sha3_256 | sha3_384 | sha3_512 . +-type sha3_xof() :: shake128 | shake256 . -type blake2() :: blake2b | blake2s . -type compatibility_only_hash() :: md5 | md4 . @@ -528,7 +529,7 @@ stop() -> | {macs, Macs} | {curves, Curves} | {rsa_opts, RSAopts}, - Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], + Hashs :: [sha1() | sha2() | sha3() | sha3_xof() | blake2() | ripemd160 | compatibility_only_hash()], Ciphers :: [cipher()], PKs :: [rsa | dss | ecdsa | dh | ecdh | eddh | ec_gf2m], Macs :: [hmac | cmac | poly1305], @@ -558,7 +559,7 @@ supports() -> | Macs | Curves | RSAopts, - Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], + Hashs :: [sha1() | sha2() | sha3() | sha3_xof() | blake2() | ripemd160 | compatibility_only_hash()], Ciphers :: [cipher()], PKs :: [rsa | dss | ecdsa | dh | ecdh | eddh | ec_gf2m], Macs :: [hmac | cmac | poly1305], @@ -621,7 +622,8 @@ pbkdf2_hmac_nif(_, _, _, _, _) -> ?nif_stub. %%% %%%================================================================ --type hash_algorithm() :: sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() . +-type hash_algorithm() :: sha1() | sha2() | sha3() | sha3_xof() | blake2() | ripemd160 | compatibility_only_hash() . +-type hash_xof_algorithm() :: sha3_xof() . -spec hash_info(Type) -> Result when Type :: hash_algorithm(), @@ -640,6 +642,14 @@ hash(Type, Data) -> MaxBytes = max_bytes(), hash(Type, Data1, erlang:byte_size(Data1), MaxBytes). +-spec hash_xof(Type, Data, Length) -> Digest when Type :: hash_xof_algorithm(), + Data :: iodata(), + Length :: non_neg_integer(), + Digest :: binary(). +hash_xof(Type, Data, Length) -> + Data1 = iolist_to_binary(Data), + hash_xof(Type, Data1, erlang:byte_size(Data1), Length). + -opaque hash_state() :: reference(). -spec hash_init(Type) -> State when Type :: hash_algorithm(), @@ -660,6 +670,12 @@ hash_update(Context, Data) -> hash_final(Context) -> ?nif_call(hash_final_nif(Context)). +-spec hash_final_xof(State, Length) -> Digest when State :: hash_state(), + Length :: non_neg_integer(), + Digest :: binary(). +hash_final_xof(Context, Length) -> + notsup_to_error(hash_final_xof_nif(Context, Length)). + %%%================================================================ %%% %%% MACs (Message Authentication Codes) @@ -2184,6 +2200,12 @@ hash(Hash, Data, Size, Max) -> State1 = hash_update(State0, Data, Size, Max), hash_final(State1). +hash_xof(Hash, Data, Size, Length) -> + Max = max_bytes(), + State0 = hash_init(Hash), + State1 = hash_update(State0, Data, Size, Max), + hash_final_xof(State1, Length). + hash_update(State, Data, Size, MaxBytes) when Size =< MaxBytes -> ?nif_call(hash_update_nif(State, Data), {1,2}); hash_update(State0, Data, _, MaxBytes) -> @@ -2196,6 +2218,7 @@ hash_nif(_Hash, _Data) -> ?nif_stub. hash_init_nif(_Hash) -> ?nif_stub. hash_update_nif(_State, _Data) -> ?nif_stub. hash_final_nif(_State) -> ?nif_stub. +hash_final_xof_nif(_State, _Length) -> ?nif_stub. %%%================================================================ diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 959e1a09a0..dc105d1f8c 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -82,6 +82,8 @@ generate_compute/1, hash/0, hash/1, + hash_xof/0, + hash_xof/1, hash_info/0, hash_info/1, hmac/0, @@ -250,6 +252,8 @@ groups() -> {group, sha3_384}, {group, sha3_512}, {group, sha512}, + {group, shake128}, + {group, shake256}, {group, sha}, {group, dh}, @@ -375,6 +379,8 @@ groups() -> {sha3_256, [], [hash, hmac, hmac_update]}, {sha3_384, [], [hash, hmac, hmac_update]}, {sha3_512, [], [hash, hmac, hmac_update]}, + {shake128, [], [hash_xof]}, + {shake256, [], [hash_xof]}, {blake2b, [], [hash, hmac, hmac_update]}, {blake2s, [], [hash, hmac, hmac_update]}, {no_blake2b, [], [no_hash, no_hmac]}, @@ -776,6 +782,14 @@ hash(Config) when is_list(Config) -> hash(Type, Msgs, Digests), hash(Type, lists:map(fun iolistify/1, Msgs), Digests), hash_increment(Type, Inc, IncrDigest). + +hash_xof() -> + [{doc, "Test all different hash_xof functions"}]. +hash_xof(Config) when is_list(Config) -> + {Type, MsgsLE, Digests, Lengths} = proplists:get_value(hash_xof, Config), + Msgs = lazy_eval(MsgsLE), + hash_xof(Type, Msgs, Digests, Lengths). + %%-------------------------------------------------------------------- no_hash() -> [{doc, "Test all disabled hash functions"}]. @@ -1249,7 +1263,7 @@ use_all_ec_sign_verify(_Config) -> Msg = <<"hello world!">>, Sups = crypto:supports(), Curves = proplists:get_value(curves, Sups), - Hashs = proplists:get_value(hashs, Sups), + Hashs = proplists:get_value(hashs, Sups) -- [shake128, shake256], ct:log("Lib: ~p~nFIPS: ~p~nCurves:~n~p~nHashs: ~p", [crypto:info_lib(), crypto:info_fips(), Curves, @@ -1538,6 +1552,16 @@ hash(Type, [Msg | RestMsg], [Digest| RestDigest]) -> ct:fail({{crypto, hash, [Type, Msg]}, {expected, Digest}, {got, Other}}) end. +hash_xof(_, [], [], []) -> + ok; +hash_xof(Type, [Msg | RestMsg], [Digest | RestDigest], [Length | RestLength]) -> + case crypto:hash_xof(Type, Msg, Length) of + Digest -> + hash_xof(Type, RestMsg, RestDigest, RestLength); + Other -> + ct:fail({{crypto, hash_xof, [Type, Msg, Length]}, {expected, Digest}, {got, Other}}) + end. + hash_increment(Type, Increments, Digest) -> State = crypto:hash_init(Type), case hash_increment(State, Increments) of @@ -2164,6 +2188,12 @@ group_config(sha3_384 = Type, Config) -> group_config(sha3_512 = Type, Config) -> {Msgs,Digests} = sha3_test_vectors(Type), [{hash, {Type, Msgs, Digests}} | Config]; +group_config(shake128 = Type, Config) -> + {Msgs,Digests,Lengths} = sha3_shake128_test_vectors(Type), + [{hash_xof, {Type, Msgs, Digests, Lengths}} | Config]; +group_config(shake256 = Type, Config) -> + {Msgs,Digests,Lengths} = sha3_shake256_test_vectors(Type), + [{hash_xof, {Type, Msgs, Digests, Lengths}} | Config]; group_config(blake2b = Type, Config) -> {Msgs, Digests} = blake2_test_vectors(Type), [{hash, {Type, Msgs, Digests}} | Config]; @@ -2560,7 +2590,106 @@ sha3_test_vectors(sha3_512) -> ] }. - +%%% https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/secure-hashing +sha3_shake128_test_vectors(shake128) -> + {[%% SHAKE128 ShortMsg + hexstr2bin(""), + hexstr2bin("0e"), + hexstr2bin("d9e8"), + hexstr2bin("4c386d97ace346b2a06faab35663ce8a4c54c295b5b9f6161efafce451ca8f617ab7d5ab88ffe117d6a67cdb0bc5250a3f2556c65f0c09b1d2577ba45cc930a443a33711b175af215a338a8d5e8b918a7176a8fb390e54e5f79f7a236a006a5bf1241b30efecb8b9733f5c32195d1bf22b70419d0c65de9bd7f982c94317456eca610a700a0d05c86bf27b3302e2c92ab53ba815a0b9afbcb88e1afe"), + hexstr2bin("5d8f84b2f208b58a68e88ce8efb543a8404f0ec0c9805c760ad359d13faab84d3f8bb1d2a4bb45e72c0ec9245ffda2e572f94e466cffa44b876d5c5ed914d1ff338e06b74ad1e74d1405d23d759356661b7f3b03a7f7c2dc0d2c2dbe3d60822803408d472b752424ea76af1d79a0e7920388dde0c1903e9364b8d6d7b3b75430754b4d6b91cd83d5740866aab34bdbd0f1bd3dc504f1a1d753ba5f938241ce7f52544e0cc2018cc67b6401ce6abdbc8aafc5629bb643730fa3daced8f425787d61069910073ac760c631876fe81d1127034a544820ad3aa51cbf2d904f8cda936c063561a8a0bd0b1f1801777394630fb6f11cb68a588000861283a2dc9d7d2739ff2ae5ed5af5304cc176cd544a39a99064c1cb3b6bcc88a97ad9f6e381e8a3929781861e91f73516d3ee59d3661b5f584b4b717d0fa7a54da03674ac5fa36d3d76412a826c4c8445f7720337119198"), + %% SHAKE128 LongMsg + hexstr2bin("a6fe00064257aa318b621c5eb311d32bb8004c2fa1a969d205d71762cc5d2e633907992629d1b69d9557ff6d5e8deb454ab00f6e497c89a4fea09e257a6fa2074bd818ceb5981b3e3faefd6e720f2d1edd9c5e4a5c51e5009abf636ed5bca53fe159c8287014a1bd904f5c8a7501625f79ac81eb618f478ce21cae6664acffb30572f059e1ad0fc2912264e8f1ca52af26c8bf78e09d75f3dd9fc734afa8770abe0bd78c90cc2ff448105fb16dd2c5b7edd8611a62e537db9331f5023e16d6ec150cc6e706d7c7fcbfff930c7281831fd5c4aff86ece57ed0db882f59a5fe403105d0592ca38a081fed84922873f538ee774f13b8cc09bd0521db4374aec69f4bae6dcb66455822c0b84c91a3474ffac2ad06f0a4423cd2c6a49d4f0d6242d6a1890937b5d9835a5f0ea5b1d01884d22a6c1718e1f60b3ab5e232947c76ef70b344171083c688093b5f1475377e3069863"), + hexstr2bin("c747ee72b00680920ef1f2e16d32567e7d8900b76749832a61a3dc2a8126ec65ce872cc39d9e9c911bb2fcde26f63f2561533d2d8768a82effd04135b1a34bc9193f40cf9948c4ab14edd7a8b86d4e9a1bda84c28a1e4df19e3b2bf00fd7c520cdf7945e4d0be029212beb07cca06c24d4a63e3d19705abc646d032543ca40b015388f3861efd8536cf2a0c30a957f0db43dd467db68b7d77ced2f9af584a5c3178899247cfc94711acd5ea31956709be229ac08790d4c8a0c431918cc0edd9ffb721aeb463bfccb56418d70a2eb281787f230e89a67011d3df82766706fcf8cea7b7a5569b3f9a946d895f24c5a8ec9ca1ec7cee77fbfd73693c34e8aaf62d40c2d222a8a6c71521219def1c0ad89f8d437d883075c6c02209c5496c2d780be880f7832546f4e0fe38cd035a5ec03edccc5cdd0a306f2e4229b4207bfed109186a9e5356cc37628e1bad485a41de3f1db4e0d7b638637c56519f4692afcf9963084d2fa7d70d3e5332a50ab654a114a295c004ceee325ef109ed484aa092515ac27a54a2d7f4b8d0264ae6d8f7a7440dd760e0ded25a3a94cb0491fe81e7b55221ac8ed24f7c86a75f15494cab6f61d20259f840755983a41d9bb048b42e3a0bbe6a1ab0be031c2b558ace8c0057e2719ef1b7ae57aeeeb16a092f93c7e453c9e129a4c8a436edee9211c30166ff5a8f15feab82f9c8c04f48d37c7a3d98972f5261997482766063ad50e3677fc41e6cffa74038cc35acd02291036675b988ae5ab8855ee940382d0fbe9c089b54b2adc4a2226a4f8408d69fb860d310c58cc98284e4e23922c99d21b1597f4a1fbbb3fd65ae72042268e508c0216a8f5b2803625d34a902ac99468befa8361c83ae321268510a8aaf416e062ea17751d8eede62a7f11dd63c3bbbc6b004b7833eee8aac30c67e337926e4007a7d93f8cd55cd1c0ba6ab1460056f3aa8fb9e0e90013013455b57af1e8de9a03a224af4b6d151f6eec1fb4d969ad38030f9342f259ac4f01cac114e0dfad13dc42bf6d614060f7bd7df68f26e608b1ffe8a24557dffbbbb3d2555a0d5d25d51cee7420d635858134d5b8ea3560ae862582ae921f7c476ed1d437f47f4b8c084a100970b333cbe75b5c2d6c0cbde8e0fd457fd05217da90e33a30893e922bd3ef0f4707ff247c4eba7b1cbeb5e7fee7109adca8f998fa48716b08c153184a5f9253f68ac2efdee0da984e8838c438c2e5e78ec21be6f0be04cf8746c5c731d4bb44dea4047c3ac5421d8c719b3edd9a05955ec5c0eb041b23ce75bd9c862759f34ac6fbceb3d7257a709c21e83c3796f60d115af08e7d8757d099d1e13c7516c0cd7837277cff270bfb464127a58cdf42b67228b4b7a529273dbc289bd36434ad35ba0118bff8f37c2454795e63259c70f748d8134ec22a1b7da66ad2323adb3fea5501a8640bb255558c70975d4af6093a5a11163c1db1bbf81f408e413f32005411dedf53808c1ff5b8cfd97b431594e6717cc467fee79017ef52d8e8bd6277dbb540c5749fa8606c9e3d2da55720996355c025b8acf42140dd47b70ffa922ce80bd163afaf2fa321076399c3cf3fa118ce86411771fff6dcb3c7349e3feba8e1d936c0edfa486d7ebae828bdf39c336fa476216d93309854377c567dd4957230e81ae414c61c48ea8176df7b0b5cd7cf9e37cca05bb08e0e320b259501b7123b0023d80e5cd3bbcd22b2c8e65faa15a086724a9a11a760746b1fa9cc0e7a116ce8f9a3c104944f906646eae437ac764b22cea441fd3d97edac5eeb0b4205f62a6a454d1706878a9915da4d5eda8a81363bf4e17f6798bae0bb70f23be78149dc4cafdff7a9d5f751f715fa3d916dbfca988b093b96e351b761fec642431593ec57de604836a61b20d3a0f867a47db66f033c255dd344e5d966b0700a9eea5ac1537bd8f4f521806f73224fc65be1f08e42af3bcc66e3fdae922df779422729c3082d50e22e6fada49d440841fdc2f58d1e8b567297e9219ead1e637976f5855a3e9dcefa29b72a291aae2abfd4474bd65f797b36ae7ed9117e2abdf1817cf8eebc735d242d228064b207bd2a26997ccfcd20b400f2c9ae6fcefa0cc8ace309e74b56024884d836bf2cf8ca693cdf10e22f2908b8acf32720653558e83f5341adb3eec74e7400d4a0522857645e428d764f3b80472423cd8c0b84a93882f326a8bce4b31e46c3e271ea406f03d4ea7df2f3ff2293a282261c888f08a0bcec42412de0cea14248e999bde7172ee5fe67c0e41cf8856e38bc36a3fd7c0837651dded08921bdafb8718093c659cddaa6d447ea3b0a05cf6c174425770e5839873de96b88b35bd73a008e34843cc8fb0c4befcc045efc0f47bdacd7a4c5780b792cc8c7848d6400f7202b67cac80790555420541efe5074f038d87e2da951f56472e5ac26b5be731658bff5400672e1ac77424832f2eb478f102ee5b9c2c93e7095a6e216355378c030b2a78dfdf236373dbd71e740b3cf4ecb18d3f119c930c41f7530fb1509561d316c435ab66dd61e2dfa97e6f2925653f3fc0f7cb53a118998b95f7e5bca5754e256914e37cc0a8ad05f82d7106f22c60f4462049e022ae686d93746117dc18eccd65b128af81aa3ac55f960a0317cd0a857942edd0bc304733f80fc3e0addf27555674db7d84e904b45c62cacc2ac99459d6da4793b240be6520247eb58f3a9673c7ea8c453ed7a3595afe13b09fa486e7eb50e7a7c9422687a85d01f514bda2af168cbbb534188f0af7d2856eeae05ac623313e9b55d71801fd03ee2dc2cae44deed5da3bf90e306f9f8a804590ffa85b1b05ccdcec521b6a9ac3a3ce1e04d3f69d507857e23b2ffdde70a4d32664c83d3f4912eae155a9281e16c3649addea3fce770687a421e61c113f53961fb262e31c759143f94439d0d377bd7fadc89f6feec40549a354fb60f4512d1cf16ae0d31f15c88688c055eb6761028ddb478c2d7ad2152c6436c852e43091c62ff0ca83c3943911143c75de465d22fda7fd08eeeefb968e6b59d5786d5812203fd6987bb68b6382202b22300cd6af6da83bce3ad7db0a1e3d30cb37fc522ec9b5aed3314465c848411ee3770767f85c30a62487b1eaf02a3e5de2181da9581105cbf33b19bde7d1db8a2219e28bae27f5aea45c0995e950f333e29f4db82ea4c4c080ff82fe32bd63ff76af0abc317f4bb7d62e5b764610f0552544091902d3572ef5053ca149798ed1499d0c6412c0c77432850406a6e8ca24229cdc46854b5159d7d293319c8490c2136a543dc27add5631e6afc17c56c533731043bb266b37e391ee13399c6d2e248cad7e83d12c11c05cd9bb2146d996059f24bc0156c16f6ecbbe5923d9a1cb8afdcce7e0cf42f9553bb4c2c086ef911cff60b19396dc8c3dedd9c18ac4b0ed57ef4f1ccf0d6df887272e3309cd176adc2c5ad0fc771b02d360b6e8c8ee18f83de9aaa9c92bfb93c8d7aca87789df77e59e5976a6781b67e1dcec3b88d658908f75761b49ab7a5fbca5967f943da63cabde033f2c0a3c2fcbaad03624fdee3adf0b27cddafc78b0b5199244da590a4da153ee86f7be1bcf1363096537c5485baf13464df79f553b54bc1684d48c8a1f68a851da83f438ca3b8dfe553bdcf577724638182a24be541e05a9abfd18a4266443f3c66b5d01b5b6943562aba58402bafca3b8ecfd129f737eb246cebdc6f716af3ac7185904f6070e3529f1af62b20baeefbcaaa8e081f79fc6d1c3cc81060242ad591568b11fcf647427b85b2fd9c6e5927ffdb8a6800cd16dd966c550d7b9c6b1f8ed0ca4a7523f88c86a5939fe8f29229f50acd33e35cc0dca4248b4e0af73ef22f9c54e0141e7d9a881fa1f94c6f216774de604f104e82d20c0c73bc55602bb63d6231bd21105ed2499ee4f30f0d8f4b44ff4aa6cfcae42797cb143da4fdbbd381c96ac7e420249e90c865767bf3623b2c16931e68f86f430d0a77ba3ef675b1c07a06e8a477566527452b68d64a25a1f6cb44a12201aa8f3670a21c6c74e2b8c51d0d07ab3c7c8ed4be3eb369395de468c56316470d1d71daa69f49769f88515759880b3f981f8360dba73d7ce9946c4badfb641d435152c053762ba49ef1e23dd67edea9b3c6ab93436df0578251a1662a1175edcccd9bb1202da93d9aa5d5403df76731b4f96e349f98b2412b1deb83e5af30fed019b275b4ceaa8bba2a902836fb8e1ec440340d96316d840c7fb26885dc7d9a76be01d4b96018d4270a7dbd68933bba0a2465e4d13e08664eff5edeb3a14407e643ee96d4ecda6108cd02c67cc6564baf2c6982f7ca6b30192557ad17056d8e4930e382af79d87a86f9890d199dc71adae1a9afb1da7e29947a059d5810e177c70e4761f20ffa67c9187958e81e766fb4c98b32a778299be885d1761fca860a876482fb45b6286ff8b13a7091534c92656e78d580ab54cc3cce21ff133c458fb2bafc184e774adb3b4d341dbb442fca1458d95ee57ce705b9de5e01dd6d074d6f991c80b85948f3bc73cf260ef80860e90d14967df1ca8e873cf1452465b28de7874cfa29650affa0ab73553477c09c0d179345f115e5afcd704090912d2f97410ef45049a27847127e8870d2c93dc6a8febc33e8ed20b0b403b5adcccc67b3a6b67a48c2187b133872764093f76926e148f9270b8364d8637cb7627c4a41293e0647ab39cff9aa3a11a58d1c072b04e17fb1eb02de97710263ad7949b0a4403086c34059d0db5541e37d1f18046aa7d02a4b457406a3c1bb4101b8dd057a41a5be56d1c7220a5c756f67bd2df31058d5d231fe09046e11b49d1ae8dd51aa74c43769242ecd6d07fd3dc93103de045f18fecade6adfe2a815ff60d1f49c1c21b678620334daeb9d01d50c4143cc2e04c2367eac3f9fe6d007ef3b3ac8efb890a209ab3481506e68abc099b695c6768ffb03b5322f4d61cfb2f613ec01516609b173aa7fbe37ddae2185d757a0d3c447eaebd736e3ef3dee3d9b50b048cb0ed30b3dc9ae93971fb88bac2dee0dfb0075b8784c8103906175d0675a4c45649837d7ef6f87a7791ece12bd0541834612618c327ddb3635b5ac95e19a889f24ec7d3e1203d4a3e4c138de41c61aabbde2538fa0d00b24e821dab191762efe13f2ed672def5b2ee1f03d16c0d73f2d675993e94aaedb76b08d494fff57040c3a93ce7ac89ae5f1779326e2c0a278b04d5d68c1b88011d496e13f95b4a6d46ea0b56aa16b6c3994ae933b6bcb8e8fdc75d71c3f73ac76e5a00505ab37c964cf4d4107029fcd21be8ed6c4d968034a5be9b0f58d029cd252d8e1e3ba102a7cd54a20eec5eb25631720450954206973d2b1ce3cd0edb6edc2465e20eac82fe6289e386713d8dcc8486bc852037929d0ecd31b3833bd648edef1620662e64269715173b914ed5a18f97748e60efda92a4f597e33dbf90d7f4ee1d6db4bf2cacc586cb82eefb2081bd14dab0e9e345248a34ade73f3291886b91ea3e8cc742fd884f6ee0ccdaf4c9879f4db12dba58cf491af2541a1d5ef6cc8b1af750ef5d8559ef7ff9cd56d8f599974be3aecd8c0f4c08f3ae50d86f9f822a1e4ca39fd2f0b4d78d22630733a24d8d63ecdf9555411daf205a761c39ef46ff6292e74129bc13a7f4ff20549b3c2de2142152ca5b21384e16ecfc215462659b865ebe7be84410d175e477d7864fb545db158ab4f40c268625da45a03595886f08247bb8852516d45e76fc087b396e29375df9edb68e62e80ba6f51153f509ea6678b18f86eba0213d2b0f5fe0a5d2fca75f22ed191467e406d78cfd361d74ae0fcfe25973e43758807534466d016e40a99d982ba22abfeac5e07fbc625b432a829ea0bec2bb66eac6f9fd6456d8aa85bc9dbc41e85dd59cb7664fd75f4b1fd05d695f1809eda9488f0ded326498124c6ed41198321a435a59bd7fe6837945300eb4800d9fd7690237356638eccc30052ed54ab735ec83b43b6a726285426de9f504b6081b1530986e151ac075e17e308b159a275a41012acb01d9c3cb52f43eceee638002339a8b72015d918674675ee229b26125b4562e86f81892cc0772defab61b0dcbaef15c01aea104a4f195f0f26615dd0eb2fd500dd1b1481dfc542af7003f29fdd3ff7a8b51a79237e554970a6f2723abd9611fbe13266c279e22a34261623d0f6eff778dbec3e2d5cc5da1b8b978bd53742bee1620fa677f5978e86b3e06ef86cc8bc3773a784f05230bb17bb98d12bc2fb85bb4afbfd29bdaa16ae01aaf99ea792efed659476f3c65d946c203d6bc3c7880e90803c935f38e369b3f6bbec4a0f1537a399fb39be21150c23a1ce8125acb20068ec1a8fa1ce0c309d7c5d70850c183d60f624f8d9cc20cb4225923ddadb41034b76df1977e7eb12747ad6add50665a82c6656d9cd2d467ad496d2c569bafdc14206d15b6aeb4ab841d76fcac565c2210fd1da0420ae4fc1dedbd6415ab3addf54843a7a0142efc90c7d70efda91f850121df9a25db602476adce6da362c1b088ad860a1c555a2eedb5321469ef2cfdc3c33f8db09ac28b57f814ab7838f0de2eaea8cf63c8a2dcfb841c9f498f00e6e96fb22d6aefa89b226b24902cc434c157fa205a1f04a5713aa2fb22450f47fad265dbebcd307ede766572ce096b0a04a4ee915cb69fee763ba0522fc4f64e842f64213b4013b1cfb47583c7b6b3ed02961a5029c2d0ca666eeab62a41caa12d40e906ae7bffedd97d19dbefbd255a659bfc0153bc6de5cc5500aba453f2cc50c74310b7d95b8fa630d0efd88545de883702096d1bcdad027e80871074154c892320b3e9d0bd139060294404878be5f4e7dafcd85956acc35bc8ed111a989e50d50084b6faab16329b815ce66c6e0d363ed530180dbf51697d785208cff32e225d40f791cf9c14073387af008c62c0c69c8f67ce915e377f43ed3a79fccb98883c80712cc8167978d89acee570108d109167f84e9cc9bb7f4fb62ae7396859fcf33da5ca6c80c311eb392107afeddebebe0d662a887879e4014187d2fe8feefb01e6fa0d35819d7cfbf139e99451423b62ee35e4dce0ef48094df5bfc478c3fcf90ae257bb9f44f48b419dfde5f133f12068ebf012998805b809e92023d9759d4917f35c4710761309a8b1c6fd34407a972bfcfeb218d0b7ae23d68685187036629ca16d90ab9d7307a509d74290e4520795ed4343d565d85c1a326570cc4062d617711e00c7178ef5b52aa16ead8bf222353170306593e2326ba13f6f4b62ad3406e6e02fd990b1645788b7c9d0c3e557986e08103ff76fc1869796e93c636fdcf9875666798594c40aa87d0ca118a6182df77d5bcb0ccb99f989ee6715af45515db6e35d1d62a3a55d0e737f94f6bef47472329b9d50a02c32fcb8714dc2a831a20a096c4bb45878bd3f4dbb32ea9a7cea42da283d4f67e8d17487906b2a12fe65054b2c7af74dceb99f3e47f160b3021e0dd6aa11759d604365de3cf4621008251d2b0621a7de17b0b2f7ae66f5b3348e17ba99b04fab2317552edf5fe729ef50d62b91d64293d4511efc36fa15f5749914f86fa05a3b81abc6ec102445bb2476a4cf581f5409951947aeaa214e0452ed6a12a44811a93c9f4fb847e7fa5e48052d78b15358a9ca57a19150730a5a4ff9ab8bd04e028e946033f61ef1618dba9ac4ca4618dcfef4bad146d275e781dc2c4dd66bf89956397d3837edf737c40ca60f367ee63d6394b13b31989041f58813c584dffe629a8c64b0e03d5fd99bae95514fd43206823ab963e41b58c63fc43ea719b4b5eac61db522a195a1c37f21075f78de92d4d55751dfc0f345c0cb111b670782c59e535dcd21de8f5381d8f46a7e44066f04d76b18f36a98ddee221f2889c0b293f9dcdd3b230972eb9143a94d3223e44d32f2e65d634374878adf2ee483e7dca799cd0c7daab4e8d08e49b00e1faae0a3dbd279bd93ee2ee9434cc0dde9ef5976daae7c131d6379b8b555995bab4b84196bd065a809cd1069940585832049c19848fbac8a1f877dfab2e1d584498867045f876abb3952e18ef993dd62cb89ad72612f0e4125b39e1d0398e5368ed3d1fb4361d945a721eacfe07248fb5011659bcfd311d6a45a32a633c558d0ffd08d74a3fa3205d7ccb4b2d52b58cb3293828ac16277c92ffdd41ca00190e1f0e0d1e6fdab154a4f85d7ce64197d410762acb6026780872d110ae6f4d83df8269747f8bd1ef3a1831fcc8919d736fb23111ca3ef4cccaf20264fab8eb3b071e56667f06ea04d99f6eceb6942ef06683a118aeb04accc8fbdc00ac0fa1e88edd1c0ba849ee993922f6fece58bcb7c0691064fe1dee1f0c73a50d4cdb38327da5058a81baa455638a12b298450f6f854ec42042eda96e9e44a6b2638479c3db60570b92c2d6e15ebd0a139dc4eec1be190672038743618454271878722940b0f55bb864ca4ae432a52648643b235572b21e82c755849fc88d7066130c1354044c59b77e38c907e829ae79751374d2a901547919be34d172d5c8c5bcbcfea5e04d993068751bf28730e9d059b5aae16bedf07f7e2a1a9e8816795dfceab1db4be0624cabea084c17a4207297878d29c16039bb8001ffaf1437c9555afcf59f4eedfe32cefe58bda16c9b8ab7f3e453b2427856a6acabab1d2001d65f5b9299f05cb30c5918d2ef2b70a23e426079064779f6787891e63bacb96807916f50703915caffd5a24087c0987db6bf3612eb87eef8477f197807c8ea48117ba0f7cc31008a35c903683065cc731bb6dc78a97a89c9ee42ac28b6799be71ba2574e57e1063dd6bda3129024f4cd95afbc73e92833b0934ab3bff1c7b48f4e90fa1eb6500d290939ea084f04f8d44b333dca539ad2f45f1d94065fbb1d86d2ccf32f9486fe98f7c64011160ec0cd66c9c7478ed74fde7945b9c2a95cbe14cedea849978cc2d0c8eb0df48d4834030dfac2b043e793b6094a88be76b37f836a4f833467693f1aa331b97a5bbc3dbd694d96ce19d385c439b26bc16fc64919d0a5eab7ad255fbdb01fac6b2872c142a24aac69b9a20c4f2f07c9923c9f0220256b479c11c90903193d4e8f9e70a9dbdf796a49ca5c12a113d00afa844694de942601a93a5c2532031308ad63c0ded048633935f50a7e000e9695c1efc1e59c426080a7d1e69a93982a408f1f6a4769078f82f6e2b238b548e0d4af271adfa15aa02c5d7d70526e00095ffb7b74cbee4185ab54385f2707e8362e8bd1596937026f6d95e700340b6338ceba1ee854a621ce1e17a016354016200b1f98846aa46254ab15b7a128b1e840f494b2cdc9daccf14107c1e149a7fc27d33121a5cc31a4d74ea6945816a9b7a83850dc2c11d26d767eec44c74b83bfd2ef8a17c37626ed80be10262fe63cf9f804b8460c16d62ae63c8dd0d1241d8aaac5f220e750cb68d8631b162d80afd6b9bf929875bf2e2bc8e2b30e05babd8336be31e41842673a66a68f0c5acd4d7572d0a77970f42199a4da26a56df6aad2fe420e0d5e34448eb2ed33afbfb35dffaba1bf92039df89c038bae3e11c02ea08aba5240c10ea88a45a1d0a8631b269bec99a28b39a3fc5b6b5d1381f7018f15638cc5274ab8dc56a62b2e9e4feef172be20170b17ec72ff67b81c15299f165810222f6a001a281b5df1153a891206aca89ee7baa761a5af7c0493a3af840b9219e358b1ec1dd301f35d4d241b71ad70337bda42f0eadc9434a93ed28f96b6ea073608a314a7272fefd69d030cf22ee6e520b848fa705ed6160fe54bd3bf5e89608506e882a16aced9c3cf80657cd03749f34977ced9749caa9f52b683e64d96af371b293ef4e5053a8ea9422df9dd8be45d5574730f660e79bf4cbaa5f3c93a79b40f0e4e86e0fd999ef4f26c509b0940c7a3eaf1f87c560ad89aff43cd1b9d4863aa3ebc41a3dd7e5b77372b6953dae497fc7f517efe99e553052e645e8be6a3aeb362900c75ce712dfcba712c4c25583728db9a883302939655ef118d603e13fcf421d0cea0f8fb7c49224681d013250defa7d4fd64b69b0b52e95142e4cc1fb6332486716a82a3b02818b25025ccd283198b07c7d9e08519c3c52c655db94f423912b9dc1c95f2315e44be819477e7ff6d2e3ccddaa6da27722aaadf142c2b09ce9472f7fd586f68b64d71fc653decebb4397bf7af30219f25c1d496514e3c73b952b8aa57f4a2bbf7dcd4a9e0456aaeb653ca2d9fa7e2e8a532b1735c4609e9c4f393dd70901393e898ed704db8e9b03b253357f333a66aba24495e7c3d1ad1b5200b7892554b59532ac63af3bdef590b57bd5df4fbf38d2b3fa540fa5bf89455802963036bd173fe3967ed1b7d0611ed907027e72854b9c22cccff1ec72a9b6cbcf4870bf92d4bb16ed6730d1fb540ab78559e17e14e564593c68530aa384dc27b75caf427e84736711db211cc854e0bb873a8cb4f6bcc7d1b9cb8782bab111e66b215041f89d154e944946facfca3d6f8d32e4a9a3283326d55823e730a514a24b55185069850d27e51006ee6a4a7d3df7b19dde92be42c31f9cac9a9dd740c5e8664bd079d2c79d94edd57a305c3cc16754fcc64df55320e917862f6ade22dbc2b38f54721cc11a24215720aa55d8745c9ba5f35d53f0eac24735182000275e5aeb374f1ac9a6280368f07440edfce137d41de81950b86b4baa5de74613eab8216bf842bc4fd331c31335e78c8af67289bcb9073a81f63ce5bf551f555fd993a6c436b1df35d4d1e3be9bfcf59371a2d2a42be3095e56ca1104bf12c66a805b6b72672f1c023792d5c9971714bddb1ca338db0e3e4c36bd6e08f07421ba5b89eff52e1ccb3b57df19d2bbd85260bf2a6bd94aeb21e232614d4a9db5efc5d7a994dae00d8557c21140587eb337028a76ad03f00c5ca6c6704f382b5a99238124fa822163d2de02f3b9d9643a5b39c96bd7476ed457006f44e4500dd4b649f4a6918966d23c44a6d106c3e0c0076ce8848b221be5a99599599d20c9a6bfbf8e88fa5f24bb1a020a53b7a68906f422844b39128bc2bcccd148da38f3631ddacb568581ef90f4bcb9d64cffd15189e1194a51a732ce191ca38d3bf6ff7c69424799d3803bf664db41e3274e3e42312f13ec654ea59a370b8c8132e962652a8c7fcd54d87e4de2c42ef4d700445b926310bbf47768a9af2ff8df4f9ea0afc9ffc05b9a093b29e72dcda5ecb467f026337c338fbfd06ee9e074f1f2755eded032d5ef138106fcfa6eca50d8d93dc25506c2ed8796816c6fc38d53cc0be154d5bc1cd241ee4649851d91901bfd2a24a76381348c8a0024a95fbe9f2cd0ccb4330201bfa58f9d922fa0bfee369eb1a9b5b043c30f08c12a4f0b651e9542957f709b2101df035169429ea1c5a91c58c3d89366f36fa138ef996d3d6ca60fc01926ede7332ed82621767de3a0f414a405b88387505ed1fbd740db8403df66d113ce70f0ee458c707d33a4f22a980617d1bde701643d197e83adfcf57e92064ce1cfdaaa7d838ec4bef2da59ad157f28da515481999ae625776937ee377e5c9825b371807b995d87794c5486b2fcce6b622b52648ae8d6a524eb22fab8e4e8473201ec1f62d7875d2e98ca05d6aea2341ba4474692b759621eac34978d7617ee994380d3bdce8aaa6170916372bb69276944c9945b76fce0b734e3412fa289d2edf1065a070ed5f3945c410c3193a1b9cd954fda4fb82fa5911dc57419eabca89282b2feb8e8f7cd59c46f7c7dd7da7fcb0e5084e9eb60354ebf41ccd6bf1ee419713d4e5e0d7eac04416340c1e871cdf4286220a4913cf412a1eea7d0b02ef9f564fbd560ecc8f4264bc2d5a8561648bb7bfb3a1a4a2476bf5e22a1a23e6b7eb8a675106c07654a524c14c78ba126549361823dc566a531975c9426c5ea52c6d5ee3f7bc9dde212fadacf83064a20960e71ada810873dc92d25a9b3b6148386cf84c19424d0bdf2a270598482b1a361af34c415861c6c81f331e9d4fd7d7c232802a1e5e790574dcba559689b981d6bd8236be2b0a6a0108c09bb0706658b1051f036ed9c9d5ca8e8c2259bbb3dfd092bb61bac0aa676d95a35b3d65b6d5ef2efc61717ae482c887c8f749a6c43c0ce1c7549ee0eac6f60e01e6aaf7705563f441cb8f0b731455119459ce1192a172f2ce97d053728e83ba05c261ee3ce1f5a113ca30521d945b7218d250c6c515ea0cf7b3bae1e9c7fbe2d479c812c1808d7c9ce5bc198b09c1f37ee9378b06a15c786a2fc252f87a48ce9993492aa83a13cc04a49b26647b6fdceb27eb29fd4cb07f5093526d46ae51c67a964b0098176fda738f354151f34c758912d468511ece67d168d1fb7c6bfe129f0c365a3a208aee102ff234fa77420580345b0faa1f9e781435a7c441b937f9317c7101365c4a4f030b3bc6061238d6f815e6e8b25809fc42823b903ea1e4c4586f66bd33d4984775ab76be1ec9172e183bdfb90ed0f53982d8f7271a2aedbc3b6c7c55d98e5d1c8e980de5e9a6c5560830ac4b698f4c78fdbd42517e5d3a2603fb1b93d2947f177559660310b02b96f17cb766699ef401dabc84d51240b44dd7a4ab3dc3eee8805376ebe03aef3135fba566d1b604d71c425a267c7dde4178e0f0941c57a27e9f4f780243106cc07d4d352356f9c9db115d218e0e683ccaee46663ff834ba2bbb92e615f5392ebeb410b455bbdaf6ea27529825537a6438f8308f1299023dd51e43488bbf5e3d15236ba863c90092eba84a4753125c5d71dec6f176c66a34660e8b27cc6727a2ab12879bb74814a5c5da6cb78115b52f5fe83216d44913a6c5d4920940b53b646ac98b14b800bd386206e16f15222ee3e95e998692e8a79bdab87f794f73438b388fa09cceb9a451a973d0f779026f7ff68726c71a6ccb1c8d017afc6167e620ffaeeeba613a71fb826b62d8137d1f0e70acfe63d29c292fc6f9ad1cde5acd2aa66c9cf6d94ccb3bd534bf994d5ae6953e154ce2244269df6de63e602e15a0b11512d3d89fb75a1b8a170ec28e636e1e480a36ca772433316a115550b95d06be34e7bae91943c7a28a5c78dc3d5e5cb076d3e4a5320c2d2fa5c21b75e7fda757eb357ab0b0b7bad8bb4b246ec4145b54b0b77ffabfc002e00741b75621bb2c8722423ab5ccdddac6062c3a008a4d1111deff2f528508584c76d5527ccd613d53c1582381d67bd0735346e41af7d026efe8f41b34ecaed87cdbc7146d55b581c5b0bd5d9b345a43e88be24845736743f6b6db6c285c7829771f5f24b31f80a1e4d7b90f8b5042dc2b94488e0866d8102f30ff49b63d30f34528e5abd6958944b81e9661da91fb434658c0dfec38ec34a13fa7db67f79ce0efe0a1d05c1d9a134a6e9a7c0f5e4c1f40be926378516b5854c91a4ac2f285967c2edbbccf2179b6a1ad0f88b221fe1922ad0541341a1d2351b7a1bd5097b2bdf4f9da96b608efd4c5c53bac4c41d9d8f30f62a2a8f9ca3e9ff2845c974cedfe602be7879a3ac415bf9c67116426a505bbcbf1ad5d57395046610b50217afc01c7ab4ab7021fc023961cdadf82382a4783c9fa8bda9d33024feb7ec1768aa86d31614b4d761faba3522cde3d2edf304752f1c563a73180fcc17a24d61ce2b9eaa63532727ea24c40fcb3b953f1837eac302b9a5e4d7de8e5a4f546b89decf7222f0f464da3106ca09b9bb44a528cdd1f255fc0a453e7336501432c8f810f2abad3d7d0076cd1d36b37cb174ada6215483a75805c41d92826df4e1e824eb2195d9b7518c7624d494ad92b2a7a0f6deece765522aea0ba90cd46fd64b98e3f8db18dd166b60bec96e487a335037b15305ae2e7eec3f6f233053e38455dc81d4e5eb4d402fcf21781e34b1b7f0da6c3a29384142bc3cee9d974d0e70a9685b4240d156433c09c34d4ec2d7b51bf17dd0d2266c223d6ad064ac203f3fc8b4c6cd766ba6a432a93ad55aa34abf34156cd39ae90da1c221db34d98e0426065a5ad2e10aacf78635ee8cdc26893adbcebf9eee008455fd2a558ac9e6dbd814b18dff2b4ed791b93854f03dfd11cc7585fcbf460d8e9340cce5b2d7bb44ca6662e6aebf98c5273e19048c1a7782c883c38a912d508b0e2f81e2599d7bdd816de6eb6e1888e41ca76083dcdc264ac399101fd12e79e81b3c01f1eb75c8e471ada612b9c08fb2d667a7595f0d99d85227e3f2eba3fa443ed8959ae51b03183fa828d32e322db0090145db8f2942c9d8b2c7ca4d38521e0d97e533e2d7bae72feff7c31807e37a1906c4a0e9b9a18debcc23fc724611e3fbfc4953415660bfe6e79b69239c316f8aeabc3ffb394692ffc6911ff7779a248e786a7face18520e9117c1bbd79981ed32f951f301292877954979b49278b5e398ea12f6233a8b89b002271622aa7f68637d7a524e025ca0cd89ff4e3e045ff81c185d92db74bc5913627009e19b18518b60c98462c60d91ed522b3543c5a96fb9391035bd6def65025e22fede175490f0c9c3f1051e43323ccb86d169659a30b875b34693ce2b95c51ec75cdfc43c0ddae1f201ffb2076b041d978b78893b2461f3cf2f233e79d1a3cbe68bad4ad7b193d0dd1d172960248b6fe6c937db9231fbdceb573666e001e68d08f310522e6262bd0cab172c57451beafcf46f03aa40d057902693ee680dd48bb4fccf8447098ed00815c562f3b6fab56075e8a9883a268d4504047d19ddb6ee720eb4872c8f603358c1e9743c5f70fa9ab26a74360546a692594fbbfc5ad3ddca918504e5da200b84194d093a29c7de3315bd0cbb8b78c9ff8d3d52186a758145ed0ba278d416831583b8caf6fa458bb205a81a1fb2031e150896e775f6e2026a8a0e0aef6489c09269f15e77bc34299feaf5466a6431ce5b3322251ce76387c2d056e35171c3ed8332c16fc555dadf832b6a37a58848015a74a8b114234038d428aad6a7e50a307046f0a2e9af80d6e532d096d80691de9834f2366594c13e28f8d38bf8a7a7b1e20d46a1eab06d364d66ee0a09c6548e9fdcc65a18212ec146a99aa94c9fe29961ef8b14d042eb26432f8e3c42afd939f7d2edc534b1cee8b7a4c5d9acb50210735486a2a44c40ce50f576d38aab670526802f20c12eeadc02501cc21ed1513da2a400f46db5b557fb206f78eaa2471db64c5313c87fcac5af14690daddb15497547c1b2336792e9cd1841d3687ac2a72fa023454f8cb6ca465a72615ebb975c74a59183fe491fab4d46b48b0f1c1ecc17e3655f613df613508900673b591f9c2826380fd743a4c603109049e470a3b3f87de208037ad1c95508dbc64b9df43418337dffc53d686277df9cb4f15e9b8559d662e93f2b9114db41cb279670604acd3229fbd11fb4a5321ca1753dcf4c69c356adaa1c387c1b6c9b983225b8a907cd8d04aa02d1d0babb2d6733d0a9ff88bd658af2a389db078f7e098eff5f1892a2aad6b5603c04c79a7cb697b3d6fd09cf6bc269da06f5ffcd52e8e42c4bf514d3f367c3f34698f5586f5649ed3c8b1581d01db8cb98142cac80fbdbb56688b4e9ce7f7dc4d8d36b7c700ef086c5f4d45cfc63877e257eb6b66e0b1ca196189b6ba5e58bdadbfe528ee6a83487587e6de7575c7df9967cc47385df4ef8c5eff0e1df62943d3561d11639eabff7dc7309aa4ad04346d4567e6a2d411ac3da53dcd4e4be40f6d044119dd09d36de3b19cf2e3b86816dc6540d23cff72809bd8d87cb1cc1390c92ef39aa158944445cb9e8fa9347de7453be9926d1ca1fd256e5184eef32248765a972c66f758f32279a7b97bbdb026565db49fc9d82a8268da2b7e5c5dbac7030bce478462abfaf8406b18b190a9647e3f4462c99e76a0e4a0e8e19068277cedb6eb64803717dc45b43b260179c4223b451b83e2fc00e91756fc2711af02da94d0811be0c107178949aa59d1c58ad00c23ac6bac6d73bb8f60e7e7669ed770970d11c63cacf42d31753b2cb8f0780e6f4f79829c5f29a8ab533e8ae5c41fdf72b1300bea57898ddccb60371da7162de32cf571396dad20f9a2628a359b3667a6614e45806d11d9f9c0bbd17645efcee640c2bfea0578c4f8e2e67ddc90c5a9b091ef1b753a477206d990b469596f62172f43b744b3170537794cde315148361fd774e65f6b74e6829a91384d5e0f77c45d0b0fcff2f25223fa1813ca39386d4286725ea55d24b80b683fd5c8c06a709c3e7b11f241a54c4cc9aac73dd510efc73389bccdabf156ffe6ff25322ceafcfd583676322baa44208ca750d2a15c1e2c630da95eb45553de7b51af3ff01b7d664eafa5362d7bea3d9780594f1a9478d0a990accf22bb3f35b9c9cc38c676627566591744a3949ef29eb83909cd3490525b952fa563ad698416da04b2fc765c9953c545043a7ee2edf6c1887925c6f5301d9dba67f3a67348071501451ea5bc407b76c6eb6eb5569e343ec033b0c607e8540cd58d5f30dd002edd2a55b99f121961e015609cb25a81db8268bffaac58e98384f99c02c7172ae2c4e86b39acfe729b83a91a61007f17513efa84600458c240789efa8af3b256c75ed40c3b59c1ea7ce40a6f382a24ed39b6ee078d4897f10f6a4e5f54212ee240fd65af2b401cefaa83b1d5ae89fc61f99ccbefdef386f444dc8000fae9ed7be43335ca75da500098e527f410ba561415530c73d19fb64a7541cca5eba88e7235929cf4c7a31bcbcc8ad5212b45d282117c385a1197a672353c9ca4b3078263999e23a48d14c08fb9d752134a96e22a727ac7f8fb95c7f9f91f6883cd6159de94a8625032b9feeae57d2710fb38258bccc143d70bf4ab50457cc8d9a7608531cb63c18e54fd175bd3cebcc0b993ed046502348adf4eb1038ed7360ced6f850b8232d9c56078ba51095e9b990254c942ef90027ac20e898249b12893ebf8e9da6da8138be19758790a9f3c1061d54ac5d10d462d70be855b0a9d29154f87241cf235f75854e362223c2326ad54afd1ebadcc682957c9450c55edaff774b4b441a45efda136e777fb226ce3f6574f0f42ca7ff0e4d3d94904903535b0e7ec2af6e8a5befd041dfa4261eeec51f62d5369db7a78e69148f1d26e0642e835a7e89b538c83c258e179b65f2ba8669de53eaf456e58a50ca2d00423d36fe3da728dad698604a781fc66c59ea5b09acae121a19cae18b6a2717408762f056a42d67abb9f24ba032ba951de84ef9d6519bbbcdf35bc4181ff2f5e5db96bbbb20b3e403468eb1c6bc282cb8fc65bb76be6df60c6c53b6bad7f56ef82823bfb7190346ea65b3b9457c3132447c78ca705dec6ade6bf8def8faf62b7f554f23d1f7e33668feab8ba4dc9c50ca50cf4e4c95e0c04bd994655af0bed4854a631092221c5c86f9bc506b43332d5160b26e93cd109f5b9c6d72682115100fefa6ad8003765f4b3e2f302319bc72e1946952cc6f45b0179dcb2eabf6c287272c62232500b8e8c0b54c5253c500c682d43b237fe2479b86c4f8c1aad58fe2d4e9dd990982ac9eeb475adf83b1c7d23d087209348ede07b0fa3b52ed2742c91f0e6138b2f2182ad6daa3fe3b44d93d1918542b74dbd29c9e25db09fd55a4e30e833427aab6c53389576dd804e6fdb01fec2ab2c631ea03572e210f61c57dd9ac309e8f79387b620aa7019565fffdd3566c3bc5b3f8efe4e3fe558ca1e19552fefde998a5de015f63b0a75ce7f79f4dc681d18dc298aae64b7cbfa74610b3f70ed35b69b81221d19c5ca6551037acc7771132327b79e0b2968fa689d7b9a170e3983e2b3a3ba160bb9248a96b25ebe87205a7db66035cf0cc21545adc4625648536d805b1c04ab5ca3ad3807d2a2659be4678b00c9fe547f6190f41de1e087cfd937220a70be478d03b9001d0af23c163bbeddaf4b3425f1b5e206664a9115f529953a4a3833751d67fc755eb1aabc96a4b4e81a6a93ce143dacff2f64f0ab58c2ec5b82e900ae6e3edc0a356d48619dae0e2ccabe79b757674630d1c78b928b43c0e84cf9a27e2ff6dc2149a467c58bb61f2e48ad727c2bd1426de5f8ce85ed0aa77310e0954eb32e716d03ed7552e15b5c6f1cbdfa7711fa25e8e46f35dffeb6048e773465aa9854305fecaeed845f1c6a61af769e529a50754445a15e1b40a46c9cd87b701c6ff69d5c7c6f5cf42567bda8795dca79356295029215b3b253744420cd1c510e7a942ce52bb5300d04dbca9612da0b6573af19965252b35f97942facbd9d851b689b43ac831703395dd715dd0471dda1c5947791feeee95d8d9283a3328598adbaaa85f50aeb432390e823cfb6cbbda8abc143680020d246c5dde4a6c45ae84fe11078713bc87c465e8d88f0b23e2804a6a3e19afebeeaa5a0f4c729db84107c6c8b7f838e251b0c174599d27f5fa92046baf6ad431fbef4df75bfaef0a79dbdbd6a2fae8a97abff4b9eeb078696bd95fc84d71195a9bbaeb1cf12989c2bdc7e643aed74b976ab9a7bf800e26079d1d04880276a4f035d4dc86f748936877948f17c5d588368966818ff5005857f8269bb3b5c09d7b3306c93d569f37850e0a7d5bce3ec80e408d2b3037ce36e2e5bbcb524e982d81fc165a4af4769ba0eaf15917b15832a5f5f84caa2cc0e92fcaa63c6bb4467c8cf8cc36b4bb15e7417311ca51566cef85116d8d06be1e99c610ad9709d112b1238ff2068392c344f011327913a5c12636fdb5964bd42f6d30209984318c43f4cf294e71bf60cada362c1b0d6486a1003e20980e81553c8ef46031aaedcd236f162eaa20710f70d09a2c952f4de152a527e1dd9db12b8d249a3a3fb2447cf8478306ccbaa186fe4ea5796887d49e1d3c79aa4eece23d6a56ab1589433f820ba79b197ce855adbb7edd7b5cbc04a54aedbae2b1a31c731b4cae59d906e4c5a4b5e8d08452976cf186cc9a3877e2de21e274f0cf6a67b5e72f2b6df5a33d2e0b99f191ab9f6eabe68efa3fc65f7831ea402e3e70e7cceb1827aabba5c152a5877c3ec5b878e352e4bfeed0cc1dcd87ec3271335bc552fdf45bb4aab3082913618658d57484fc49314030b71358e9c670dec4375aaa02d3c4f4d0a2e522cc5ee2dec627a76cc378153a485a63b9d5d8be6cafb2333fe256cf19870b05b443704fad102518d740c8dad5bf2384e898a31e05ba0caf2bfd6c5ff3604ed969942ff31abf36916e7a1fc6381b793d8c3a11d8d16eae961658febaaae3d12ae6c985154e5ac1d8d97967238ce7c3574dc40933f752feb0aafdf5296598fdbfd6ea59fc706930b7bf458637f8a86b9bd53d72789fef65c58cf337f44c083a62dffd92f1a974eaf3c8e7664ba5e8d8cec9ad36f0f3cb4c9c6742ae6388baf1d9a90c1a3c210457579d66eda6bb2e72bbd41c3ce0187019fee1f340be413688eddd1ed825ffa0dfaae6ca05926103a1299c936c93396636b3a20cd74c9e056966233cb4f46cd1eab34a1b9555bfeecddfc919ae507c33098353985a56c8aefa0c226d96f08f1ee03b72628438fe775ea604b7ea5e0961fc369c9cb124e7b9a5a783b96662910f290a06a7bc834626bc46f5740806097875932422c4fe22afd1feed280c92992c2de8adf0247ebe844f874358f090a4fede6394317b2ca117e5da09c371c8d0fe559d0f47a4718f188031966b8b44017d9e6e93d2bef58d794b037f917b4c5d0c07f0d5b324a3e06ab50b98b6bf69497c1d7cca0c75a30e5d6a41f69a136569deacaba73d8c5dda0357deef407549a384c355e1e7887c4b4caa5c17e7225d1cfdad942cebc4e7a56acf724311962a381cf8c07f6bdfc96dce731b897f8e7a9df070740dc7b5896827848f3d32085530cc24874ca162d19b692d47f46b4105153f5f12f2d9458b4cc3ddfd84aab512b21caf70f290fadefea740a802b12145ab73e13881285d3478c54483c18183a9e695014416408d36be0f1506595b188488135dac65bce6cd14b520eb9cc05c61f5be82ca63ba042202291229a1700d08d3a73d37449dc69e4a1cd8fbbc6e2783071f9bcc7e0548eaf27ea0d983f9e71bb5b8e8a1c7d4e7ef4081d67cca93cc574341b210122940d6b79528dda6fd900356cdc6b295672f081e6c81356d383225d2ab3160c52fd32e79f610feb484ef08dd6e3f32576188300d6541f04e634335dce57e823f1eeb395f2ac1045297ec747323bbe8b4c7d9b135f57e73151fe6ce8878033e29fe6f4092b8a4b72a3c6f8d3f75a29d55a30ad1bda3a0876971033afeeb265e2bd8468b3a2820dee47d609d6aeb5c19e1fc02d9cdf80ff4f26383bf67c651a8ecb35a627d039c432862a7ea1b0e512401dbab118f7b348259fc4ca6eb92ebe61597920d86d082ac9cf22dbd7b64ea865d47ecc029b475f946444737b1d65cde4d8cd6d37b1739e2eb35d85db2778d069c48ad2fc32b3b2060445cb9e21a0b593d9b0efe856129adbb339114e7cfa36e66e6d6a78669efc050037476a5e7621e8dcb22d4cb34bad4b5f5412842f29584652a506e99f5f7a1bd8975b423c3c8f7e425fb9e761b54218082cee2ec6ac9113e1b01cc2557bd8b50df49bc8339a5aaeaaf7fa3ad9b5f5ac4e2228a0dbfa9dc9cc2d20b5218a746b10cd6d6eece62004873972d993e43482b1e2368cc779caf2525ab7247bf1f99871806498c0c65c1a6792c82618b68b35b3dbbc3521c0950814e75482dd48c4e7b0cbf5f847a17b5abc724eb2f0a3763fc47d1e5c7355a88594516d62bdf562b34efeb5effa0e0a3006769b02844161fb7454c14524bf61f551078646a69e4f95bc9718f240729554d61e11976c06eb0343e16991be545dd1cb9813912f9a07f97a3393a1c36589139bb766e4106a8df1a3a16bc4b8baf8a57481b9f4a18f18791842ba70eb5d7206c7f0ff6b09f2a72114134c40ac69cedadc32ddc3a4d3fd7f0742f1d230d0cca8fb593faec125e31c35c58fb7f1c74cfc7349f390b20f3395d7b2de683c561564a63ecc21de7abe6d5926aa7d3179ff84ca259fee519737280a20d4855653c8e4493019669a77857f2ec52112d3f8530e5164273efc1284d00f0b78cc2760738264cbabefbd92f4dc070f881b61b8f3f3a9744e0529f7a91866e13f9f17f291d9797ff81f9053790e728453e963323ecf22d468c353cd205b9220ad90a080a05407bb8298cb4f8e8101c1f92f40fab00b6e031c0c11e967342168272379912b8c800cdfcdb6cda1de35370c3f8388d442db9ebc41ecf30a1c852640e9ebba9fdd92e26430fb845f607d51e1acc35337d00bf5de073d70657f4dc99f367f5379d180eac454f1da13d25aa12df958e0376d6fb6f5ce8bc825f6ebe0869bbd024612fa7454cc42556b8e4db355a7ecc67f658730a7fa7080a80dd0098bac2d3d80ec601da9e9557cc30ed6e6ed3ac76c6c42e311492d5420881e9a8332ca386bc6f84255484eb3b6b6ce76ea4914a1487a9ff3d77dd9df3e32b3bc0c0098771364b121a9946ee5a9c367be0c1dd7e2e146d739d19a825a203467b4f2697951daf6917e8932cc53bc55451ceacff704dd4d8335be43b95d7d09a515e4f1ff61190923c98287e9967508f103f4d3f4af82fb1e77fbedd161bb2ee63f422e8d1d53f8e293142839164a3af603c8ce431b88dc2370f91d8d0f4110b5c19873c4ff2262585bbc884d6bc287cebcc51384f1456d1c9154472eaed55bae2d5666198efab1ff93fa2c98ff645be56f78481cc94f2a74898bff48c8be6c19d31a7a0d35438432cc3b89ae59b643c4cc50d08d0c0ca0c5aa1eaa6e0582ce83734334128aa345ac752f23f0032ecbf733fd415822144c8a737adcf0bdb90f955e7f08cc077072c71f138ff0743450f736cc2f00a3ec9a2d862f0b8389c78baa250c7c9c3461881d3aa72242357d6f4f555bde0919fc263952a9baa8d244df5c379a03f599cd121948488f47bf50936a100ae2e3fcd95d3859896eb32fa7870636a6437a80399d45d8862b838a5459c1e777e233b4facba51b3c9c350a65ff4282af9f9db02d861d6bce8b781eff3ef15c21305aead6de0a1841c7c0fb16c48bac049cd1822f791a62a033b7f5c79f72de7571cbf70420d1b11785d96a20636c2373bfb87e95c2c76da998a5973407c0ff031b2971f92086a64c3ad0e019484e894b6ca52e170faedd293ed38bcb04662e4880d5d42c49a3485e7b59e5780fbd3e1a4849e929bc2b41cdaac44c0311ccfcc46763a62428b227acabd6f2a31632c26ec6f9683464e72c0550fa4062cc8feaf5235560673a3db4d560ca9050f849a6976d35f3cd764d0c4eb5e8117071096959492d8b6bfa9080671ec736d8225c6b42afa2445b8cd763286e002792cbd4087f4d5e8fa8e6750ab05a3f5ecf210570ef02b0b274f510b110dde50b60197a38b5a45b05b48a35c46f97a5809c70bf680b0ad3c6906b7386a67a09256f06568acae2b55605526d3de35b9c44d69df4b1052c9e09344c9a4447e6ef1df60f136efd8cf02921eb85e7ca1a8603e4154fdc8bec71f2d704df7a7399095dbb901b65ec079dd770920a527258f5566b531b5b0dcef20c0f40cdb9c804da5d1625e8f004f710e997f284fe7a9b5ca4d2e401acf95880cd337cef6293bef4a4ac5d06add6bfebe80e99605c41563e8791547d52dcfdc0aefa15af05952bcabcd491eeffefd6a15b2ad336ec751e1d774d00f8dd27331439b9a7a7231aefdf1e25612c56aeefb3ab935c733f74577bec6c960ebc76ad0df5f8af424bb35f943115472952924fa561de71afb5dda6dc5f5660197139a0105c3d991adfed0facae3ab07064489af90642df36ffff60a168b11bc38beed0cdd50ef19b6026846addfc499c1c2046111186e1c36e78297d610315776d14a38908d3554e6d0e326158e05f06dab7c5900c0115bf91d675c300a2d9bfb5398ce4603110ae66f2be22a76d7d3b308dbe086e5807960a78ba14a036f506c4cc1162638ec025de726c21321a30b3be0083ccb0e223e128fd8ae4c825e3e74c4ac5859b62f05f441be0ef917cf61ac3bebb5f6c35f71b8dd35ca5f10622cb9c441dc070540db5384da0d03dec59aaf216c95308d9fdcf997893fab34e3e4e35368e956ab894c21b861085a00eb81eb9b7322a0f24a457ec7be535776492c9b7a09c97132702006e53d472505abdd09b0e8a66f7cc875ef74bb4f8b5efdcd89400cc27e225f1eac1455128d736c75b069b367c4b38234f3a7687b7175658f54ad591da601bd0fb84d91b1bad4951289ccbcaec81ede493267ad1e2e42ea39a4c9d9221059f2bbd596d96bd3af907c3be4ecc2a245457e55bc7281ed4df495239d7218e5b166eaa9f9158a018dc47b6fb629ed7bb0209904ae9fedd41fec6977300d494c928164dc7d2d546bf599d7cd4c5c757be8d563e6461ca80b3f387e488a93fb6e83abf0172a09123a6f7757fcad6ea3c974cea4a48bd1fabb35429c3fb4ea77720c4c580209920b9583c6c5a5e145bf8edc7124e586a72d0c8a718a03b5fc0581b5cc624421ae89a5baeee76ff1cb11f1a928aa523d848825c52c123d95afecc30793d7857b46d2919b578503e4e6e01b625a0093ba52522f6eddfa955ded765f89097e6f1d432f82af6a6908c28076348f6eec4f05f7d4510eeb0bb2149de97ec49621a8118be8a07c8d5a2793801963b8ce372e245d56b42cbe0d3dc2a78e473ea386d85f0845128c291f0860ce12d066e1a9da216a1f7f50bec8dd6ff2ade3819676a80da08e0167a781523052a9a4200f09f8030879db0253043b24688e7915d9b9bf253d0a94936d48ae0cf425076fc02133d31eaaf0cbe588ffe9f0783280d217c7aec602f49ae53cc3fb9cd020da5cb7309893fbe532dc1dd0c472b5364ceb5865b123be480476243a0532f018e94b949a5a51b4e3c0a08b56bfd78bac008b4343353c86b95787caba329ece1624f68017223f1d11fb999a3ae10543bc2519a20942f9a26f1d82f4ec8bc8b24314c3424ff7457e986d2f112432e5249042ff3b5fe0425991ddce6b771798e3c7401ba4ece14b1e9f9ecbbabbbb5f784ddfa4d6eb18fff54820d3f705dff7abc930676c93e839464f15d35ba8835c851e85703d58f118bcbb6df95f3be9cb50716d0d63ee15f73ed05ab1bd9392c29529bacb374c7475f7134ca86d"), + %% SHAKE128 VariableOut + hexstr2bin("84e950051876050dc851fbd99e6247b8"), + hexstr2bin("1822b7cc3c4ea4f2440a362b117f808a"), + hexstr2bin("2ab3a70f3b01836d8efceb67490c3c38"), + hexstr2bin("0a13ad2c7a239b4ba73ea6592ae84ea9") + ], + [%% SHAKE128 ShortMsg + hexstr2bin("7f9c2ba4e88f827d616045507605853e"), + hexstr2bin("fa996dafaa208d72287c23bc4ed4bfd5"), + hexstr2bin("c7211512340734235bb8d3c4651495aa"), + hexstr2bin("ac71d8e087ae133f3da590e1a2b54d48"), + hexstr2bin("b4813895ae01b43c9d9ed85a8b03aaf4"), + %% SHAKE128 LongMsg + hexstr2bin("3109d9472ca436e805c6b3db2251a9bc"), + hexstr2bin("0f994e4aa804282cff22ad7d6229ef2c"), + %% SHAKE128 VariableOut + hexstr2bin("8599bd89f63a848c49ca593ec37a12c6"), + hexstr2bin("19e740d7d87bc322edeee86a05eb59b64bb86f90dc7b98f781720b7cac37fdaf293ce6bd047a14fe"), + hexstr2bin("ca7ca55bf123aba45287268c4050ab030b1415f4497d5fe8dbc5386ae37d24384a2fd6a715fcad48ff9e810c1d378fa70f1503767e9e338e33697206f863dc8015b4d1e9b8f81ddee22aac59d52055a1b0784a364369cc50f403045a1bdb25b639"), + hexstr2bin("5feaf99c15f48851943ff9baa6e5055d8377f0dd347aa4dbece51ad3a6d9ce0c01aee9fe2260b80a4673a909b532adcdd1e421c32d6460535b5fe392a58d2634979a5a104d6c470aa3306c400b061db91c463b2848297bca2bc26d1864ba49d7ff949ebca50fbf79a5e63716dc82b600bd52ca7437ed774d169f6bf02e46487956fba2230f34cd2a0485484d") + ], + [%% SHAKE128 ShortMsg + 128, + 128, + 128, + 128, + 128, + %% SHAKE128 LongMsg + 128, + 128, + %% SHAKE128 VariableOut + 128, + 320, + 776, + 1120 + ] + }. + +%%% https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/secure-hashing +sha3_shake256_test_vectors(shake256) -> + {[%% SHAKE256 ShortMsg + hexstr2bin(""), + hexstr2bin("0f"), + hexstr2bin("0dc1"), + hexstr2bin("a4d7897eaf5c49979b361c39a67f47e26c2f75e5ffe0645539d4de245138eb8cadaa45aef7fa0c7a732dbbce90c85be2bd4bf6e37dfb4fdebee4d0e0671fc45c3051c6ccb674799bcfda7a431a6e93b3db3e32f30636190a9a2e5620302876e0d4d2f6201353fac4554341df6efb591c6f100f5dc21a2aa176ba592bd7db69e14237bbf2371df6bbb072f9ecb1f714e621c97768d82eea6bf98ebf4a82c005262188ff894a5dd549866f88b00ee82bd99872515d71fac230ccb472c55a60"), + hexstr2bin("104fefe89f08d15d36a2233f42a7defa917c5ad2642e06cac56d5cc51ad914ecfb7d984f4199b9cf5fa5a03bf69207b9a353a9681c9cf6437bea0c49d9c3e3db1f3fc76519c70c40cc1dfdd70a9c150943c272cf9eeb861f485f10100c8f4a3e259c6470501932782512225ba64d70b219cf9d5013a21d25d6d65062dcc6b3deb49d58b90d18933f118df70ff42c807ccc851233a34a221eca56b38971ef858475488988794a975d3894633a19c1ae2f05e9b9c0756affd3cfe823ccf29228f60fa7e025bc39a79943325126409460926b057a3fb28a1b098b938872883804fd2bc245d7fd6d29bcda6ca6198f2eff6ea7e03ef78133de8ba65fc8c45a688160719fa1e7646d878ea44c4b5c2e16f48b"), + %% SHAKE256 LongMsg + hexstr2bin("dc5a100fa16df1583c79722a0d72833d3bf22c109b8889dbd35213c6bfce205813edae3242695cfd9f59b9a1c203c1b72ef1a5423147cb990b5316a85266675894e2644c3f9578cebe451a09e58c53788fe77a9e850943f8a275f830354b0593a762bac55e984db3e0661eca3cb83f67a6fb348e6177f7dee2df40c4322602f094953905681be3954fe44c4c902c8f6bba565a788b38f13411ba76ce0f9f6756a2a2687424c5435a51e62df7a8934b6e141f74c6ccf539e3782d22b5955d3baf1ab2cf7b5c3f74ec2f9447344e937957fd7f0bdfec56d5d25f61cde18c0986e244ecf780d6307e313117256948d4230ebb9ea62bb302cfe80d7dfebabc4a51d7687967ed5b416a139e974c005fff507a96"), + hexstr2bin("8488b6b0a5cb3b778d9e6169cddfdd7468127361da571661589a51e95a0ae5c057ccb457cff0dd588fa4298131a7a6b3e201914668f279a06328a206d234a8b1da3f6425c3f913be44d3b58c006eb4615053c62743e41477e3939c68ba51d4a5954990c388cf25d13293cf30421b50a53a70daac74172f0f624207b81948b2c31153fed826585bb8e34d3d150096c2a729c0957c02711290679fab9df8018a34d5c3770da67efd9cdf67c8e442c75290a5abbf40f5dc05217b114fc05e64a4206903c30a0f2852a4ac4b38450e96488834991f9f553f4fea2500bdc8535947ae5679aa76693ea3f9d45f55df6eebebb660f27e422b813914edd4bb5ee0c8bc06238ff4f9bda738c2df0de75b69ae8443b01ea4d07e8c0bc75bf6122a0b994c5158ac683c124d592853bcb56007c75b56ac8559f257470d2af8cc8ad3deb43cb77e37d0e770174e53bad38897511eb818dd8f6550bd9a86664bbfea026101d75f1c9af2a2dfed5e6a9bbc28a84e9be3b1caa58f91b86dc0a2dc38fc5e9b696472ba3c961368b7d3957fde5d3a07272348fc2da05e50a80107349d869094d22a17e08586e5125df3feeda1eff582a8c205f991615e8e3492bdda57513f5426727717dd89bee6c807ca6d6da583c90806114b37973a79f7bbeb56cf2514e44b84b9e6ac6647eade562487aa4889e42d389479890d991adac3ba316cc9b74b2ba807d65ae62728882a32c4c0a0b2d9019fb50ced8a2477c5f451f29507cf91ac26866e4fd106a8afc91cab1875a3b26a859d8bcdd5839aa194d921b4a504bfb8456036f4ef8e71397c0bc5188f07775230747e90b75d8b54ec7947306c00db364fbdb6af07658a108b279829b6842ea0e9616e9ef85a50c8445aeb712468f00d8ac477e9e333fe1a3e97aeb4b1c2e13cf88ce25f6023f3e3e4a395df95e703273f9e000b49a1d241dd848f1f496a970da51e623bb6607c8f137dcde286231e6bb572d97b0256106baf15c1d3462459fd383f769ac4b8f1b31a101ce2c1c448c136698ba3d6fbc082fd73ce3970972ef3f816a7e450d45c37418309470c738a1108ab9dea8c38e86d0fc53eb88e6c1ed82fcc7c81235cc402d0fe1d7930d8eb43745f97b7eeb492142d3497b76a4b319223887ed905f732fe0fabdee0f59a7340c2ae0274c218c55f9fe863a0cada700a1c733354be4dad1246b0d6e422612aee3abf6488d10b8a36ac0829da77c4e22864b5255b15073d26bc358dee81d7193624bb485be336e8819ebc72cd39af8fd54275dc54d3cdea23ba638a64d8a24700c7c93355c041e728bb2ec5e1d74480f7fddbdfa9c046bdef886155b0f06abc356c12afde2fd38f134c83aa7e1b2abc77de11b6c0a41bbd736e58763891d881bb5bbd435ef994e92b98e6a90ccb094b3e2d3ce90347f4f397b470756682939d7cdcf3783de62efc56e97d4bf3b08b3633328700d871776544a1d0dba18d44185f37dcd4e3be46d07d8570179b9256843d341884a9e94f1d607ea130507893fb4bb39fe1b849876e89b5e8966d39123d467d17fddd782223d4930afadd0c274d2a996527254f96dd60c10d4ecf6c686a3553adf1dce6997d892834c3af11a916e331552d90971e25ecf106c92af5ff12f2ff02ea0efb5b35668547122e4fc05e5ec3987b62eac54f138dfc8fe79a813d64c83d6fdb7e115352f26a70c28e152e8397559904103362bfdc16266c2493f375f88609ab3962dd37b6c564d395508d3af5b510dcd11166d7277d18235adf3d0cd6073ee6f2f7ddfc1a8176c92d18846b24e06654f5076443b658ed72060d2211a9e599f472775a8966c62e0c1e151afe6dfd26d343bc62ebd1d02bc3eec5ab6dba4b9f3d818b5dc6e2bb150430f057db1834d5c084d96bb7c00d5c7ddbaedf06515fda2ae6dbed9bd589b21a2a5143ea6daffd858005848fca3607089670c6c5b549d1216b0e12f089578db4d6dfe714b3932589ffe9b75b3973b5cce4b4689230320c8bd6ccd4ad444e2f3cfa434a3f340bd634c92d0b0353eeed319022ce56084214d1f0bf3cba4f961f185f257454363c97eea578896f61cc379f6061fff786a07059818d51fff256e5b2cd2b6e3c0251f01f5d723787aa381b7e1e1a035ff275a88d51de1c6474b70a700c003fdde070692fb7c41f347858223690f4e37494e845d4275ba3d21fe015bce34aef2117c28b877c16495c1154d32f7f98a620b322ce95cdabf02f39f98efd1bb287eff16edd8d740fbbe8be55b7d1b2ad544a994acb3a09770b650b2cfe9a7cdf0e2eb49ce8aa66587fc1c7835485f3dcf6da8c89c42aa467fa5add9fecfdce7a8cc8a97632461b999dcd740535b108cd92b7d50742cd6622c7f8f77ae08dd92a0aa4182145ca5c791bf60d04dfbfba2733cbbcebcc8b16517a25a0e662b265988eaa49c8a99cefa00323d9a7dc15d192defd885eebcc7e9b19cb210bfbd4396b23c28391e4269cb75d3c2117b21d15b89d35838726f864a9aed1c210325b1e4cced410a480cf2d662f282d7bec3d4e3e775bf38f05779122401fcb1d90b0dce399c8362a05fe685483d145fc0fdf43cf7558219d10b1cdafd89d845c9c393bd78b863e4bf91d172a601eb1c2a56abd119d1255dad00fdff87d3335f3de5f03d1acf7702ee693008fcff7b8f4364684acf972d76dccf791a437c9250ade8afb78050113fc53c8ec2cd644a4ba48da60499160ee7f8cd2a95c0e479dca2523ec759d8d71dcc924a2a980edbb3d659019906bc5e3389909f0c70b95e55a3a909e6f7d84d0f7cf0fecadd2db46a514cf76289187febf7a022e8e359b45e3a53bd90fabcc7975edf190e0d6f48c1ade0040ba0b90873db7272b589542058def8a65fd314fb47dab3257eb9321d7c349893feddfc6c6d9d9dfaece31706c7f9fe709e017f3c837923a0976114b6dc8dfd33b6a171303bc67918198130ec3e23db98e6063a2ffcc29716e28595897522846fcd8bb5e850de9a76a5d430552dd5168bd261f439f64f8404daa72329d89ab4a49e0d18f95f5e88832c5c73b6199645c5cda52bef126880d70a364bfc794f048ec2a638d7b9ceb9ca17aa99a383efdfc332405fb3d3b60419dbc38e95d9b7fa6ab5d78427f297a4b7e21f1091ff3a5b20caa3fe1cbcb09459d9df596a6c8e1ce4f211995e9c51dfd9365a56f5de0e3e6ad17085ae2fd098038eda3b0eb419d26a0204d6e00975d1aa6ebcfb63b2e3ffedf11684ecbf9ea5935554b1e742e3b1865ff1cb50dc8c10a805e76c5010541636a4068a3cfe8333a5e7cf3c889b8430b68698e52587fd235e964a384f78c727d164e0f4234f26d8a790ba69672073e676f04ac2e29e51604a8a55ddb64ec59b4abe06613423e1fb04fa969707642b49343787d2fc82217dd87cbacb2924c7f71f0a173fe43f5d116a8ff463edf912bf9ac6e093e7c5d5d294c7b98f1c7d56a1aae3640ec7d3e0c234cb444e8f31d2d00718e59448fbe6e51c05f2a804cafa419ad4ad5c33942563f02716b56fb0e41105c7f7427476ed4bbab1f69476901b5d4b583386dbf283ad293dec4713e8cf09133ea62d1ed9517d1d5635e2f65f323b58e8973d057193d44c8a2561f06bec0ebbf762c28c85abdfa05d20f1e8aaf7822ff058ed6493de9fe67662a4999cc3b87c8ccd91622e6509d051486fd1e5a3b85d29f8f66dcbbc56638d146d840dfe9cb16937182416158249bd2cbb56a1ff545cdfe80675abc59e5058cb4b2431f51207074fd536ef25bf690b4e42cb6242dcab9cb68c309672ee4943e7cc1eab3fbef9f178473f1c1e37bab61db4c4a916b7c5dad506132e5cbc9ae90a4eb7dc36f9e6c230ec912f7771bb682b8fc6d8f38c3834d3f291c5db8fd1b2a361a437936979c41a43d19bca03e77be812368461a07f1523d0311f87ee9a3b3393585fbd740fa184cb74d37e44cb8be40ab83dd9bb0706553d757dea4f40ed4b6248b71da034d51101ff891cead3f29fc7b067c4326c82e878d3736b2852334b4cf9d04c8475c93f85d60f1e3c9d4a8be7b394d99d1c154e30df9283106ab7c3b1e03db2773f0870b5acd80ad0941e38d936475251a23806f3fe5dea8de6f5354f9042082f1d1d451fc8966b3d34d8171abce1f6f67b559320c437d9f7534620548eaf85e28c723d22cdbca7d90bf2b72abc07100214088345a5a309e33c55058f3fb8d45809a1c97d30ad494a96088997b82935fb18cb47d13731d2e80e9c655e28007be611166a7d75fae7f84f841ed045cae147d6186161dd66cd6c4676c03fdc08e5233f6aa0a22227dc68b4ffd762c96eaa8ef87a60662cb461f6708e96a2d0548875e0f8fc99a9b6cf2252c653ddc776ea26ec5bb55c89ad390d194e692589560e84a25beeb2911a08ab9582049fd48638a61e56d0b398a3ab16ad30a8467b9fcfa60ffde78b6ce4416b19cd7a37552f0e932ad66319ce4a40a9b8cfe4743b945470667987928187e1e74be32512eda3e4548e64bcfd8f13824ca8969e3d2dd6256a52394f56938762b6a86d85d0e2b75290b58c7ebd7f61f866190f461fe6d7bafa6fc79f4607656c4fbbe72ceff8cbc1ae3ca808f803e80b8c66a47c8aeb0badb6fbaf07ef51943728f2ca7ade8443ba238051ec2bb10844363df0920caef45616a25a09c106b67bc8e9a2001b09c1b4872d24a8d70b352ae9eb6e271397244c9fe04325599b2ea2a60ce85d241dc8494f9f7acf687701481e1b97cc4e581e09c92214db27837f4446d907d29824c9d0f363ceb08e98fef4593ae7c20b564406090429ab99642ab931eaab00d5b338f408f6bafc87c580d811b5e8d8e91628ea05f30c074d91a3a9ed9e2c786b4db81b06d4aa240291296bde391551561c5efe167b368eb2a09c7ed07026a9c758a9ce539a36a77d907b281520e7cbd0dd38dad10923a2572557b0911d0405887890869593caf75727d8756d09fab8e7ed5908bc297dd2a97477d795d15f2c1d020ecc437dd1a43ddbd40c3c50a8a919a122004e588fa028327f6c5559827df3996707c395052e8030f03519918c0cd4ad4639fe12e38b61c3d0d76602388bbaa35be1bc95c006ccbbaf886b591f64a776184e41e51404c4cd0eb13e57cbd5c0c43fb294ec8ccd81c0849ed7f3dd3110703a95b05b9b9cff92ab7244e6c6dcb4509522c305d5d33e03f1b0b60e40029e4fffb8dfc2d4c440d919a3202bb400e3fcfe9aed6e35c85fea8996ac14d249af4a0a1016c77a1b56f4ebcc46931e02fe0dde836aa2270d65e8be8910841e86d212dc33fbe5d5f2907a94462eb96f235127a784d13e1422fe83033089f88f4a951ed8bd058a82fa9bde95a94f0b12129bcefc17979ea7a9784dea058eceaef566e291f3ba420b83c795828e3e04b1212c685ea454403bf171adf5708592f817d5f2aa708088ca3ea233cd3d6232c70f817811b36ddf5b9b6927981a04c1e53250c354cc0eeb597bba3de86f6211ce1e4992e7ed6aa87117cd4f546a82f800d565d9535eabd643db8e18667c943e30ddc33d338ac14836f71c89bcda9691d57183a908c0edd5f3de8c67c9ed9ef489eebcdb7744084af7a9e57b23f2b62bdd7748b7076b4b66e3e54380754ae8e1539165023581b60d5db0a6784c3b482283871387b1d65b05be8022070b1ce89108867a25f0cc411bb3abde15782ce3125201829faa9a833ede4fc6700dcc3460233e5acb3b9c3002a3dc0ae74a400d77e387047344206a7ebda4af3fcfe152fb2e298566787a194e98f1ab42350d2dcf9278178730e06741e71f2aeaf3e4aabea3c8989c29de534c2d6d3ddabaecda85f9553a5fd3830024e52fe7f60c5ec2170e9bf2b0d24174a7bbbf41056e778a3772cefb71ce60cb94225754d56ba83fb0d9242259143c03ddde584356a235c360b915c9415412302c4916002da4695a089df1b6093236775b0fd8b333089ffc7e16fa2263370fc439a3ec6853491ac78ac7751c35eeb945abcaa12f4f042ebc0c36cca53bd9b1fe58dd5afbeacc24feb3e03ec3539c4cf54c8951f5ff275a8877730da05faec155165fe24958a020761062af5a06bd6a1cb9e65bd5c7d4bbc86c564dd23338965eeea602bf215785a42cd9573eb3d48912704c3f1d3daecb12377b0d895b19a9723e4a4cfbd964dd2f65beb6efef74328d0c357c5ac527f0fc853bbd543b948345a1a7115de55a8f7e578d5efc1e1eeec2ea3c840cf2b2d5cdc232edb1596bac3fa5570073d9ab25078ecfbfc1c427267a96bbf6a25ad1d22c7adb0f3f38e1acc6b5d49acc3ad9265157e04a0f764f621aae5e34c942278e8f0baea24c5eb8dd714faa30c56d571be26f899e03a8da3a197d7dded05d06537c111847673f33d6d5ff42630df5c77e4112061c6cd06a16fda0a545661000d96a17c25f1e5ef5215b3127b1f4788cb5e40bf055998171713d946f18d8efb978df8c1759e0326f458781d4ae174a296b4199c2ca183072d15fe525c9c76062ca42fa91e84b6e249bd17de3a7f50f1ff6f06817fa777200dfeb0f83c8aff6cf464acfcc5abe4697cccf61af400c2bf2f166d34a5725aa8dc5656fe396013731654d882151c7605b30d7e9c206cb1524e31b87e470a16e8c5a35a2dfc259965bbfd1b02b3de55fb1609172b65cc07f483484e270d789128ee6f234d2de179880ae5de464af96796cf472a8e6b27abeaf6e497f79ec3b5b07f411926749d9b252c9687148076961d034f1d8edb85857fd08cfbb6a3f368c7a1a2a47f7760e78348988035fd5cc9d7ddaa5af35dee7dfa39c9e22bc31194b67b192c22e13245ec87712f716ab3e80d82df4785986b7ba25822b41fa72420c6372e19d5581b9da611abfe8f6f84da081d764e0b350367c1561fd95af734f43cfa69d3618424d7fa0dcd5459b3d6890dd4f8d64fdbbf299d5a48baa45a4af863ef53c28e0f54db30a3609895dfb1dc67cfec461733a5a97e8d26bdc5786c14823b734f239eb49190fdeb340c9a1dbb5c37da082ce5c0d0ae17404b84faa530e00951e0eadc4ae16240f68f868d230e36344a8356981584741f1ecdaa88fa34c67f76dfa96e20b4ed970a11cb60cabd3bedc51f5cf4aeae17cd49a9ffc975087cc569b229ff3ef07a28ae22a2ba1bb1b48bd24408a39237e04a48494b661d944a04b2c33235aa7f511b6a2ad4431cd0e835fa5a204ac90517bea3135606fac17864da788cb86e47ba99c6912aa4d7032114c4c96d945c020451e371e85215754d0c683bfe319232fdca4bbf69855433a550b8fab7265214ee298fcac8f275991225e47351481c2761d5801d5c8d9a46be105e53c6abb3251f712f85bf8d221610d6d7874da4160d2fe2ed11ca65b126b17c3f3d61c9871cdd4f4c0c95cb25d28c48c8e1baee87ceb01b1b047ab5a1b9c9bee2215228d7e171eb6dc6125685b7d1a23fe860cd785129a6915f352472e94dfbcf655f71aa5c236e1a1bfdd34aac260dc40407eaf07ee5abd788c35a2d9c845e8f73514c09753a00db7320ed7d98d2b1e09d76ecf73d2b5376e9ec00dabcb70ca9b1ef60fb1400c83936414a25651eb51a1b767c957642c5f5e878be7f302f30a58903568821bbefbbcc29924afa5242fbe7cf42d311732adcef2e3b26e031f00c6253d5b884b49cea9ea06ef43fb67f0a6749e5b8f606aab465d720d6b4087465a3a986942533f84c126e862c1c9195cf04ab0e4d0e436c04b23b149894fbe538b6cb1123ec7f2b6b915a81f9792dabd226bf64993f7199543d22c07368516b0e02811d6ec2e25ccf842d07643ed0ea9c364b90a472da582d7054cb6f18532d6def95527b8da2b4fe51afc27453c37ab49a18397690c4d215b298de948c4954e0e5cd5a5437085a8262a85731a04e82a2bcee08187f782f966cb8a1ebad7b07ec4cf3da01a65c76e31de071a30eb82aef0395a591293e9638047abb7b61306f1b8ccd7d88c4e632a7ec1fb6d3834b9c22d6d4b7e9745d2f82e3a7d54c91b1b4f40d24e8f377edb458362ccec8e84dfdc39f74e6284a1f23fc710796672647fdff38bfc64aa9bb247d62d0628a2607785ac53edc5792ca9a2e1661a7d3063872c57368bd934196ab9fdad31a2223c13270b0f73e7e34a7bba13b7eaf981bcc3dcd27e575d827c7714505fd9d97c9c3568cc76ab7a317feafdc462757bf4b4d839fbdff982043969df74b1594f1ff540a37745eedd025179e67a122474f80016fc49b7ecb747d4a573e14e5a639acbd76f81f9a880c4539afcbb2372650a13226c32b2ab8dbbd2dc9a0c8337c76ba47c4572ee731605fae0a8fe3309dcb9626d324d4cb075deb8518a7cfc8bdb0103719dc0a0d90d25cbcd83b9bd453c647259c70fd286002ef9b6fdb926c12ef96d8a0101bd44c6d22ac415841eaf14c9c79a92e100a69d3b1b7f4c377e6233c34717e7ac9040d369ac66c6d3f3b1e627d6923e67439cdd71921cfe7e700979ed249dea37d647d37ac518202fc8f8312d23c3221917d3216102029cf4b80171e58a9e6119dbd42774197aa9fe0adc763833bfd65cf9e41997cfd2a04092f4ac3aed9be4e93d704fe3fde0dcd535f935e78aead3d9663a1674701f90d8bd5d995bd218fe2c57a6cd5906fd213bd6b8f0999dd692e4e95fa885771bf72373a6b9c6bc981539d6e21031a9222fa869a626ecf0f5ac25ffcf6059698a3bc7725a9e1a118bb2b817acc90c78e250916b4b595008da88031603be5eabdce72b2603a6b485766f1db6815857dad02c8d6812f8415a07af02c8971303d27671c4fb1c84246cfa92e12fea3e92635e2bf2cc97945f78ef89bb93f41ef761f9742b8e1f86f85c5ad698e051fc4a96fec7781533ae08c9f4c083e4953297ded866d54b874f9eb84992ab744301eeea349773ee81b5bbe1d9bbf25704ee63ad95b248bfde4e8625396faf5857f00dd192047c950c4c0744e1049d47cda2e952b3ff0c57aef45d179cc5fb08eb4715e189a708c53dc17967cd02db535a00a762a59586af53e171f3f62e35028767a59e9df79edb80c660bed06f32e624881c51fd2e4e237796636ed810e3d1b6d22b6d516ea5649fbb7bde9d824a9cd7a9c6a52d11c9608473afdcb5067df2f42715d22684c2ecc511193ba6e478d3cdd624d0b7f89ec2f509c5c20f8769f2c9d6f3618fc25dd0ab87d75a7f1e4427819db569203b24d763d933b36a0c3e3d925252a01fb00cbcbd6014bf789843049d70f6e2e861f22a3b628825371d714ff839a1f238c6260d753a8fdf35daebf76bf077b293059d17f626219c093399b665fd6fec8d8d504f5d87774e396ea57c978ca66c928e7c4d2ea49e9c52d71ded65fe2b03606e01fef0d78960f5bc06008ab60185797516164f5e5d8a9670847172ec51cb53f0c198b99b24cbed5b942fcffd2781aa42012fc2c3fcfb66a853df07287298fabdf5a92252d7f392c5c656d1d276d5c78db3e64375eb7325e70272bd431c10a71e34acb58f215c987313123ac67e1633a0a4c62bd4a76666b526f8efbcacb70baa05100db08f8b40ec10fe5559abb791336a6bf660cf69ab7b17ce2905a2e07ad9dd8f755770e42eb93657cc0c9e3e42be6b342dcab1d166d18b6ee3aead418736245796f4841bd43309cb194fc40eca9a2dbb5c42fd521b30c857ca280a4ea3377024ed182776b411e1d4a7939bcdbb6f286dec9a10504b11751526f2786c71665753e222d9a960ae74f0f198a3ef7795f1c879108950891c082394ae462f544b308110a529184c91299ca19f6af8a1b700576c46c6c72788f80f9dbc120bde4e9228e33f7447da42865412485f5dac3aeaa740bd5748d900db45ac9c5a9671e8f2a7f42279bcbc3b01a3de040f9309a9f71480e4c94a001c18424bdf8e33b1c2e8aaae2e77eb54f321f86f52ea3d3c8cdc3bc74d8b4f2f334591e5e63b781034da9d7b941d5827037dee40c58dc0d74c00996e582bccc8dec9c12550d0d80fe75efd06f3c887b4d46be1fcdc48a24c6fea99f0b519576f773b3c649ffce81322dc45cade07fe80fe727fedf7c9661e7110d9f8b4d25385a510802e33fe0cccafd90421b36a311dda2814beb88e90cff819b5e107e555ade0c0190cbd72da9ec6dbb4b6045da3de00035cc98ee6a6d35d4bda340ed0e23aa2e0837a8ce9c60c106ad3f24160db54088a94a8fb800cbb294da37c3a136dde73f84be222d1438704fcfe611af25c224c3468c77ee95709e4859e342a2b6cebcc5ddeee2fb7f7a21d18707cd1a08aab437745cf99610a22079a46d65f9773e524a7a8f9040c46ffe123e6c005fa0b51a6e9c80bdba58aa60ae229dd3ad6077d055c569d405cb5c2c9c5ddb5384d32d7bf024c16a6157d0876f3b3d35c658c05f6f254354517c8cd6f74a1d6d938b58024376e7a894f45bbd455713324d4960271e02e6bb2cbf4622a96ffaf893af80d4f1f8ba215a179529b6dee973491aa04fedc51baf9e7cd1383c3339998d27eb1c8214027abba8c4f44c38859fc2b3c80061d4a8d8b09b764138fda4fa88dbe1007c31865ebf9031458fceaf5fc0fc2061049855e069047d65233701c38cb3aca869b60f736fb524450606315b4cddef9b5d79ec6cd29f2ecb8748199ca297933dd00d6561d200e41c3949d23a6d3e7fd5f4e03327f155d09ceb98b42bd06fdde73a9d50d1f059bbf1b00059a52c0716385233f4b6d00cadffd88f9a1b2023d3cd2ece986bd2a9efecf4918f438f20fdecfedbf5c477ad0b1dd59592129a9bbd18340804d67fd6c9378c663cbda1717c188041fff7ec21331e6a1265b845bfe3efd4b35dbbffca91f700e528ba67e2dae461ccd9f7394e6d291f6edbb729788c7b97376bc15b6b8f57d9781ae3eb5c617928e34d6d661e01667d11bcb44eac01e404fdf6919aa30b7d9e86dbe034c557ff5df02101797836fe0b63d05abd448b51a46bc6d2c4594c91d77734584a07059d895a1e5a7c43551951cddf4e7d2970eafdeaa22de2759bbc0cb2eaf33608b37c33890c217669a6ded8b5b03b9641f8d999e3544afbde4025c6a1c85affb2a9c4e3e5b3fd70d1717627840cc711766d4300bca2d0177e83f68cab9535adeb2e9c21d6da977163fe09c7a3bb9af5af1553b8df55683e3cab54e738182b499d8b2c76c44b96810794be3e78855d4ad43a2e4bb235b9790002695eb01ae5d9af099d6917cca3a8a57dbd166d32b818cc22ff419f096ccfb2ce5d797c630b6809ee1783372666ef25ff2fd6e98952688115ec200c9b28493b5858c002bb44fb9382c14a8c21865d6f633c075453d441da5d1d7263b196ee81e881bcabea389c350fc4487f84acfb6cf10b301558e5acbf41bbbe0b145dc66dc600f4dfb79052d7db480576197fd002f0c5d996602eb01808ee8d6c8c26d9e739fef007ecf426612f7408daa6a8e41aaa918b3e335755cdfbdd66eee09930d88aa339894f0b1ebb5370d914f4ce3f9d6598cc759807a3c762b1d1f9da5dd226021656cfa97e45cd8f65008acb9aea716a0127a359e6a3ec859156f5401bc7bbe780cf3fc1f4a0802b7b05a6f2b4ec8001f521303c9d5c25aba67e4adb7a187d81082d43611639ba6ed971cf33c34db0719c44850ad4d65b1d18d0607323cc561ad7f7424f8c1764a2a8cc6c546f15eb041d17088a321a74240ed5a0d0bdeb2df814b5a454fd42670d0f9ce71e624d1800a1222286602f5a06b01370b9f17ee964e8f0d03812c3a9f7376ba75941fd1fae4fee3bc4061df30b271439f7678b74a67b2e30c372a114aa47c014cdb72be8565002bc0ecbed297d9b606453c61db506f4fa2bbdc5e48f6d682496416ae15a40cbe60b2a57a6e2a2337d5792b73994cc7bd929edb5950187c21e957f02265fb3e92392e0535008c418f57c7fe1e9f93f3a55921ce8ba54417c36794d2803def9d240b8fa67f4822972503ae1ed34cca856f5cf12065e6fe3bff4aa74847ec058adc5e43e3a9938196814f87f8223f54ffa414697b8863ab5b2e191bcbda2eb19a23b8076754712de22cb202249bc5d7c4cc26073a63c9dfe7e8494ec7b744883db29f368bf06f8bfc9fc1992e909ea7774ff856d778780084a651cce68febfe07d17a5ddc1dfd20385304b970b1285879b811e4fd370cb193f0d92282a473976b38dfe9c23d988f37dee1ef957de17f6e9ce91df64fd667a85035cc369579c8fa314666f8cfd02d7dabefa7f7475860a857048e6cf38397343cfe467ce37d2717533451ca45229f7c32847f30bdfbc0c98efa92bbf292d4ba1fa90db00eec1645273036c14e0e39a61b56d1bc3249f4349d11e78dad7284cc80bcae9a88453b2625b082784af1909dcecd4bf454d19f9c1a00e93c84a13bce08af9576e026ea1cc73bc159bc8b48d5b56e4a93962117427e3e145328ad7e4100084eef96ffc7bb94388fd02329a6c067906a045696efea423fe56c0a05a0c81f4d576f2d3e4865054d2a961f5b5ee06dfa06b146c1add60e7cf8e1993867c5bee53b700360e0bba2d03df5f1aeb5d83101cd34c72fc5982b7bbb5e66b427bc5766679cca162511e568468a4a726ffa493e0a09684fcbc331b2a92ce307552e7157683fe1bc8a2a558bc0cd9a74d1e92ef569a99a9024c9a9c84ee7aeb094aea52eb9e3dad284476ba1292a61f480075afe488fee9f653e0c1329191a4e5a9f6b62e92315e6a80bb04f0c4b219ffed7458f4346c4cc0172c139a43bc61f1b732c8a5278336769e99bfbd68a69c5a2fb774ed648cdb49e1a71925cca32c15bec710b924db3d34e63a72e04729b1410d678a37e1069d77f8785b3bf3ed0c3904dd16d253e07c8da0c6a5c329e45f9cceac233a73819b33a6c9614581430a45d29a63597a3bfa671364e0cdd67eae53bbadc65de1a5db6631bfb4a93f4c2c2170297eb46294bfac883dac6615a0a64e1c93fb0bf51b67f6e284dbd1f2b17cb626b0ef4dd391d442477990f6aff2507539854ba9758b47cd01405e23651902ea3c583f8b95dd4013983f15e56d1bb3b159ac3ca02a9c81d1d61ac6095f2dc5a6f948f5d56f1e0b05c3b7c19c15c4d7e8c5305858dcf71feca83b99eaa795cdd3a7a999952afd3e020f0b1f816f94c503c638273953442805c1cb622e06f822578ca2b3328e21e0f1c660b4fd453484eba2ec30ac562aa50367f1a63c44fecd1cf05ccc146333f9b92ee10a795d4b1c86a175fb17af519dd0e3d5ef7e2950d5081dfe6ba371da92448fb3a85b808296be0dfb10e3246218537da0c45b7c9642055ab1194780bfb1793301160c7681f97a2ca5f8797e051f747c7aae3b07771cfaee9d5d685789614061f1ddf507ef4da2489c3a3f84e1b81b2a8a3486909c6cce22e9bf6ebfe1a78160ab847dfb759b1da7085e9f009fee30cdf9dea5e0ad3a6b3ebec7c1806f8d247958930b19baead59110ec00b8e6c69baeee34febf518cb88a43fec962da1a12f42092092a6742b150dc6dca765f4a441592bffaf5306c9d4da1ebee2dcb5631192a0adef623ab992b95696400b28114490d060149f3e102d1ff1799a8e94a6a2f9f83f245972e75696366381e8673f19486917c8442f3cf5c8608309cddfda680a683addf6ad22d536deead9a8329c43c25e9d16ec4abec375d481dd14e77cb526894778c45a0aa31dc882b3b52aa768a14956acf221042aaae5fc60fe5c1c2ee0d3db813d09ed24bd7bd1ac89bfeedc6781dad6f042bb7fc9ed3951e4ae2749cbc07d83db2d68ee88713e30a052e13387c0a041610023477572ef0c524e2c8a49d783fd29450c7511414ecc93941cf2845e3d68bf1a0a26101f741cd1bcaa54dd91a559a1d1363cfa5489eccdf3f08ddfe308b974d0ae703c3b7efffd8a1f5e09875835cf6274ca8623661dd9e31a8a0cd7a8b02cd4aeda5c6f56221036f9fa5010f2fe2c694682474669ec21e1e96b9da473d9418669a8f323902fb443d8be43d72c3f23662a18d0360bb44080eba704e220b14d769d8082e0c37574112a1e57326a555ee9add2936ad222b0409fc0c51905c4cb1588e59e2f469fba5f6c9efccb13fd39544064236d4393fb91844d70a1b99024f3da4cb777b00b016f1eb4c709df733d366031e2b5d6a48d0427a17a24a28ff581f01a41529643e3c0cef428672373f520abc053cd11c0df090861541a5720b75078f2ce385fd6dc14ebd1a37b666bdb737194191b6b657c20850190db5dfed4efc35ad0a1734054ac03d1caaf3eb76adf602195397a2eeeeb5c16b3101d86f8a06db6db2676fcd8268588f08b03032da4e5fd4aaf9479a1b9cbf52d312c357d12d69b501fe178e0e50f4bd78769cb8e64718bbbd660d23f1c733181c6e096ecd7ac8cd226395109980faec23cc97d27ae91516b5655c501a3d0603339f137c24f4defd3b1e4f1bc9fe5365303a89463e3d3f2db97f9cea3b061e5c402d48497ea4948d75b8af7746d4e570c848daddd94a7c47f23a35b9ff2b656a6d9e0f9c8490d3c4f7b81cebb90a9a54137ff35ff418b5ad882190850bb3c277e07cd23fddc281eceaa6fe41caa6595b38663a9436b0d3a8a2b913160aa61cc9c14ef8f83242874b750bbc3f20b209cb608618444819bc7e688c2734a20c5da9399de9902b16ce29c16dc0ae643594edb265f636d15b5a77032135a67bee6c5f70f00ccf6f483dd0f55771553d38e109578d3a337a7f93dd9b00a3353e5331338dcfcb7ca7e0bb873a4e37fe54c2d86776864e03eb8c51f657b2daa99344946db8c28d617ae43295b8dc7a6f50db614e38d6986ecb8a7b0c60be43ca21c87d541b52bb44915e89439688a271db8b6408c507823cc282f965deac455f91e7d431db06bbee9355a8d9fdda17cc9adbc9db44a172347a7bb349f80547328aa9814c2367f62729d2779f145b424837d27810dd2ad21f987d9002d64fb309b5176ced19fcd4b6fbdcdddc049289ad5a795aeeba2c09c5c7907bd7a115f0adba387e5ce54dbe65b07bc5daf40855c2aad655bf002e7feeadbd590f81dc66e640159de0b0e21afaf1b714df0d430f50d0f452c9dab99c62f20b13f39d2e7d7befb4b2c3a244513e6599b33476cd1d1f73d83a566e54c8e83f8391cb1e929d36711fa64be2faccf9b5288a21582ee9f73c8cf3ace4876b4a3c2bbbdf2b912edd6b5e5fc1998cf2f3886d81eddbecd9a0d317671365daed4e900d7976edb298398877c76dee832af3876e81e2269157e1df817cd519d6b6bde331b4d98c7d43126d74c5960a65dec35a35704662a5ca8409d21c3927022e5edf9d52512f7a9830c55c408514bd7529a8ca3862b6ae9a4ff8132e86b7e6befb2a5b3d24fe32fd5a36382283351057ee1933bf7e4b362e0b544c9a05f30bcc5a6c735b9a222cdb5a680f52d695e8ebda567aefe249c4f080c78348fe01f8c2444eeb96fdc3f09bf605e9b8c65bcc9bb407a7aed5d7e14775f7c220b80935068177cdd5930807a3f7b2cf3c9db045cf46d4c0e6a01d139bb38c10097d344b02d9912100a61af85d2419c7c6a1bbdaed5663cf7e8c44e555032f76ee93a9c11c1441edfe7c214b2f47e2dc3e1ca5135ead5fdb5edf74dcdb2b88fab443911fbef9313bb95d44898326b5138c5495788a136eed6ee1d48296f33ea744d514ee4f5209121893ab39c08b38769892b492d94f380c96dc8d5eee25fa82869159b537e5707b03657fe34e3c160c4a1d29e8b42b6640be16c6f84c87639562bf89fcdd7b0f7c621ae80f9501f5dd8b3f1d02816da091ced8a8d916e322e7c1a6cb4cabdb8e201dfb0d46fda47ef2413eb869a9de8d08ae6a5b88b6ffde80898f482b08cf946c1d50170feeeb5852566321a49546d6ac4614068d92a34069a4c228245c1dc96a48612df3d0a02b05b0f6813bac6d23867e1ed2711499efa9f88792bc108ed721877578790b89a114393fffb7551a1b9c3fcfe6bba2a894486f26ae7d8aea6399c3d52029076f6086d785c49ffa1766f86f716bf3a697d74ff0e6a5ee33f918dbe2ca4c8b2c1910637412f87b022aada2f84a88a2872f5fc562119f9773829beece9a2428e3515ab4f72cc1d4d51c98a0c0a3664fad38d5b2afedc301b34f97da6d8dd18792dde680481c4a30dbb4cefcb1c32c82807ae01ebec33ab35926b2a8a9ed7b30a3865bcf4f2ab91a7cbcf8beb2f403c368bc291277da2c35cfd0945f680049ac7c1461a9f49d371720fabdecaa7c139bcdc85895c27cc0107b15e60794106482874fa2171fbfebb12f38c69083a48391176414b087ef1ecbc7bc50a91463f43a2affc320b8d6fe6f113cbab124b47c2eae59056aae7c6fee1a8f794bd041205dcfeb6fcda72b97d92ece425875f425f8fa6ed4cb1f6bdb02db540af8b0f41072dddc4c25d406df78eeb8068c5eaba779646d29dbc304898e74d4aab59cc3ed01bc9a023027aefed3bac1ad55c7232939521caba3808fa21a4cd7028c7cf5df92d3ed3af6896d1b73f8b0e0bb98a2d4bf2d45032a3f534d2ecb25ba8438d30dae8c9d9c80e0895ede124e46c7b971cf935eaed978e999b353598579a5b8a5a0dc4e71cfd5164bf09e6e63888012d8433d4b20d49664a9fb105ec0e7f8e5b3bfa582327cfd11fbf93b76b74def8c1e58455e629c8b9acb4eee429ead84a54493908ea84a01bbe1a03b90113e1a0d8d3534fe3eda5f7147d4746de77352e1a14425db9be2e8fa450e28263a020b7e6b0392927a361bd364b42a4863e0688213f9ecd1d3ebbab86e6b899ffb42cac801e9e584df57c6a76b179839da9dd06ee0da6ed8170e7fdca5dee7631aa65dd19710b73b35e668d5a42447a91352928f5010325931f3404e328fc9da5fd5ce8375a08f1f448a0a445704be01d7a6b4cbadc3c0bae74105f65da7b76c947825c5e2c0f698d3ecb2deccde85a7ee41400c3d1aa16502a0d4bc6e4e39817f36687e2898730eacb36b8a4bdb250520cc3d394a3e5cc04abab788acabd881eab8a1988d518ee995883a301b23258135efb43a1a5779358634b54260e71bd49ba6d7fa5a13cf91571329b985dd0d85f451acf7f58c812b80b9d6d2d2298be25aa7138cd605bf6fbafb3f0bd1326678269b2d572505fb0afeef23204a9aae13f1635c364b50e5e985e9840f4749b6c293744045f1eb6f39c97ea31410620fd02683bae0ff1bc968828401ad2e4b88cb0385eba6a0666522a90f5f976ba8337fc3af35723b8ff4557775bdced05f61fa3e7638637b9d662c654a5d8fa594a48a8e8ed6d447d4ed162d63c05873aecbb90bee2f64cab9f9bfc7d6a7e60450e95750c9a4ef6b113c156cf71335605f4d703495d791d5e1a743955c4582d1018a50d007a7a9ef6f6ab8fbd9bd4bdf9b70857f1438d2e33ef72098833d4d53d20fdbd8f1e8eb1a15fe69a4b0c684378aba61b5135a103df871bf98f4992dd4e901622e2f6e9e362b808f2d2437c6b253b56c9e3946312c369d865d56610c9a6ee84c9b6842a889db79915fdf17fb961fe90e012f031ff62004deae529bd1ee8a4a01e7446af35c4b70d902866e0a2c15ddb0a79d99c0e3df2d30549051641864b1e9da6df065f6a3edcedec445dbbc5e19b8592911fe3d9ba0e218b78aad6026a46c3c88dbe8a10c78d8ba3ea3568e212590dd7e72c04a7be65ec06f4f930ef5acf2d4403463a3ae9a5f609f9063e9878a9efb42c4ed54a8860a1462a70169fc50e36b3a02bd45b0dacb968d14ea17b8cf4c7d1e0f6fa14a0850de369adcba6018a7c9214de0cf3ac18525b038d35edb7c0129b95c2d7988f597e493300d494c8ca6b69b1eb38976d6f3d5b2030607181924c9bcb20f268ad446ef22727c5ee63548fa507c1eba8aa17e6d387e6b59982f1f23ce1afdef792dc7c78a671141569af65cea6f09ac9eaa8eb7edafeb9e83455900989bdf1b7d8981347417b539c54316a60eff5ca5db7a8c7976a06c1c01e43a27c13d9df3bd6edcfe888b4e905bb9129e6635ffee6898de7c7e59d71d38932a62f20820bf626740ec6dcac58c3f83500d9a18dbcf6e8cee88abe015e753da9177d3054ec793f915c8ccdc03769a293aba52b0d68f15df2365a53f0dc27841d72f0b3b105279aec3584ccaef969334366f3506d9788418e4da157ea6f8d051645243b34fa93b0db265969b7033532baef2eb21cdd0e4bf9bb0a3a3cad08291a1a82273f115039e4bb6515770960196090c1d0a98e05c5cafaff6b86091b68af4f85185dfdcf6eaf4152607d608c6feaf8573c724f01578889fb7c44f8496c2c60c77f002463e781648bdb28acfcae9082594625748379a8e488dd081f147c380aa3dbd419d03d01a18bd60733334e27cb429e21b2fe94b297221e3dd20f415889b4bf383276c7b664dcf197b7e2c6e73f9d95057ef5312c1e41b054b58c33e48a7489ab22ea20064af9ae689f6dc1755bd4db3f3ce9e77be67223eb3e2dcc82695ddbbb5f8cde77b05c7ce86e56a648ac82c13d615f20288c3de97fd06683b8c2fb15d74cd13b91219fdda5891fedd7ae21f79ea0b763c87c0f817120c8f1de13b6d88d7f79ffd93b1e511e843c10b8dd3ceeb16fa53ac232518c7d83c78f88d82b844c6bd973b6596267b1e42da752c62e0744228b0dbc5bce5ad38de4c53f8cc3b9a296624fcf0906920e2d872a4bd59fe1956deb64fd34b08d31a3932b78ae19d91e83b6dc69c6756091d4bb11eaa7ecfe43b648d92b79b2841679653abab888732eb849b24d1e2943db5e4be6d599afcdcd7025c2cd774eefe053dd7b7c38ba1841a89d59bf7caccc494ebc21834742c0913004ba39d9e0212de76e5633ab0928f5e9144b576e7a5a7a0a8e4a8a1d60874d8a5848170e00b85d989a53878c0035b9de50ff9b601c8edc3624da40a8637240c3b2c4db703b25fddbd40bde5829bf1c1ce60ef20e0dc9ee4632494d64d622198086e90dcd09ea9d45cde7be9c00358b0925c201a4b8ac3ebf192016df33174655570d1460c776ba5aca6f8ed40e29854792d130d0c7a62bf55bf7237404b245a695a985d82e7ab38f6c1be57cbe1ba93cdfb5062e953864ad7d26f9cfc59395e8e556e89f99473758157559129a8c457d4ed607fabbe3473bef48004ee02bc5b0a89d9323826ca44978f430122dec187d82ccace1a627622544c2ed7dfefb438261d63c00846b98d7e9112117e2bb60b1fdf16b5fde8533fdf03007d97c829dd2a39949b319c46387f8649280439adcbf12fcd9e1df7d4b3a8407f5ede3424325e07f32b7bcf7c2de29671ad7dadadae984476f04338ff64f72c31e27cdbab1e09ac08d9ae886054efe8110ab232bf9e5b4599133552de3b14efa0723866c98c75535b0b916efc95eea5b1a6686eea83815bbf6fbcee792046b05474db7d8361e822752aed2a57d926f4dae96a09364353ad6f6c9fb4d6fcd697d4522dc7e386ab41dd9f8a637906e0fe123b7facabc719643172a84bffb50ccda872f6edf0e306d91bd130c26b0664eae4046eff52f71ba78de99d5cfc35307a583efe96b2604012b827a7ee34bffae1d5ac00a932bba09f0c39478e"), + %% SHAKE256 VariableOut + hexstr2bin("3e20cf32669fa3fd6e94e519b52a1dba33cd1f3a6947975e9829e4db326d2a18"), + hexstr2bin("6ae23f058f0f2264a18cd609acc26dd4dbc00f5c3ee9e13ecaea2bb5a2f0bb6b"), + hexstr2bin("e3ef127eadfafaf40408cebb28705df30b68d99dfa1893507ef3062d85461715"), + hexstr2bin("afc9ef4e2e46c719120b68a65aa872273d0873fc6ea353859ff6f034443005e6"), + hexstr2bin("8d8001e2c096f1b88e7c9224a086efd4797fbf74a8033a2d422a2b6b8f6747e4") + ], + [%% SHAKE256 ShortMsg + hexstr2bin("46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f"), + hexstr2bin("aabb07488ff9edd05d6a603b7791b60a16d45093608f1badc0c9cc9a9154f215"), + hexstr2bin("8e2df9d379bb034aee064e965f960ebb418a9bb535025fb96427f678cf207877"), + hexstr2bin("9510ff5231813a865918badd0011f05915364165492ef17b85929a63e4951589"), + hexstr2bin("46293a63c235750d58a24edca5ba637b96cae74325c6c8122c4155c0d15805e6"), + %% SHAKE256 LongMsg + hexstr2bin("2bac5716803a9cda8f9e84365ab0a681327b5ba34fdedfb1c12e6e807f45284b"), + hexstr2bin("51b560fe5c3cc4c9e457e65f15f1b1619d18dbac916ca83a67a4d022301d5229"), + %% SHAKE256 VariableOut + hexstr2bin("3389aea66244b91428f089"), + hexstr2bin("b9b92544fb25cfe4ec6fe437d8da2bbe00f7bdaface3de97b8775a44d753c3adca3f7c6f183cc864"), + hexstr2bin("7314002948c057006d4fc21e3e19c258fb5bdd57728fe93c9c6ef265b6d9f559ca73da32c427e135ba0db900d9003b19c9cf116f542a760418b1a435ac75ed5ab4ef151808c3849c3bce11c3cd285dd75e5c9fd0a0b32a89640a68e6e5b270f966f33911cfdffd03488b52b4c7fd1b2219de133e77519c426a63b9d8afac2ccab273ebd23765616b04446d6ac403f46ac0c147eda629eb7583c8bd00dc7c30fcd6711b36f99f80ac"), + hexstr2bin("45c65255731e3679b4662f55b02bc5d1c8038a1d778fe91144a5c7d3a286c78c54f52135134a3c6a19a9e6e546de21b2e8a7e280290709f0e482a51bffa95137a381268d10195862818309b2a4954c656d1725c7ad1a29973162832d62afd538cf74e1b70d1775a9f77dc7c7380ea034f5b1869af46c1c26bce29e1980f0de9e55543e7eda19a56453c8b7d58a28ad7a33bc243c7242ffda5409cfd8f8ffd4b350c6d0023f27f93e9eb46a871367706170074d8a2080f0a8b68b8fc6b14b8b4da256e9e64dcb7771640e992eea2334e641"), + hexstr2bin("2e975f6a8a14f0704d51b13667d8195c219f71e6345696c49fa4b9d08e9225d3d39393425152c97e71dd24601c11abcfa0f12f53c680bd3ae757b8134a9c10d429615869217fdd5885c4db174985703a6d6de94a667eac3023443a8337ae1bc601b76d7d38ec3c34463105f0d3949d78e562a039e4469548b609395de5a4fd43c46ca9fd6ee29ada5efc07d84d553249450dab4a49c483ded250c9338f85cd937ae66bb436f3b4026e859fda1ca571432f3bfc09e7c03ca4d183b741111ca0483d0edabc03feb23b17ee48e844ba2408d9dcfd0139d2e8c7310125aee801c61ab7900d1efc47c078281766f361c5e6111346235e1dc38325666c") + ], + [%% SHAKE256 ShortMsg + 256, + 256, + 256, + 256, + 256, + %% SHAKE256 LongMsg + 256, + 256, + %% SHAKE256 VariableOut + 88, + 320, + 1344, + 1672, + 2000 + ] + }. %%% http://www.wolfgang-ehrhardt.de/hmac-sha3-testvectors.html diff --git a/lib/dialyzer/README b/lib/dialyzer/README index 951a92469b..bbdab21cbf 100644 --- a/lib/dialyzer/README +++ b/lib/dialyzer/README @@ -29,10 +29,11 @@ the Erlang/OTP installation for discrepancies. The complete set of Dialyzer options is: dialyzer [--help] [--version] [--shell] [--quiet] [--verbose] - [-pa dir]* [--plt plt] [-Ddefine]* [-I include_dir]* - [--output_plt file] [-Wwarn]* [--src] + [-pa dir]* [--plt plt] [-Ddefine]* [-I include_dir]* + [--output_plt file] [-Wwarn]* [--src] [-c applications] [-r applications] [-o outfile] [--build_plt] [--add_to_plt] [--remove_from_plt] [--check_plt] + [--incremental] [--plt_info] [--get_warnings] Use "dialyzer --help" to see an explanation of these options as well as diff --git a/lib/dialyzer/doc/src/dialyzer.xml b/lib/dialyzer/doc/src/dialyzer.xml index 63a6fb8aca..753c5f2257 100644 --- a/lib/dialyzer/doc/src/dialyzer.xml +++ b/lib/dialyzer/doc/src/dialyzer.xml @@ -86,7 +86,7 @@ dialyzer [--add_to_plt] [--apps applications] [--build_plt] [--check_plt] [-Ddefine]* [-Dname]* [--dump_callgraph file] [--error_location flag] [files_or_dirs] [--fullpath] [--get_warnings] [--gui] [--help] [-I include_dir]* - [--no_check_plt] [--no_indentation] [-o outfile] + [--incremental] [--no_check_plt] [--no_indentation] [-o outfile] [--output_plt file] [-pa dir]* [--plt plt] [--plt_info] [--plts plt*] [--quiet] [-r dirs] [--raw] [--remove_from_plt] [--shell] [--src] [--statistics] [--verbose] [--version] @@ -195,6 +195,15 @@ dialyzer --apps inets ssl ./ebin ../other_lib/ebin/my_module.beam</code> <p>Skip the PLT check when running Dialyzer. This is useful when working with installed PLTs that never change.</p> </item> + <tag><c>--incremental</c></tag> + <item> + <p>The analysis starts from an existing incremental PLT, or builds one from + scratch if one does not exist, and runs the minimal amount of additional + analysis to report all issues in the given set of apps. Notably, incremental + PLT files are not compatible with "classic" PLT files, and vice versa. + The initial incremental PLT will be updated unless an alternative output + incremental PLT is given.</p> + </item> <tag><c>--no_indentation</c></tag> <item> <p>Do not insert line breaks in types, contracts, and Erlang diff --git a/lib/dialyzer/doc/src/dialyzer_chapter.xml b/lib/dialyzer/doc/src/dialyzer_chapter.xml index 8f0abe3403..0f68639424 100644 --- a/lib/dialyzer/doc/src/dialyzer_chapter.xml +++ b/lib/dialyzer/doc/src/dialyzer_chapter.xml @@ -227,6 +227,139 @@ dialyzer -I my_includes -DDEBUG -Dvsn=42 -I one_more_dir</code> </section> <section> + <title>Dialyzer's Model of Analysis</title> + <p>Dialyzer operates somewhere between a classical type checker and a more + general static-analysis tool: It checks and consumes function specs, + yet doesn't require them, and it can find bugs across modules which consider + the dataflow of the programs under analysis. This means Dialyzer can find + genuine bugs in complex code, and is pragmatic in the face of missing + specs or limited information about the codebase, only reporting issues + which it can prove have the potential to cause a genuine issue at runtime. + This means Dialyzer will sometimes not report every bug, since it cannot + always find this proof. + </p> + <section> + <title>How Dialyzer Utilises Function Specifications</title> + <p>Dialyzer infers types for all top-level functions in a module. If the module + also has a spec given in the source-code, Dialyzer will compare the inferred + type to the spec. The comparison checks, for each argument and the return, + that the inferred and specified types overlap - which is to say, the types have + at least one possible runtime value in common. Notice that Dialyzer does not + check that one type contains a subset of values of the other, or that they're + precisely equal: This allows Dialyzer to make simplifying assumptions to preserve + performance and avoid reporting program flows which could potentially succeed at + runtime. + </p> + + <p>If the inferred and specified types do not overlap, Dialyzer will warn that + the spec is invalid with respect to the implementation. If they do overlap, + however, Dialyzer will proceed under the assumption that the correct type for + the given function is the intersection of the inferred type and the specified + type (the rationale being that the user may know something that Dialyzer itself + cannot deduce). One implication of this is that if the user gives a spec for + a function which overlaps with Dialyzer's inferred type, but is more restrictive, + Dialyzer will trust those restrictions. This may then generate an error elsewhere + which follows from the erroneously restricted spec. + </p> + + <p><em>Examples:</em></p> + + <p>Non-overlapping argument:</p> + + <code> +-spec foo(boolean()) -> string(). +%% Dialyzer will infer: foo(integer()) -> string(). +foo(N) -> + integer_to_list(N).</code> + + <p>Since the type of the argument in the spec is different from + the type that Dialyzer inferred, Dialyzer will generate the + following warning:</p> + + <pre> +some_module.erl:7:2: Invalid type specification for function some_module:foo/1. + The success typing is t:foo + (integer()) -> string() + But the spec is t:foo + (boolean()) -> string() + They do not overlap in the 1st argument</pre> + + <p>Non-overlapping return:</p> + + <code> +-spec bar(a | b) -> atom(). +%% Dialyzer will infer: bar(a | b) -> binary(). +bar(a) -> <<"a">>; +bar(b) -> <<"b">>.</code> + + <p>Since the return value in the spec and the return value inferred + by Dialyzer are different, Dialyzer will generate the following + warning:</p> + + <pre> +some_module.erl:11:2: Invalid type specification for function some_module:bar/1. + The success typing is t:bar + ('a' | 'b') -> <<_:8>> + But the spec is t:bar + ('a' | 'b') -> atom() + The return types do not overlap</pre> + + <p>Overlapping spec and inferred type:</p> + + <code> +-spec baz(a | b) -> non_neg_integer(). +%% Dialyzer will infer: baz(b | c | d) -> -1 | 0 | 1. +baz(b) -> -1; +baz(c) -> 0; +baz(d) -> 1.</code> + + <p>Dialyzer will "trust" the spec and using the intersection of + the spec and inferred type:</p> + +<pre> +baz(b) -> 0 | 1.</pre> + + <p>Notice how the <c>c</c> and <c>d</c> from the argument to <c>baz/1</c> + and the <c>-1</c> in the return from the inferred type were + dropped once the spec and inferred type were intersected. + This could result in warnings being emitted for later functions.</p> + + <p>For example, if <c>baz/1</c> is called like this:</p> + +<code> +call_baz1(A) -> + case baz(A) of + -1 -> negative; + 0 -> zero; + 1 -> positive + end.</code> + + <p>Dialyzer will generate the following warning:</p> + + <pre> +some_module.erl:25:9: The pattern + -1 can never match the type + 0 | 1</pre> + + <p>If <c>baz/1</c> is called like this:</p> + + <code> +call_baz2() -> + baz(a).</code> + + <p>Dialyzer will generate the following warnings:</p> + + <pre> +some_module.erl:30:1: Function call_baz2/0 has no local return +some_module.erl:31:9: The call t:baz + ('a') will never return since it differs in the 1st argument + from the success typing arguments: + ('b' | 'c' | 'd')</pre> + + </section> + </section> + + <section> <title>Feedback and Bug Reports</title> <p>We very much welcome user feedback - even wishlists! If you notice anything weird, especially if Dialyzer reports diff --git a/lib/dialyzer/src/Makefile b/lib/dialyzer/src/Makefile index c934ecdc2b..2f0f1f6b71 100644 --- a/lib/dialyzer/src/Makefile +++ b/lib/dialyzer/src/Makefile @@ -62,7 +62,10 @@ MODULES = \ dialyzer_dot \ dialyzer_explanation \ dialyzer_gui_wx \ + dialyzer_incremental \ dialyzer_options \ + dialyzer_iplt \ + dialyzer_cplt \ dialyzer_plt \ dialyzer_succ_typings \ dialyzer_timing \ @@ -118,6 +121,12 @@ $(EBIN)/dialyzer_cl_parse.$(EMULATOR): dialyzer_cl_parse.erl ../vsn.mk $(EBIN)/dialyzer_plt.$(EMULATOR): dialyzer_plt.erl ../vsn.mk $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) dialyzer_plt.erl +$(EBIN)/dialyzer_cplt.$(EMULATOR): dialyzer_cplt.erl ../vsn.mk + $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) dialyzer_cplt.erl + +$(EBIN)/dialyzer_iplt.$(EMULATOR): dialyzer_iplt.erl ../vsn.mk + $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) dialyzer_iplt.erl + $(EBIN)/dialyzer_gui_wx.$(EMULATOR): dialyzer_gui_wx.erl ../vsn.mk $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) dialyzer_gui_wx.erl diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index f9c7304b9f..9693aa66cd 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -39,8 +39,11 @@ dialyzer_dot, dialyzer_explanation, dialyzer_gui_wx, + dialyzer_incremental, dialyzer_options, dialyzer_plt, + dialyzer_cplt, + dialyzer_iplt, dialyzer_succ_typings, dialyzer_typesig, dialyzer_utils, @@ -53,6 +56,6 @@ {registered, []}, {applications, [compiler, kernel, stdlib]}, {env, []}, - {runtime_dependencies, ["wx-2.0","syntax_tools-2.0","stdlib-3.15", + {runtime_dependencies, ["wx-2.0","syntax_tools-2.0","stdlib-4.0", "kernel-8.0","erts-12.0", "compiler-8.0"]}]}. diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index 83d3c03e7e..04335334aa 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -29,6 +29,8 @@ %%-------------------------------------------------------------------- -export([plain_cl/0, run/1, + run_report_modules_analyzed/1, + run_report_modules_changed_and_analyzed/1, gui/0, gui/1, plt_info/1, @@ -41,6 +43,14 @@ %% Interfaces: %% - plain_cl/0 : to be used ONLY by the dialyzer C program. %% - run/1: Erlang interface for a command line-like analysis +%% - run_report_modules_analyzed/1: Erlang interface for a command line-like +%% analysis, but also returns the list of modules that +%% had to be analyzed to compute the result +%% - run_report_modules_analyzed/1: Erlang interface for a command line-like +%% analysis, but also returns the list of modules that +%% had to be analyzed to compute the result, plus the +%% set of modules that have changed since the PLT was +%% created (if applicable) %% - gui/0/1: Erlang interface for the gui. %% - format_warning/1: Get the string representation of a warning. %% - format_warning/2: Likewise, but with an option whether @@ -88,6 +98,7 @@ cl_check_init(#options{analysis_type = AnalType} = Opts) -> plt_build -> {ok, ?RET_NOTHING_SUSPICIOUS}; plt_add -> {ok, ?RET_NOTHING_SUSPICIOUS}; plt_remove -> {ok, ?RET_NOTHING_SUSPICIOUS}; + incremental -> {ok, ?RET_NOTHING_SUSPICIOUS}; Other when Other =:= succ_typings; Other =:= plt_check -> F = fun() -> NewOpts = Opts#options{analysis_type = plt_check}, @@ -109,16 +120,37 @@ print_plt_info(#options{init_plts = PLTs, output_file = OutputFile}) -> get_plt_info([PLT|PLTs]) -> String = - case dialyzer_plt:included_files(PLT) of - {ok, Files} -> - io_lib:format("The PLT ~ts includes the following files:\n~tp\n\n", - [PLT, Files]); - {error, read_error} -> - Msg = io_lib:format("Could not read the PLT file ~tp\n\n", [PLT]), - throw({dialyzer_error, Msg}); - {error, no_such_file} -> - Msg = io_lib:format("The PLT file ~tp does not exist\n\n", [PLT]), - throw({dialyzer_error, Msg}) + case dialyzer_plt:plt_kind(PLT) of + cplt -> + case dialyzer_cplt:included_files(PLT) of + {ok, Files} -> + io_lib:format("The classic PLT ~ts includes the following files:\n~tp\n\n", + [PLT, Files]); + {error, read_error} -> + Msg = io_lib:format("Could not read the classic PLT file ~tp\n\n", [PLT]), + throw({dialyzer_error, Msg}); + {error, no_such_file} -> + Msg = io_lib:format("The classic PLT file ~tp does not exist\n\n", [PLT]), + throw({dialyzer_error, Msg}) + end; + iplt -> + case dialyzer_iplt:included_modules(PLT) of + {ok, Modules} -> + io_lib:format("The incremental PLT ~ts includes the following modules:\n~tp\n\n", + [PLT, Modules]); + {error, read_error} -> + Msg = io_lib:format("Could not read the incremental PLT file ~tp\n\n", [PLT]), + throw({dialyzer_error, Msg}); + {error, no_such_file} -> + Msg = io_lib:format("The incremental PLT file ~tp does not exist\n\n", [PLT]), + throw({dialyzer_error, Msg}) + end; + bad_file -> + Msg = io_lib:format("Could not read the PLT file ~tp\n\n", [PLT]), + throw({dialyzer_error, Msg}); + no_file -> + Msg = io_lib:format("The PLT file ~tp does not exist\n\n", [PLT]), + throw({dialyzer_error, Msg}) end, String ++ get_plt_info(PLTs); get_plt_info([]) -> "". @@ -142,10 +174,17 @@ do_print_plt_info(PLTInfo, OutputFile) -> end. cl(Opts) -> - F = fun() -> - {Ret, _Warnings} = dialyzer_cl:start(Opts), - Ret - end, + F = + fun() -> + {Ret, _Warnings} = + case Opts#options.analysis_type of + incremental -> + dialyzer_incremental:start(Opts); + _ -> + dialyzer_cl:start(Opts) + end, + Ret + end, doit(F). -spec run(Options) -> Warnings when @@ -153,15 +192,41 @@ cl(Opts) -> Warnings :: [dial_warning()]. run(Opts) -> + {Warnings, _ModulesAnalyzed} = run_report_modules_analyzed(Opts), + Warnings. + +-spec run_report_modules_analyzed(Options) -> {Warnings, ModulesAnalyzed} when + Options :: [dial_option()], + Warnings :: [dial_warning()], + ModulesAnalyzed :: [module()]. + +-spec run_report_modules_changed_and_analyzed(Options) -> {Warnings, ModulesChanged, ModulesAnalyzed} when + Options :: [dial_option()], + Warnings :: [dial_warning()], + ModulesChanged :: undefined | [module()], + ModulesAnalyzed :: [module()]. + +run_report_modules_analyzed(Opts) -> + {Warnings, _ModulesChanged, ModulesAnalyzed} = run_report_modules_changed_and_analyzed(Opts), + {Warnings, ModulesAnalyzed}. + +run_report_modules_changed_and_analyzed(Opts) -> try dialyzer_options:build([{report_mode, quiet}, {erlang_mode, true}|Opts]) of {error, Msg} -> throw({dialyzer_error, Msg}); OptsRecord -> ok = check_init(OptsRecord), - case dialyzer_cl:start(OptsRecord) of - {?RET_DISCREPANCIES, Warnings} -> Warnings; - {?RET_NOTHING_SUSPICIOUS, _} -> [] + AnalysisResult = + case OptsRecord#options.analysis_type of + incremental -> + dialyzer_incremental:start_report_modules_changed_and_analyzed(OptsRecord); + _ -> + dialyzer_cl:start_report_modules_changed_and_analyzed(OptsRecord) + end, + case AnalysisResult of + {{?RET_DISCREPANCIES, Warnings}, ModulesChanged, ModulesAnalyzed} -> {Warnings, ModulesChanged, ModulesAnalyzed}; + {{?RET_NOTHING_SUSPICIOUS, _}, ModulesChanged, ModulesAnalyzed} -> {[], ModulesChanged, ModulesAnalyzed} end catch throw:{dialyzer_error, ErrorMsg} -> @@ -219,15 +284,26 @@ check_gui_options(#options{analysis_type = Mode}) -> throw({dialyzer_error, Msg}). -spec plt_info(Plt) -> - {'ok', Result} | {'error', Reason} when + {'ok', ClassicResult | IncrementalResult } | {'error', Reason} when Plt :: file:filename(), - Result :: [{'files', [file:filename()]}], + ClassicResult :: [{'files', [file:filename()]}], + IncrementalResult :: {incremental, [{'modules', [module()]}]}, Reason :: 'not_valid' | 'no_such_file' | 'read_error'. plt_info(Plt) -> - case dialyzer_plt:included_files(Plt) of - {ok, Files} -> {ok, [{files, Files}]}; - Error -> Error + case dialyzer_plt:plt_kind(Plt) of + cplt -> + case dialyzer_cplt:included_files(Plt) of + {ok, Files} -> {ok, [{files, Files}]}; + Error -> Error + end; + iplt -> + case dialyzer_iplt:included_modules(Plt) of + {ok, Modules} -> {ok, {incremental, [{modules, Modules}]}}; + Error -> Error + end; + bad_file -> {error, not_valid}; + no_file -> {error, no_such_file} end. @@ -440,9 +516,16 @@ message_to_string({contract_range, [Contract, M, F, ArgStrings, " return for ~tw~ts on position ~s is ~ts\n", [con(M, F, Contract, I), F, a(ArgStrings, I), pos(Location, E), t(CRet, I)]); -message_to_string({invalid_contract, [M, F, A, Sig]}, I, _E) -> - io_lib:format("Invalid type specification for function ~w:~tw/~w." - " The success typing is ~ts\n", [M, F, A, sig(Sig, I)]); +message_to_string({invalid_contract, [M, F, A, none, Contract, Sig]}, I, _E) -> + io_lib:format("Invalid type specification for function ~w:~tw/~w.\n" + " The success typing is ~ts\n" + " But the spec is ~ts\n", [M, F, A, con(M, F, Sig, I), con(M, F, Contract, I)]); +message_to_string({invalid_contract, [M, F, A, InvalidContractDetails, Contract, Sig]}, I, _E) -> + io_lib:format("Invalid type specification for function ~w:~tw/~w.\n" + " The success typing is ~ts\n" + " But the spec is ~ts\n" + "~ts", + [M, F, A, con(M, F, Sig, I), con(M, F, Contract, I), format_invalid_contract_details(InvalidContractDetails)]); message_to_string({contract_with_opaque, [M, F, A, OpaqueType, SigType]}, I, _E) -> io_lib:format("The specification for ~w:~tw/~w" @@ -510,19 +593,19 @@ message_to_string({callback_type_mismatch, [B, F, A, ST, CT]}, I, _E) -> " the callback of the ~w behaviour\n", [F, A, t("("++ST++")", I), t(CT, I), B]); message_to_string({callback_arg_type_mismatch, [B, F, A, N, ST, CT]}, I, _E) -> - io_lib:format("The inferred type for the ~s argument of ~tw/~w (~ts) is" - " not a supertype of ~ts, which is expected type for this" + io_lib:format("The inferred type for the ~s argument of ~tw/~w (~ts)" + " has nothing in common with ~ts, which is expected type for this" " argument in the callback of the ~w behaviour\n", [ordinal(N), F, A, t(ST, I), t(CT, I), B]); message_to_string({callback_spec_type_mismatch, [B, F, A, ST, CT]}, I, _E) -> - io_lib:format("The return type ~ts in the specification of ~tw/~w is not a" - " subtype of ~ts, which is the expected return type for the" + io_lib:format("The return type ~ts in the specification of ~tw/~w has nothing" + " in common with ~ts, which is the expected return type for the" " callback of the ~w behaviour\n", [t(ST, I), F, A, t(CT, I), B]); message_to_string({callback_spec_arg_type_mismatch, [B, F, A, N, ST, CT]}, I, _E) -> - io_lib:format("The specified type for the ~ts argument of ~tw/~w (~ts) is" - " not a supertype of ~ts, which is expected type for this" + io_lib:format("The specified type for the ~ts argument of ~tw/~w (~ts) has" + " nothing in common with ~ts, which is expected type for this" " argument in the callback of the ~w behaviour\n", [ordinal(N), F, A, t(ST, I), t(CT, I), B]); message_to_string({callback_missing, [B, F, A]}, _I, _E) -> @@ -545,6 +628,27 @@ message_to_string({unknown_behaviour, B}, _I, _E) -> %% Auxiliary functions below %%----------------------------------------------------------------------------- +format_invalid_contract_details({InvalidArgIdxs, IsRangeInvalid}) -> + ArgOrd = form_position_string(InvalidArgIdxs), + ArgDesc = + case InvalidArgIdxs of + [] -> ""; + [_] -> io_lib:format("They do not overlap in the ~ts argument", [ArgOrd]); + [_|_] -> io_lib:format("They do not overlap in the ~ts arguments", [ArgOrd]) + end, + RangeDesc = + case IsRangeInvalid of + true -> "return types do not overlap"; + false -> "" + end, + case {ArgDesc, RangeDesc} of + {"", ""} -> ""; + {"", [_|_]} -> io_lib:format(" The ~ts\n", [RangeDesc]); + {[_|_], ""} -> io_lib:format(" ~ts\n", [ArgDesc]); + {[_|_], [_|_]} -> io_lib:format(" ~ts, and the ~ts\n", [ArgDesc, RangeDesc]) + end. + + call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, {IsOverloaded, Contract}, I) -> PositionString = form_position_string(ArgNs), diff --git a/lib/dialyzer/src/dialyzer.hrl b/lib/dialyzer/src/dialyzer.hrl index 8f96de884e..c1fcda169a 100644 --- a/lib/dialyzer/src/dialyzer.hrl +++ b/lib/dialyzer/src/dialyzer.hrl @@ -108,7 +108,7 @@ %%-------------------------------------------------------------------- -type anal_type() :: 'succ_typings' | 'plt_build'. --type anal_type1() :: anal_type() | 'plt_add' | 'plt_check' | 'plt_remove'. +-type anal_type1() :: anal_type() | 'plt_add' | 'plt_check' | 'plt_remove' | 'incremental'. -type contr_constr() :: {'subtype', erl_types:erl_type(), erl_types:erl_type()}. -type contract_pair() :: {erl_types:erl_type(), [contr_constr()]}. -type dial_define() :: {atom(), term()}. @@ -142,13 +142,16 @@ | {'plts', [FileName :: file:filename()]} | {'include_dirs', [DirName :: file:filename()]} | {'output_file', FileName :: file:filename()} + | {'metrics_file', FileName :: file:filename()} + | {'module_lookup_file', FileName :: file:filename()} | {'output_plt', FileName :: file:filename()} | {'check_plt', boolean()} | {'analysis_type', 'succ_typings' | 'plt_add' | 'plt_build' | 'plt_check' | - 'plt_remove'} + 'plt_remove' | + 'incremental'} | {'warnings', [warn_option()]} | {'get_warnings', boolean()} | {'error_location', error_location()}. @@ -172,7 +175,21 @@ -define(ERROR_LOCATION, column). -type doc_plt() :: 'undefined' | dialyzer_plt:plt(). --record(plt_info, {files :: [dialyzer_plt:file_md5()], mod_deps :: dict:dict()}). +-record(plt_info, {files :: [dialyzer_cplt:file_md5()], + mod_deps = dict:new() :: dialyzer_callgraph:mod_deps()}). +-record(iplt_info, {files :: [dialyzer_iplt:module_md5()], + mod_deps = dict:new() :: dialyzer_callgraph:mod_deps(), + warning_map = none :: 'none' | dialyzer_iplt:warning_map(), + legal_warnings = none :: none | dial_warn_tags()}). + +-record(plt, {info :: ets:tid(), %% {mfa() | integer(), ret_args_types()} + types :: ets:tid(), %% {module(), erl_types:type_table()} + contracts :: ets:tid(), %% {mfa(), #contract{}} + callbacks :: ets:tid(), %% {module(), + %% [{mfa(), + %% dialyzer_contracts:file_contract()}] + exported_types :: ets:tid() %% {module(), sets:set()} + }). -record(analysis, {analysis_pid :: pid() | 'undefined', type = succ_typings :: anal_type(), @@ -187,15 +204,18 @@ timing = false :: boolean() | 'debug', timing_server = none :: dialyzer_timing:timing_server(), callgraph_file = "" :: file:filename(), + mod_deps_file = "" :: file:filename(), solvers :: [solver()]}). -record(options, {files = [] :: [file:filename()], files_rec = [] :: [file:filename()], + warning_files = [] :: [file:filename()], + warning_files_rec = [] :: [file:filename()], analysis_type = succ_typings :: anal_type1(), timing = false :: boolean() | 'debug', defines = [] :: [dial_define()], from = byte_code :: start_from(), - get_warnings = maybe :: boolean() | 'maybe', + get_warnings = 'maybe' :: boolean() | 'maybe', init_plts = [] :: [file:filename()], include_dirs = [] :: [file:filename()], output_plt = none :: 'none' | file:filename(), @@ -208,14 +228,18 @@ filename_opt = basename :: filename_opt(), indent_opt = ?INDENT_OPT :: iopt(), callgraph_file = "" :: file:filename(), + mod_deps_file = "" :: file:filename(), check_plt = true :: boolean(), error_location = ?ERROR_LOCATION :: error_location(), + metrics_file = none :: none | file:filename(), + module_lookup_file = none :: none | file:filename(), solvers = [] :: [solver()]}). -record(contract, {contracts = [] :: [contract_pair()], args = [] :: [erl_types:erl_type()], forms = [] :: [{_, _}]}). + %%-------------------------------------------------------------------- -define(timing(Server, Msg, Var, Expr), diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index ee795a54dc..b17ae873dd 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -147,6 +147,7 @@ analysis_start(Parent, Analysis, LegalWarnings) -> State2 = analyze_callgraph(Callgraph, State1), ModTypeDeps = dict:from_list(maps:to_list(dialyzer_typegraph:module_type_deps(Analysis#analysis.use_contracts, CServer, Modules))), ModDeps = dialyzer_callgraph:merge_module_deps(ModCallDeps, ModTypeDeps), + dump_mod_deps(ModDeps, State, Analysis), send_mod_deps(Parent, ModDeps), #analysis_state{plt = Plt2, doc_plt = DocPlt, @@ -372,7 +373,7 @@ cleanup_callgraph(#analysis_state{plt = InitPlt, parent = Parent, CServer, Callgraph, Modules) -> ModCallDeps = dialyzer_callgraph:module_call_deps(Callgraph), {Callgraph1, ExtCalls} = dialyzer_callgraph:remove_external(Callgraph), - ExtCalls1 = [Call || Call = {_From, To} <- ExtCalls, + ExtCalls1 = [Call || Call = {_From1, To} <- ExtCalls, not dialyzer_plt:contains_mfa(InitPlt, To)], {BadCalls1, RealExtCalls} = if ExtCalls1 =:= [] -> {[], []}; @@ -380,11 +381,11 @@ cleanup_callgraph(#analysis_state{plt = InitPlt, parent = Parent, ModuleSet = sets:from_list(Modules, [{version, 2}]), PltModuleSet = dialyzer_plt:all_modules(InitPlt), AllModules = sets:union(ModuleSet, PltModuleSet), - Pred = fun({_From, {M, _F, _A}}) -> sets:is_element(M, AllModules) end, + Pred = fun({_From2, {M, _F, _A}}) -> sets:is_element(M, AllModules) end, lists:partition(Pred, ExtCalls1) end, NonLocalCalls = dialyzer_callgraph:non_local_calls(Callgraph1), - BadCalls2 = [Call || Call = {_From, To} <- NonLocalCalls, + BadCalls2 = [Call || Call = {_From3, To} <- NonLocalCalls, not dialyzer_codeserver:is_exported(To, CServer)], case BadCalls1 ++ BadCalls2 of [] -> ok; @@ -583,7 +584,7 @@ is_ok_fun({_Filename, _Loc, {_M, _F, _A} = MFA}, Codeserver) -> is_ok_tag(Tag, {_F, _L, MorMFA}, Codeserver) -> not dialyzer_utils:is_suppressed_tag(MorMFA, Tag, Codeserver). - + send_analysis_done(Parent, Plt, DocPlt) -> Parent ! {self(), done, Plt, DocPlt}, ok. @@ -617,8 +618,7 @@ format_bad_calls([{{_, _, _}, {_, module_info, A}}|Left], CodeServer, Acc) format_bad_calls(Left, CodeServer, Acc); format_bad_calls([{FromMFA, {M, F, A} = To}|Left], CodeServer, Acc) -> Msg = {call_to_missing, [M, F, A]}, - {File, Loc} = find_call_file_and_location(FromMFA, To, CodeServer), - WarningInfo = {File, Loc, FromMFA}, + WarningInfo = find_call_file_and_location(FromMFA, To, CodeServer), NewAcc = [{?WARN_CALLGRAPH, WarningInfo, Msg}|Acc], format_bad_calls(Left, CodeServer, NewAcc); format_bad_calls([], _CodeServer, Acc) -> @@ -640,7 +640,7 @@ find_call_file_and_location({Module, _, _} = FromMFA, ToMFA, CodeServer) -> Ann = cerl:get_ann(SubTree), File = get_file(CodeServer, Module, Ann), Location = get_location(SubTree), - [{File, Location}|Acc]; + [{File, Location, FromMFA}|Acc]; {erlang, make_fun, 3} -> [CA1, CA2, CA3] = cerl:call_args(SubTree), case @@ -657,7 +657,7 @@ find_call_file_and_location({Module, _, _} = FromMFA, ToMFA, CodeServer) -> ToMFA -> Ann = cerl:get_ann(SubTree), [{get_file(CodeServer, Module, Ann), - get_location(SubTree)}|Acc]; + get_location(SubTree), FromMFA}|Acc]; _ -> Acc end; @@ -712,3 +712,26 @@ dump_callgraph(CallGraph, State, #analysis{callgraph_file = File}, _Ext) -> [File, Reason]), send_log(State#analysis_state.parent, Msg) end. + + +dump_mod_deps(_ModDeps, _State, #analysis{mod_deps_file = ""}) -> ok; +dump_mod_deps(ModDeps, State, #analysis{mod_deps_file = File} = Analysis) -> + Extension = filename:extension(File), + Start_Msg = io_lib:format("Dumping the full module dependencies graph... ", []), + send_log(State#analysis_state.parent, Start_Msg), + {T1, _} = statistics(wall_clock), + dump_mod_deps(ModDeps, State, Analysis, Extension), + {T2, _} = statistics(wall_clock), + Finish_Msg = io_lib:format("done in ~2f secs\n", [(T2-T1)/1000]), + send_log(State#analysis_state.parent, Finish_Msg), + ok. + +dump_mod_deps(ModDeps, _State, #analysis{mod_deps_file = File}, ".dot") -> + dialyzer_callgraph:mod_deps_to_dot(ModDeps, File); +dump_mod_deps(ModDeps, _State, #analysis{mod_deps_file = File}, ".ps") -> + Args = "-Gratio=compress -Gsize=\"100,100\"", + dialyzer_callgraph:mod_deps_to_ps(ModDeps, File, Args); +dump_mod_deps(_ModDeps, State, #analysis{mod_deps_file = File}, Ext) -> + Msg = io_lib:format("Could not write full modules dependencies file ~tp, Reason: Unrecognised file extension '~ts'. Only .dot and .ps are supported\n", + [File, Ext]), + send_log(State#analysis_state.parent, Msg). diff --git a/lib/dialyzer/src/dialyzer_behaviours.erl b/lib/dialyzer/src/dialyzer_behaviours.erl index d5c8ac0886..d3fbbcb2e1 100644 --- a/lib/dialyzer/src/dialyzer_behaviours.erl +++ b/lib/dialyzer/src/dialyzer_behaviours.erl @@ -126,20 +126,18 @@ check_callback(RetArgTypes, CbMFA, Behaviour, Callback, CbReturnType = dialyzer_contracts:get_contract_return(Callback), CbArgTypes = dialyzer_contracts:get_contract_args(Callback), {ReturnType, ArgTypes} = RetArgTypes, - Acc1 = case erl_types:t_is_subtype(ReturnType, CbReturnType) of - true -> - Acc0; - false -> - case erl_types:t_is_none(erl_types:t_inf(ReturnType, CbReturnType)) of - false -> - Acc0; - true -> - [{callback_type_mismatch, - [Behaviour, Function, Arity, - erl_types:t_to_string(ReturnType, Records), - erl_types:t_to_string(CbReturnType, Records)]}|Acc0] - end - end, + Acc1 = + % Allow none() as the return type to be backwards compatible + % with logic that allows crashes in callbacks + case (not erl_types:t_is_none(ReturnType)) andalso erl_types:t_is_none(erl_types:t_inf(ReturnType, CbReturnType)) of + false -> + Acc0; + true -> + [{callback_type_mismatch, + [Behaviour, Function, Arity, + erl_types:t_to_string(ReturnType, Records), + erl_types:t_to_string(CbReturnType, Records)]}|Acc0] + end, Acc2 = case erl_types:any_none(erl_types:t_inf_lists(ArgTypes, CbArgTypes)) of false -> Acc1; true -> @@ -156,10 +154,12 @@ check_callback(RetArgTypes, CbMFA, Behaviour, Callback, SpecArgTypes = [erl_types:subst_all_vars_to_any(ArgT0) || ArgT0 <- SpecArgTypes0], Acc3 = - case erl_types:t_is_subtype(SpecReturnType, CbReturnType) of - true -> - Acc2; + % Allow none() as the return type to be backwards compatible + % with logic that allows crashes in callbacks + case (not erl_types:t_is_none(SpecReturnType)) andalso erl_types:t_is_none(erl_types:t_inf(SpecReturnType, CbReturnType)) of false -> + Acc2; + true -> ExtraType = erl_types:t_subtract(SpecReturnType, CbReturnType), [{callback_spec_type_mismatch, [File, Location, Behaviour, Function, Arity, diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index aec4bd7169..a005360d64 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -37,16 +37,16 @@ modules/1, module_call_deps/1, merge_module_deps/2, - %% module_postorder/1, module_postorder_from_funs/2, new/0, get_depends_on/2, - %% get_required_by/2, in_neighbours/2, reset_from_funs/2, scan_core_tree/2, strip_module_deps/2, remove_external/1, + mod_deps_to_dot/2, + mod_deps_to_ps/3, to_dot/2, to_ps/3]). @@ -59,7 +59,6 @@ -type scc() :: [mfa_or_funlbl()]. -type mfa_call() :: {mfa_or_funlbl(), mfa_or_funlbl()}. -type mfa_calls() :: [mfa_call()]. --type mod_deps() :: dict:dict(module(), [module()]). %%----------------------------------------------------------------------------- %% A callgraph is a directed graph where the nodes are functions and a @@ -91,15 +90,16 @@ self_rec :: ets:tid(), calls :: ets:tid()}). + %% Exported Types -opaque callgraph() :: #callgraph{}. -type active_digraph() :: {'d', digraph:graph()} | {'e', - Out :: ets:tid(), - In :: ets:tid(), - Map :: ets:tid()}. + Out :: ets:tid()}. + +-type mod_deps() :: dict:dict(module(), [module()]). %%---------------------------------------------------------------------- @@ -229,8 +229,11 @@ find_non_local_calls([], Set) -> %% Only considers call dependencies, not type dependencies, which are dealt with elsewhere -spec get_depends_on(scc() | module(), callgraph()) -> [scc()]. -get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In, Maps}}) -> - lookup_scc(SCC, Out, Maps); +get_depends_on(SCC, #callgraph{active_digraph = {'e', Out}}) -> + case ets_lookup_dict(SCC, Out) of + error -> []; + {ok, Val} -> Val + end; get_depends_on(SCC, #callgraph{active_digraph = {'d', DG}}) -> digraph:out_neighbours(DG, SCC). @@ -241,17 +244,6 @@ get_depends_on(SCC, #callgraph{active_digraph = {'d', DG}}) -> %% get_required_by(SCC, #callgraph{active_digraph = {'d', DG}}) -> %% digraph:in_neighbours(DG, SCC). -lookup_scc(SCC, Table, Maps) -> - case ets_lookup_dict({'scc', SCC}, Maps) of - {ok, SCCInt} -> - case ets_lookup_dict(SCCInt, Table) of - {ok, Ints} -> - [ets:lookup_element(Maps, Int, 2) || Int <- Ints]; - error -> - [] - end; - error -> [] - end. %%---------------------------------------------------------------------- %% Handling of modules & SCCs @@ -322,17 +314,17 @@ strip_module_deps(ModDeps, StripSet) -> -spec finalize(callgraph()) -> {[scc()], callgraph()}. finalize(#callgraph{digraph = DG} = CG) -> - {ActiveDG, Postorder} = condensation(DG), - {Postorder, CG#callgraph{active_digraph = ActiveDG}}. + {ActiveDG, LabelledPostorder} = condensation(DG), + {LabelledPostorder, CG#callgraph{active_digraph = ActiveDG}}. -spec reset_from_funs([mfa_or_funlbl()], callgraph()) -> {[scc()], callgraph()}. reset_from_funs(Funs, #callgraph{digraph = DG, active_digraph = ADG} = CG) -> active_digraph_delete(ADG), SubGraph = digraph_reaching_subgraph(Funs, DG), - {NewActiveDG, Postorder} = condensation(SubGraph), + {NewActiveDG, LabelledPostorder} = condensation(SubGraph), digraph_delete(SubGraph), - {Postorder, CG#callgraph{active_digraph = NewActiveDG}}. + {LabelledPostorder, CG#callgraph{active_digraph = NewActiveDG}}. -spec module_postorder_from_funs([mfa_or_funlbl()], callgraph()) -> {[module()], callgraph()}. @@ -595,10 +587,8 @@ digraph_delete(DG) -> active_digraph_delete({'d', DG}) -> digraph:delete(DG); -active_digraph_delete({'e', Out, In, Maps}) -> - ets:delete(Out), - ets:delete(In), - ets:delete(Maps). +active_digraph_delete({'e', Out}) -> + ets:delete(Out). digraph_edges(DG) -> digraph:edges(DG). @@ -620,6 +610,27 @@ digraph_reaching_subgraph(Funs, DG) -> %% Utilities for 'dot' %%============================================================================= +-spec mod_deps_to_dot(mod_deps(), file:filename()) -> 'ok'. + +mod_deps_to_dot(ModDeps, File) -> + DepEdges = + lists:flatten( + [ + [{Mod, ModuleDependingOnIt} || ModuleDependingOnIt <- ModulesDependingOnIt] + || {Mod,ModulesDependingOnIt} <- dict:to_list(ModDeps) + ]), + dialyzer_dot:translate_list(DepEdges, File, "mod_deps"). + +-spec mod_deps_to_ps(mod_deps(), file:filename(), string()) -> 'ok'. + +mod_deps_to_ps(ModDeps, File, Args) -> + %% TODO: As with `to_dot/2`, handle Unicode names. + DotFile = filename:rootname(File) ++ ".dot", + mod_deps_to_dot(ModDeps, DotFile), + Command = io_lib:format("dot -Tps ~ts -o ~ts ~ts", [Args, File, DotFile]), + _ = os:cmd(Command), + ok. + -spec to_dot(callgraph(), file:filename()) -> 'ok'. to_dot(#callgraph{digraph = DG, esc = Esc} = CG, File) -> @@ -646,52 +657,32 @@ to_ps(#callgraph{} = CG, File, Args) -> ok. condensation(G) -> - {Pid, Ref} = erlang:spawn_monitor(do_condensation(G, self())), - receive {'DOWN', Ref, process, Pid, Result} -> - {SCCInts, OutETS, InETS, MapsETS} = Result, - NewSCCs = [ets:lookup_element(MapsETS, SCCInt, 2) || SCCInt <- SCCInts], - {{'e', OutETS, InETS, MapsETS}, NewSCCs} - end. - --spec do_condensation(digraph:graph(), pid()) -> fun(() -> no_return()). - -do_condensation(G, Parent) -> - fun() -> - [OutETS, InETS, MapsETS] = - [ets:new(Name,[{read_concurrency, true}]) || - Name <- [callgraph_deps_out, callgraph_deps_in, callgraph_scc_map]], - SCCs = digraph_utils:strong_components(G), - %% Assign unique numbers to SCCs: - Ints = lists:seq(1, length(SCCs)), - IntToSCC = lists:zip(Ints, SCCs), - IntScc = sofs:relation(IntToSCC, [{int, scc}]), - %% Create mapping from unique integers to SCCs: - ets:insert(MapsETS, IntToSCC), - %% Substitute strong components for vertices in edges using the - %% unique numbers: - C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]), - I2V = sofs:relative_product(IntScc, C2V), % [{v, int}] - Es = sofs:relation(digraph:edges(G), [{v, v}]), - R1 = sofs:relative_product(I2V, Es), - R2 = sofs:relative_product(I2V, sofs:converse(R1)), - R2Strict = sofs:strict_relation(R2), - %% Create out-neighbours: - Out = sofs:relation_to_family(sofs:converse(R2Strict)), - ets:insert(OutETS, sofs:to_external(Out)), - %% Sort the SCCs topologically: - DG = sofs:family_to_digraph(Out), - lists:foreach(fun(I) -> digraph:add_vertex(DG, I) end, Ints), - SCCInts0 = digraph_utils:topsort(DG), - digraph:delete(DG), - %% The out-neighbors of a vertex are the vertices called directly. - %% The used vertices are to occur *before* the calling vertex: - SCCInts = lists:reverse(SCCInts0), - %% Create in-neighbours: - In = sofs:relation_to_family(R2Strict), - ets:insert(InETS, sofs:to_external(In)), - %% Create mapping from SCCs to unique integers: - ets:insert(MapsETS, lists:zip([{'scc', SCC} || SCC<- SCCs], Ints)), - lists:foreach(fun(E) -> true = ets:give_away(E, Parent, any) - end, [OutETS, InETS, MapsETS]), - exit({SCCInts, OutETS, InETS, MapsETS}) - end. + OutETS = ets:new(callgraph_label_deps_out,[{read_concurrency, true}]), + SCCs = digraph_utils:strong_components(G), + %% Assign unique numbers to SCCs: + Ints = lists:seq(1, length(SCCs)), + IntToSCC = lists:zip(Ints, SCCs), + IntScc = sofs:relation(IntToSCC, [{int, scc}]), + + %% Subsitute strong components for vertices in edges using the + %% unique numbers: + C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]), + I2V = sofs:relative_product(IntScc, C2V), % [{v, int}] + Es = sofs:relation(digraph:edges(G), [{v, v}]), + R1 = sofs:relative_product(I2V, Es), + R2 = sofs:relative_product(I2V, sofs:converse(R1)), + R2Strict = sofs:strict_relation(R2), + %% Create out-neighbours: + Out = sofs:relation_to_family(sofs:converse(R2Strict)), + ets:insert(OutETS, sofs:to_external(Out)), + %% Sort the SCCs topologically: + DG = sofs:family_to_digraph(Out), + lists:foreach(fun(I) -> digraph:add_vertex(DG, I) end, Ints), + SCCInts0 = digraph_utils:topsort(DG), + digraph:delete(DG), + %% The out-neighbors of a vertex are the vertices called directly. + %% The used vertices are to occur *before* the calling vertex: + SCCInts = lists:reverse(SCCInts0), + IntToSCCMap = maps:from_list(IntToSCC), + LabelledPostorder = [{I, maps:get(I, IntToSCCMap)} || I <- SCCInts], + {{'e', OutETS}, LabelledPostorder}. diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 25fdef948e..86b29e027a 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -23,7 +23,9 @@ -module(dialyzer_cl). --export([start/1]). +-export([start/1, + start_report_modules_analyzed/1, + start_report_modules_changed_and_analyzed/1]). -include("dialyzer.hrl"). -include_lib("kernel/include/file.hrl"). % needed for #file_info{} @@ -33,12 +35,8 @@ code_server = none :: 'none' | dialyzer_codeserver:codeserver(), erlang_mode = false :: boolean(), - external_calls = [] :: [{mfa(), - {file:filename(), - erl_anno:location()}}], - external_types = [] :: [{mfa(), - {file:filename(), - erl_anno:location()}}], + external_calls = [] :: [{mfa(), warning_info()}], + external_types = [] :: [{mfa(), warning_info()}], legal_warnings = ordsets:new() :: [dial_warn_tag()], mod_deps = dict:new() :: dialyzer_callgraph:mod_deps(), output = standard_io :: io:device(), @@ -57,14 +55,26 @@ -spec start(#options{}) -> {dial_ret(), [dial_warning()]}. -start(#options{analysis_type = AnalysisType} = Options) -> +start(Opts) -> + {{Ret,Warns}, _ModulesAnalyzed} = start_report_modules_analyzed(Opts), + {Ret,Warns}. + +-spec start_report_modules_analyzed(#options{}) -> {{dial_ret(), [dial_warning()]}, [module()]}. + +start_report_modules_analyzed(Opts) -> + {{Ret,Warns}, _ModulesChanged, ModulesAnalyzed} = start_report_modules_changed_and_analyzed(Opts), + {{Ret,Warns}, ModulesAnalyzed}. + +-spec start_report_modules_changed_and_analyzed(#options{}) -> {{dial_ret(), [dial_warning()]}, undefined | [module()], [module()]}. + +start_report_modules_changed_and_analyzed(#options{analysis_type = AnalysisType} = Options) -> process_flag(trap_exit, true), case AnalysisType of plt_check -> check_plt(Options); plt_build -> build_plt(Options); plt_add -> add_to_plt(Options); plt_remove -> remove_from_plt(Options); - succ_typings -> do_analysis(Options) + succ_typings -> enrich_with_modules_changed(do_analysis(Options), undefined) end. %%-------------------------------------------------------------------- @@ -72,9 +82,9 @@ start(#options{analysis_type = AnalysisType} = Options) -> build_plt(Opts) -> Opts1 = init_opts_for_build(Opts), Files = get_files_from_opts(Opts1), - Md5 = dialyzer_plt:compute_md5_from_files(Files), - PltInfo = #plt_info{files = Md5, mod_deps = dict:new()}, - do_analysis(Files, Opts1, dialyzer_plt:new(), PltInfo). + Md5 = dialyzer_cplt:compute_md5_from_files(Files), + PltInfo = #plt_info{files = Md5}, + enrich_with_modules_changed(do_analysis(Files, Opts1, dialyzer_plt:new(), PltInfo), undefined). init_opts_for_build(Opts) -> case Opts#options.output_plt =:= none of @@ -116,8 +126,6 @@ init_opts_for_add(Opts) -> end end. -%%-------------------------------------------------------------------- - check_plt(#options{init_plts = []} = Opts) -> Opts1 = init_opts_for_check(Opts), report_check(Opts1), @@ -132,10 +140,12 @@ check_plt_aux([_] = Plt, Opts) -> plt_common(Opts2, [], []); check_plt_aux([Plt|Plts], Opts) -> case check_plt_aux([Plt], Opts) of - {?RET_NOTHING_SUSPICIOUS, []} -> check_plt_aux(Plts, Opts); - {?RET_DISCREPANCIES, Warns} -> - {_RET, MoreWarns} = check_plt_aux(Plts, Opts), - {?RET_DISCREPANCIES, Warns ++ MoreWarns} + {{?RET_NOTHING_SUSPICIOUS, []}, ModulesChanged, ModulesAnalyzed} -> + {{Ret, Warns}, MoreModulesChanged, MoreModulesAnalyzed} = check_plt_aux(Plts, Opts), + {{Ret, Warns}, ordsets:union(ModulesChanged, MoreModulesChanged), ordsets:union(ModulesAnalyzed, MoreModulesAnalyzed)}; + {{?RET_DISCREPANCIES, Warns}, ModulesChanged, ModulesAnalyzed} -> + {{_RET, MoreWarns}, MoreModulesChanged, MoreModulesAnalyzed} = check_plt_aux(Plts, Opts), + {{?RET_DISCREPANCIES, Warns ++ MoreWarns}, ordsets:union(ModulesChanged, MoreModulesChanged), ordsets:union(ModulesAnalyzed, MoreModulesAnalyzed)} end. init_opts_for_check(Opts) -> @@ -145,6 +155,7 @@ init_opts_for_check(Opts) -> Plt -> Plt end, [OutputPlt] = InitPlt, + Opts#options{files = [], files_rec = [], analysis_type = plt_check, @@ -198,24 +209,24 @@ plt_common(#options{init_plts = [InitPlt]} = Opts, RemoveFiles, AddFiles) -> quiet -> ok; _ -> io:put_chars(" yes\n") end, - {?RET_NOTHING_SUSPICIOUS, []}; + {{?RET_NOTHING_SUSPICIOUS, []}, [], []}; {old_version, Md5} -> - PltInfo = #plt_info{files = Md5, mod_deps = dict:new()}, + PltInfo = #plt_info{files = Md5}, Files = [F || {F, _} <- Md5], - do_analysis(Files, Opts, dialyzer_plt:new(), PltInfo); + enrich_with_modules_changed(do_analysis(Files, Opts, dialyzer_plt:new(), PltInfo), undefined); {differ, Md5, DiffMd5, ModDeps} -> report_failed_plt_check(Opts, DiffMd5), - {AnalFiles, RemovedMods, ModDeps1} = + {AnalFiles, RemovedMods, ModDeps1} = expand_dependent_modules(Md5, DiffMd5, ModDeps), Plt = clean_plt(InitPlt, RemovedMods), + ChangedOrRemovedMods = [ChangedOrRemovedMod || {_, ChangedOrRemovedMod} <- DiffMd5], case AnalFiles =:= [] of true -> %% Only removed stuff. Just write the PLT. - dialyzer_plt:to_file(Opts#options.output_plt, Plt, ModDeps, - #plt_info{files = Md5, mod_deps = ModDeps}), - {?RET_NOTHING_SUSPICIOUS, []}; + dialyzer_cplt:to_file(Opts#options.output_plt, Plt, ModDeps1, #plt_info{files=Md5, mod_deps=ModDeps1}), + {{?RET_NOTHING_SUSPICIOUS, []}, ChangedOrRemovedMods, []}; false -> - do_analysis(AnalFiles, Opts, Plt, #plt_info{files = Md5, mod_deps = ModDeps1}) + enrich_with_modules_changed(do_analysis(AnalFiles, Opts, Plt, #plt_info{files=Md5, mod_deps=ModDeps1}), ChangedOrRemovedMods) end; {error, no_such_file} -> Msg = io_lib:format("Could not find the PLT: ~ts\n~s", @@ -235,6 +246,12 @@ plt_common(#options{init_plts = [InitPlt]} = Opts, RemoveFiles, AddFiles) -> cl_error(Msg) end. +-spec enrich_with_modules_changed({{Ret :: dial_ret(), Warns :: [dial_warning()]}, Analyzed :: [module()]}, Changed :: undefined | [module()]) -> + {{dial_ret(), [dial_warning()]}, Changed :: undefined | [module()], Analyzed :: [module()]}. + +enrich_with_modules_changed({{Ret,Warns}, Analyzed}, Changed) -> + {{Ret,Warns}, Changed, Analyzed}. + default_plt_error_msg() -> "Use the options:\n" " --build_plt to build a new PLT; or\n" @@ -252,7 +269,7 @@ default_plt_error_msg() -> %%-------------------------------------------------------------------- check_plt(#options{init_plts = [Plt]} = Opts, RemoveFiles, AddFiles) -> - case dialyzer_plt:check_plt(Plt, RemoveFiles, AddFiles) of + case dialyzer_cplt:check_plt(Plt, RemoveFiles, AddFiles) of {old_version, _MD5} = OldVersion -> report_old_version(Opts), OldVersion; @@ -282,7 +299,7 @@ report_old_version(#options{report_mode = ReportMode, init_plts = [InitPlt]}) -> [InitPlt]) end. -report_failed_plt_check(#options{analysis_type = AnalType, +report_failed_plt_check(#options{analysis_type = AnalType, report_mode = ReportMode}, DiffMd5) -> case AnalType =:= plt_check of true -> @@ -349,10 +366,10 @@ report_md5_diff(List) -> %%-------------------------------------------------------------------- get_default_init_plt() -> - [dialyzer_plt:get_default_plt()]. + [dialyzer_cplt:get_default_cplt_filename()]. get_default_output_plt() -> - dialyzer_plt:get_default_plt(). + dialyzer_cplt:get_default_cplt_filename(). %%-------------------------------------------------------------------- @@ -367,11 +384,11 @@ do_analysis(Options) -> case Options#options.init_plts of [] -> do_analysis(Files, Options, dialyzer_plt:new(), none); PltFiles -> - Plts = [dialyzer_plt:from_file(F) || F <- PltFiles], - Plt = dialyzer_plt:merge_plts_or_report_conflicts(PltFiles, Plts), + Plts = [dialyzer_cplt:from_file(F) || F <- PltFiles], + Plt = dialyzer_cplt:merge_plts_or_report_conflicts(PltFiles, Plts), do_analysis(Files, Options, Plt, none) end. - + do_analysis(Files, Options, Plt, PltInfo) -> assert_writable(Options#options.output_plt), report_analysis_start(Options), @@ -381,7 +398,8 @@ do_analysis(Files, Options, Plt, PltInfo) -> output_plt = Options#options.output_plt, plt_info = PltInfo, erlang_mode = Options#options.erlang_mode, - report_mode = Options#options.report_mode}, + report_mode = Options#options.report_mode + }, AnalysisType = convert_analysis_type(Options#options.analysis_type, Options#options.get_warnings), InitAnalysis = #analysis{type = AnalysisType, @@ -393,13 +411,14 @@ do_analysis(Files, Options, Plt, PltInfo) -> plt = Plt, use_contracts = Options#options.use_contracts, callgraph_file = Options#options.callgraph_file, + mod_deps_file = Options#options.mod_deps_file, solvers = Options#options.solvers}, State3 = start_analysis(State2, InitAnalysis), {T1, _} = statistics(wall_clock), - Return = cl_loop(State3), + RetAndWarns = cl_loop(State3), {T2, _} = statistics(wall_clock), report_elapsed_time(T1, T2, Options), - Return. + {RetAndWarns, lists:usort([path_to_mod(F) || F <- Files])}. convert_analysis_type(plt_check, true) -> succ_typings; convert_analysis_type(plt_check, false) -> plt_build; @@ -428,14 +447,13 @@ check_if_writable(PltFile) -> true -> is_writable_file_or_dir(PltFile); false -> case filelib:is_dir(PltFile) of - true -> false; - false -> - DirName = filename:dirname(PltFile), - case filelib:is_dir(DirName) of + true -> false; + false -> + DirName = filename:dirname(PltFile), + case filelib:is_dir(DirName) of false -> case filelib:ensure_dir(PltFile) of ok -> - io:format(" Creating ~ts as it did not exist...~n", [DirName]), true; {error, _} -> false @@ -458,7 +476,7 @@ is_writable_file_or_dir(PltFile) -> clean_plt(PltFile, RemovedMods) -> %% Clean the plt from the removed modules. - Plt = dialyzer_plt:from_file(PltFile), + Plt = dialyzer_cplt:from_file(PltFile), sets:fold(fun(M, AccPlt) -> dialyzer_plt:delete_module(AccPlt, M) end, Plt, RemovedMods). @@ -469,9 +487,9 @@ expand_dependent_modules(Md5, DiffMd5, ModDeps) -> BigList = sets:to_list(BigSet), ExpandedSet = expand_dependent_modules_1(BigList, BigSet, ModDeps), NewModDeps = dialyzer_callgraph:strip_module_deps(ModDeps, BigSet), - AnalyzeMods = sets:subtract(ExpandedSet, RemovedMods), + AnalyzeMods = sets:subtract(ExpandedSet, RemovedMods), FilterFun = fun(File) -> - Mod = list_to_atom(filename:basename(File, ".beam")), + Mod = path_to_mod(File), sets:is_element(Mod, AnalyzeMods) end, {[F || {F, _} <- Md5, FilterFun(F)], BigSet, NewModDeps}. @@ -479,12 +497,12 @@ expand_dependent_modules(Md5, DiffMd5, ModDeps) -> expand_dependent_modules_1([Mod|Mods], Included, ModDeps) -> case dict:find(Mod, ModDeps) of {ok, Deps} -> - NewDeps = sets:subtract(sets:from_list(Deps), Included), + NewDeps = sets:subtract(sets:from_list(Deps), Included), case sets:size(NewDeps) =:= 0 of true -> expand_dependent_modules_1(Mods, Included, ModDeps); - false -> + false -> NewIncluded = sets:union(Included, NewDeps), - expand_dependent_modules_1(sets:to_list(NewDeps) ++ Mods, + expand_dependent_modules_1(sets:to_list(NewDeps) ++ Mods, NewIncluded, ModDeps) end; error -> @@ -493,6 +511,9 @@ expand_dependent_modules_1([Mod|Mods], Included, ModDeps) -> expand_dependent_modules_1([], Included, _ModDeps) -> Included. +path_to_mod(File) -> + list_to_atom(filename:basename(File, ".beam")). + new_state() -> #cl_state{}. @@ -536,7 +557,7 @@ maybe_close_output_file(State) -> -define(LOG_CACHE_SIZE, 10). -%%-spec cl_loop(#cl_state{}) -> +%%-spec cl_loop(#cl_state{}) -> cl_loop(State) -> cl_loop(State, []). @@ -628,7 +649,7 @@ return_value(State = #cl_state{code_server = CodeServer, true -> dialyzer_plt:delete(Plt); false -> - dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo) + dialyzer_cplt:to_file(OutputPlt, Plt, ModDeps, PltInfo) end, UnknownWarnings = unknown_warnings(State), RetValue = @@ -643,13 +664,15 @@ return_value(State = #cl_state{code_server = CodeServer, print_ext_types(State), maybe_close_output_file(State), {RetValue, []}; - true -> - AllWarnings = - UnknownWarnings ++ process_warnings(StoredWarnings), - {RetValue, set_warning_id(AllWarnings, EOpt)} + true -> + ResultingWarnings = process_warnings(StoredWarnings ++ UnknownWarnings), + {RetValue, set_warning_id(ResultingWarnings, EOpt)} end. -unknown_warnings(State = #cl_state{legal_warnings = LegalWarnings}) -> +unknown_warnings(State) -> + [Warning || {_M, Warning} <- unknown_warnings_by_module(State)]. + +unknown_warnings_by_module(#cl_state{legal_warnings = LegalWarnings} = State) -> case ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) of true -> lists:sort(unknown_functions(State)) ++ @@ -658,8 +681,11 @@ unknown_warnings(State = #cl_state{legal_warnings = LegalWarnings}) -> end. unknown_functions(#cl_state{external_calls = Calls}) -> - [{?WARN_UNKNOWN, {File, Location, ''},{unknown_function, MFA}} || - {MFA, {File, Location}} <- Calls]. + [{Mod, {?WARN_UNKNOWN, WarningInfo, {unknown_function, MFA}}} || {MFA, WarningInfo = {_, _, {Mod, _, _}}} <- Calls]. + +unknown_types(#cl_state{external_types = Types}) -> + [{Mod, {?WARN_UNKNOWN, WarningInfo, {unknown_type, MFA}}} || + {MFA, WarningInfo = {_, _, {Mod, _, _}}} <- Types]. set_warning_id(Warnings, EOpt) -> lists:map(fun({Tag, {File, Location, _MorMFA}, Msg}) -> @@ -695,17 +721,13 @@ print_ext_calls(#cl_state{output = Output, end end. -do_print_ext_calls(Output, [{{M,F,A},{File,Location}}|T], Before) -> +do_print_ext_calls(Output, [{{M,F,A},{File,Location,_FromMFA}}|T], Before) -> io:format(Output, "~s~tp:~tp/~p (~ts)\n", [Before,M,F,A,file_pos(File, Location)]), do_print_ext_calls(Output, T, Before); do_print_ext_calls(_, [], _) -> ok. -unknown_types(#cl_state{external_types = Types}) -> - [{?WARN_UNKNOWN, {File, Location, ''},{unknown_type, MFA}} || - {MFA, {File, Location}} <- Types]. - print_ext_types(#cl_state{report_mode = quiet}) -> ok; print_ext_types(#cl_state{output = Output, @@ -731,7 +753,7 @@ print_ext_types(#cl_state{output = Output, end end. -do_print_ext_types(Output, [{{M,F,A},{File,Location}}|T], Before) -> +do_print_ext_types(Output, [{{M,F,A},{File,Location,_}}|T], Before) -> io:format(Output, "~s~tp:~tp/~p (~ts)\n", [Before,M,F,A,file_pos(File, Location)]), do_print_ext_types(Output, T, Before); @@ -752,8 +774,8 @@ pos(Line) -> %% Keep one warning per MFA and File. Often too many warnings otherwise. %% limit_unknown(Unknowns) -> - L = [{{MFA, File}, Line} || {MFA, {File, Line}} <- Unknowns], - [{MFA, {File, Line}} || {{MFA, File}, [Line|_]} <- + L = [{{MFA, File}, {FromMFA, Line}} || {MFA, {File, Line, FromMFA}} <- Unknowns], + [{MFA, {File, Line, FromMFA}} || {{MFA, File}, [{FromMFA, Line}|_]} <- dialyzer_utils:family(L)]. %% Keep one warning per MFA. This is how it used to be before Erlang/OTP 24. %% limit_unknown(Unknowns) -> @@ -787,10 +809,11 @@ print_warnings(#cl_state{output = Output, end. -spec process_warnings([raw_warning()]) -> [raw_warning()]. - + process_warnings(Warnings) -> - Warnings1 = lists:keysort(2, Warnings), %% Sort on file/location (and m/mfa..) - remove_duplicate_warnings(Warnings1, []). + Warnings1 = lists:keysort(3, Warnings), %% First sort on Warning + Warnings2 = lists:keysort(2, Warnings1), %% Sort on file/location (and m/mfa..) + remove_duplicate_warnings(Warnings2, []). remove_duplicate_warnings([Duplicate, Duplicate|Left], Acc) -> remove_duplicate_warnings([Duplicate|Left], Acc); @@ -813,7 +836,7 @@ add_files(Files, From) -> add_files(Files, From, Rec) -> Files1 = [filename:absname(F) || F <- Files], - Files2 = ordsets:from_list(Files1), + Files2 = ordsets:from_list(Files1), Dirs = ordsets:filter(fun(X) -> filelib:is_dir(X) end, Files2), Files3 = ordsets:subtract(Files2, Dirs), Extension = case From of @@ -840,9 +863,8 @@ add_file_fun(Extension) -> start_analysis(State, Analysis) -> Self = self(), LegalWarnings = State#cl_state.legal_warnings, - Fun = fun() -> + Fun = fun() -> dialyzer_analysis_callgraph:start(Self, LegalWarnings, Analysis) end, BackendPid = spawn_link(Fun), State#cl_state{backend_pid = BackendPid}. - diff --git a/lib/dialyzer/src/dialyzer_cl_parse.erl b/lib/dialyzer/src/dialyzer_cl_parse.erl index e59f4027ac..a0dcf8dd0f 100644 --- a/lib/dialyzer/src/dialyzer_cl_parse.erl +++ b/lib/dialyzer/src/dialyzer_cl_parse.erl @@ -54,6 +54,11 @@ cl(["--apps"|T]) -> {Args, T2} = collect_args(T1), append_var(dialyzer_options_files_rec, Args), cl(T2); +cl(["--warning_apps"|T]) -> + T1 = get_lib_dir(T), + {Args, T2} = collect_args(T1), + append_var(dialyzer_options_warning_files_rec, Args), + cl(T2); cl(["--build_plt"|T]) -> put(dialyzer_options_analysis_type, plt_build), cl(T); @@ -113,6 +118,9 @@ cl(["-r"++_|T0]) -> cl(["--remove_from_plt"|T]) -> put(dialyzer_options_analysis_type, plt_remove), cl(T); +cl(["--incremental"|T]) -> + put(dialyzer_options_analysis_type, incremental), + cl(T); cl(["--com"++_|T]) -> NewTail = command_line(T), cl(NewTail); @@ -123,6 +131,12 @@ cl(["-o"]) -> cl(["--output",Output|T]) -> put(dialyzer_output, Output), cl(T); +cl(["--metrics_file",MetricsFile|T]) -> + put(dialyzer_metrics, MetricsFile), + cl(T); +cl(["--module_lookup_file",ModuleLookupFile|T]) -> + put(dialyzer_module_lookup, ModuleLookupFile), + cl(T); cl(["--output_plt"]) -> cl_error("No outfile specified for --output_plt"); cl(["--output_plt",Output|T]) -> @@ -199,6 +213,11 @@ cl(["--dump_callgraph"]) -> cl(["--dump_callgraph", File|T]) -> put(dialyzer_callgraph_file, File), cl(T); +cl(["--dump_full_dependencies_graph"]) -> + cl_error("No outfile specified for --dump_full_dependencies_graph"); +cl(["--dump_full_dependencies_graph", File|T]) -> + put(dialyzer_mod_deps_file, File), + cl(T); cl(["--gui"|T]) -> put(dialyzer_options_mode, gui), cl(T); @@ -271,6 +290,9 @@ init() -> %% common_options(), then the environment variables (currently only %% ERL_COMPILER_OPTIONS) would be overwritten by default values. put(dialyzer_options_mode, cl), + put(dialyzer_options_files_rec, []), + put(dialyzer_options_warning_files_rec, []), + put(dialyzer_options_report_mode, normal), put(dialyzer_warnings, []), ok. @@ -311,17 +333,20 @@ collect_args_1([], Acc) -> cl_options() -> OptsList = [{files, dialyzer_options_files}, - {files_rec, dialyzer_options_files_rec}, - {output_file, dialyzer_output}, - {output_format, dialyzer_output_format}, - {filename_opt, dialyzer_filename_opt}, - {indent_opt, dialyzer_indent_opt}, - {error_location, dialyzer_error_location_opt}, - {analysis_type, dialyzer_options_analysis_type}, - {get_warnings, dialyzer_options_get_warnings}, - {timing, dialyzer_timing}, - {callgraph_file, dialyzer_callgraph_file}], - get_options(OptsList) ++ common_options(). + {files_rec, dialyzer_options_files_rec}, + {warning_files_rec, dialyzer_options_warning_files_rec}, + {output_file, dialyzer_output}, + {metrics_file, dialyzer_metrics}, + {module_lookup_file, dialyzer_module_lookup}, + {output_format, dialyzer_output_format}, + {filename_opt, dialyzer_filename_opt}, + {indent_opt, dialyzer_indent_opt}, + {analysis_type, dialyzer_options_analysis_type}, + {get_warnings, dialyzer_options_get_warnings}, + {timing, dialyzer_timing}, + {callgraph_file, dialyzer_callgraph_file}, + {mod_deps_file, dialyzer_mod_deps_file}], + get_options(OptsList) ++ common_options(). common_options() -> OptsList = [{defines, dialyzer_options_defines}, @@ -395,7 +420,7 @@ help_message() -> [--check_plt] [-Ddefine]* [-Dname]* [--dump_callgraph file] [--error_location flag] [files_or_dirs] [--fullpath] [--get_warnings] [--gui] [--help] [-I include_dir]* - [--no_check_plt] [--no_indentation] [-o outfile] + [--incremental] [--no_check_plt] [--no_indentation] [-o outfile] [--output_plt file] [-pa dir]* [--plt plt] [--plt_info] [--plts plt*] [--quiet] [-r dirs] [--raw] [--remove_from_plt] [--shell] [--src] [--statistics] [--verbose] [--version] @@ -442,7 +467,7 @@ Options: --output_plt file Store the plt at the specified file after building it. --plt plt - Use the specified plt as the initial plt (if the plt was built + Use the specified plt as the initial plt (if the plt was built during setup the files will be checked for consistency). --plts plt* Merge the specified plts to create the initial plt -- requires @@ -493,6 +518,13 @@ Options: --no_check_plt (or -n) Skip the plt check when running Dialyzer. Useful when working with installed plts that never change. + --incremental + The analysis starts from an existing incremental PLT, or builds one from + scratch if one doesn't exist, and runs the minimal amount of additional + analysis to report all issues in the given set of apps. Notably, incremental + PLT files are not compatible with \"classic\" PLT files, and vice versa. + The initial incremental PLT will be updated unless an alternative output + incremental PLT is given. --plt_info Make Dialyzer print information about the plt and then quit. The plt can be specified with --plt(s). @@ -504,6 +536,11 @@ Options: by the file name extension. Supported extensions are: raw, dot, and ps. If something else is used as file name extension, default format '.raw' will be used. + --dump_full_dependencies_graph file + Dump the full dependency graph (i.e. dependencies induced by function + calls, usages of types in specs, behaviour implementations, etc.) into + the specified file whose format is determined by the file name + extension. Supported extensions are: dot and ps. --error_location column | line Use a pair {Line, Column} or an integer Line to pinpoint the location of warnings. The default is to use a pair {Line, Column}. When diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 713e475b17..043a99560f 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -185,7 +185,7 @@ process_contract_remote_types_module(ModuleName, CodeServer) -> RecordTable = dialyzer_codeserver:get_records_table(CodeServer), ExpTypes = dialyzer_codeserver:get_exported_types_table(CodeServer), ContractFun = - fun({MFA, {File, TmpContract, Xtra}}, C0) -> + fun({{_,_,_} = MFA, {File, TmpContract, Xtra}}, C0) -> #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) -> CFun(ExpTypes, RecordTable, C1) @@ -269,6 +269,7 @@ check_contracts(Contracts, Callgraph, FunTypes, ModOpaques) -> 'ok' | {'error', 'invalid_contract' + | {'invalid_contract', {InvalidArgIdxs :: [pos_integer()], IsReturnTypeInvalid :: boolean()}} | {'opaque_mismatch', erl_types:erl_type()} | {'overlapping_contract', [module() | atom() | byte()]} | string()} @@ -299,12 +300,11 @@ check_contract(#contract{contracts = Contracts}, SuccType, Opaques) -> ok -> InfList = [{Contract, erl_types:t_inf(Contract, SuccType, Opaques)} || Contract <- Contracts2], - case check_contract_inf_list(InfList, SuccType, Opaques) of - {error, _} = Invalid -> Invalid; + case check_contract_inf_list(InfList, SuccType, Opaques) of + {error, _} = Invalid -> Invalid; ok -> case check_extraneous(Contracts2, SuccType, Opaques) of - {error, invalid_contract} = Err -> - Err; + {error, {invalid_contract, _}} = Err -> Err; {error, {extra_range, _, _}} = Err -> MissingError = check_missing(Contracts2, SuccType, Opaques), {range_warnings, [Err | MissingError]}; @@ -320,6 +320,25 @@ check_contract(#contract{contracts = Contracts}, SuccType, Opaques) -> throw:{error, _} = Error -> Error end. +locate_invalid_elems(InfList) -> + case InfList of + [{Contract, Inf}] -> + ArgComparisons = lists:zip(erl_types:t_fun_args(Contract), + erl_types:t_fun_args(Inf)), + ProblematicArgs = + [erl_types:t_is_none(Succ) andalso (not erl_types:t_is_none(Cont)) + || {Cont,Succ} <- ArgComparisons], + ProblematicRange = + erl_types:t_is_none(erl_types:t_fun_range(Inf)) + andalso (not erl_types:t_is_none(erl_types:t_fun_range(Contract))), + ProblematicArgIdxs = [Idx || + {Idx, IsProblematic} <- + lists:enumerate(ProblematicArgs), IsProblematic], + {error, {invalid_contract, {ProblematicArgIdxs, ProblematicRange}}}; + _ -> + {error, invalid_contract} + end. + check_domains([_]) -> ok; check_domains([Dom|Doms]) -> Fun = fun(D) -> @@ -330,16 +349,19 @@ check_domains([Dom|Doms]) -> false -> error end. + %% Allow a contract if one of the overloaded contracts is possible. %% We used to be more strict, e.g., all overloaded contracts had to be %% possible. check_contract_inf_list(List, SuccType, Opaques) -> case check_contract_inf_list(List, SuccType, Opaques, []) of ok -> ok; - {error, []} -> {error, invalid_contract}; + {error, []} -> + locate_invalid_elems(List); {error, [{SigRange, ContrRange}|_]} -> case erl_types:t_find_opaque_mismatch(SigRange, ContrRange, Opaques) of - error -> {error, invalid_contract}; + error -> + locate_invalid_elems(List); {ok, _T1, T2} -> {error, {opaque_mismatch, T2}} end end. @@ -383,13 +405,12 @@ check_extraneous_1(Contract, SuccType, Opaques) -> case [CR || CR <- CRngs, erl_types:t_is_none(erl_types:t_inf(CR, STRng, Opaques))] of [] -> - case bad_extraneous_list(CRng, STRng) - orelse bad_extraneous_map(CRng, STRng) - of - true -> {error, invalid_contract}; - false -> ok + case bad_extraneous_list(CRng, STRng) orelse bad_extraneous_map(CRng, STRng) of + true -> {error, {invalid_contract, {[],true}}}; + false -> ok end; - CRs -> {error, {extra_range, erl_types:t_sup(CRs), STRng}} + CRs -> + {error, {extra_range, erl_types:t_sup(CRs), STRng}} end. bad_extraneous_list(CRng, STRng) -> @@ -819,7 +840,9 @@ get_invalid_contract_warnings_funs([{MFA, {FileLocation, Contract, _Xtra}}|Left] NewAcc = case check_contract(Contract, Sig, Opaques) of {error, invalid_contract} -> - [invalid_contract_warning(MFA, WarningInfo, Sig, RecDict)|Acc]; + [invalid_contract_warning(MFA, WarningInfo, none, Contract, Sig, RecDict)|Acc]; + {error, {invalid_contract, {_ProblematicArgIdxs, _IsRangeProblematic} = ProblemDetails}} -> + [invalid_contract_warning(MFA, WarningInfo, ProblemDetails, Contract, Sig, RecDict)|Acc]; {error, {opaque_mismatch, T2}} -> W = contract_opaque_warning(MFA, WarningInfo, T2, Sig, RecDict), [W|Acc]; @@ -864,7 +887,7 @@ get_invalid_contract_warnings_funs([{MFA, {FileLocation, Contract, _Xtra}}|Left] BifSig = erl_types:t_fun(BifArgs, BifRet), case check_contract(Contract, BifSig, Opaques) of {error, _} -> - [invalid_contract_warning(MFA, WarningInfo, BifSig, RecDict) + [invalid_contract_warning(MFA, WarningInfo, none, Contract, BifSig, RecDict) |Acc]; {range_warnings, _} -> picky_contract_check(CSig, BifSig, MFA, WarningInfo, @@ -883,9 +906,10 @@ get_invalid_contract_warnings_funs([{MFA, {FileLocation, Contract, _Xtra}}|Left] get_invalid_contract_warnings_funs([], _Plt, _RecDict, _Opaques, Acc) -> Acc. -invalid_contract_warning({M, F, A}, WarningInfo, SuccType, RecDict) -> - SuccTypeStr = dialyzer_utils:format_sig(SuccType, RecDict), - {?WARN_CONTRACT_TYPES, WarningInfo, {invalid_contract, [M, F, A, SuccTypeStr]}}. +invalid_contract_warning({M, F, A}, WarningInfo, ProblemDetails, Contract, SuccType, RecDict) -> + SuccTypeStr = lists:flatten(dialyzer_utils:format_sig(SuccType, RecDict)), + ContractTypeStr = contract_to_string(Contract), + {?WARN_CONTRACT_TYPES, WarningInfo, {invalid_contract, [M, F, A, ProblemDetails, ContractTypeStr, SuccTypeStr]}}. contract_opaque_warning({M, F, A}, WarningInfo, OpType, SuccType, RecDict) -> OpaqueStr = erl_types:t_to_string(OpType), diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl index 2ca3acc4cb..44450de048 100644 --- a/lib/dialyzer/src/dialyzer_coordinator.erl +++ b/lib/dialyzer/src/dialyzer_coordinator.erl @@ -1,4 +1,3 @@ -%% -*- erlang-indent-level: 2 -*- %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,6 +26,9 @@ %%% Exports for the typesig and dataflow analysis workers -export([wait_for_success_typings/2]). +%% Exports to handle SCC labels +-export([get_job_label/2, get_job_input/2]). + %%% Exports for the compilation workers -export([get_next_label/2]). @@ -36,9 +38,9 @@ -type collector() :: pid(). -type regulator() :: pid(). --type scc_to_pid() :: ets:tid() | 'none'. +-type job_labels_to_pid() :: ets:tid() | 'none'. --opaque coordinator() :: {collector(), regulator(), scc_to_pid()}. +-opaque coordinator() :: {collector(), regulator(), job_labels_to_pid()}. -type timing() :: dialyzer_timing:timing_server(). -type scc() :: [mfa_or_funlbl()]. @@ -46,7 +48,7 @@ 'contract_remote_types' | 'record_remote_types'. -type compile_job() :: file:filename(). --type typesig_job() :: scc(). +-type typesig_job() :: {integer(),scc()}. -type dataflow_job() :: module(). -type warnings_job() :: module(). -type contract_remote_types_job() :: module(). @@ -98,7 +100,7 @@ job_fun :: fun(), init_data :: init_data(), regulator :: regulator(), - scc_to_pid :: scc_to_pid() + job_labels_to_pid :: job_labels_to_pid() }). -include("dialyzer.hrl"). @@ -128,49 +130,28 @@ parallel_job(Mode, Jobs, InitData, Timing) -> %%-------------------------------------------------------------------- %% API functions for workers (dialyzer_worker). --spec request_activation(coordinator()) -> ok. - -request_activation({_Collector, Regulator, _SCCtoPid}) -> - Regulator ! {req, self()}, - wait_activation(). - --spec job_done(job(), job_result(), coordinator()) -> ok. - -job_done(Job, Result, {Collector, Regulator, _SCCtoPid}) -> - Regulator ! done, - Collector ! {done, Job, Result}, - ok. - --spec get_next_label(integer(), coordinator()) -> integer(). - -%% For the 'compile' worker. -get_next_label(EstimatedSize, {Collector, _Regulator, _SCCtoPid}) -> - Collector ! {next_label_request, EstimatedSize, self()}, - receive - {next_label_reply, NextLabel} -> NextLabel - end. - --spec wait_for_success_typings([scc() | module()], coordinator()) -> +-spec wait_for_success_typings([job_label()], coordinator()) -> 'ok'. %% Helper for 'sigtype' and 'dataflow' workers. -wait_for_success_typings(SCCs, {_Collector, _Regulator, SCCtoPid}) -> - F = fun(SCC) -> - %% The SCCs that SCC depends on have always been started. - try ets:lookup_element(SCCtoPid, SCC, 2) of - Pid when is_pid(Pid) -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - ok - end - catch - _:_ -> - %% Already finished. +wait_for_success_typings(Labels, {_Collector, _Regulator, JobLabelsToPid}) -> + F = + fun(JobLabel) -> + %% The jobs that job depends on have always been started. + try ets:lookup_element(JobLabelsToPid, JobLabel, 2) of + Pid when is_pid(Pid) -> + Ref = erlang:monitor(process, Pid), + receive + {'DOWN', Ref, process, Pid, _Info} -> ok end - end, - lists:foreach(F, SCCs). + catch + _:_ -> + %% Already finished. + ok + end + end, + lists:foreach(F, Labels). %%-------------------------------------------------------------------- @@ -179,18 +160,16 @@ wait_for_success_typings(SCCs, {_Collector, _Regulator, SCCtoPid}) -> spawn_jobs(Mode, Jobs, InitData, Timing) -> Collector = self(), Regulator = spawn_regulator(), - - SCCtoPid = + JobLabelsToPid = if Mode =:= 'typesig'; Mode =:= 'dataflow' -> - ets:new(scc_to_pid, [{read_concurrency, true}]); + ets:new(job_labels_to_pid, [{read_concurrency, true}]); true -> none end, + Coordinator = {Collector, Regulator, JobLabelsToPid}, - Coordinator = {Collector, Regulator, SCCtoPid}, - - JobFun = job_fun(SCCtoPid, Mode, InitData, Coordinator), + JobFun = job_fun(JobLabelsToPid, Mode, InitData, Coordinator), %% Limit the number of processes we start in order to save memory. MaxNumberOfInitJobs = 20 * dialyzer_utils:parallelism(), @@ -212,7 +191,8 @@ spawn_jobs(Mode, Jobs, InitData, Timing) -> #state{mode = Mode, active = JobCount, result = InitResult, next_label = 0, job_fun = JobFun, jobs = RestJobs, - init_data = InitData, regulator = Regulator, scc_to_pid = SCCtoPid}. + init_data = InitData, regulator = Regulator, + job_labels_to_pid = JobLabelsToPid}. launch_jobs(Jobs, _JobFun, 0) -> Jobs; @@ -227,17 +207,18 @@ job_fun(none, Mode, InitData, Coordinator) -> _ = dialyzer_worker:launch(Mode, Job, InitData, Coordinator), ok end; -job_fun(SCCtoPid, Mode, InitData, Coordinator) -> +job_fun(JobLabelsToPid, Mode, InitData, Coordinator) -> fun(Job) -> + JobLabel = get_job_label(Mode, Job), Pid = dialyzer_worker:launch(Mode, Job, InitData, Coordinator), - true = ets:insert(SCCtoPid, {Job, Pid}), + true = ets:insert(JobLabelsToPid, {JobLabel, Pid}), ok end. collect_result(#state{mode = Mode, active = Active, result = Result, next_label = NextLabel, init_data = InitData, jobs = JobsLeft, job_fun = JobFun, - regulator = Regulator, scc_to_pid = SCCtoPid} = State) -> + regulator = Regulator, job_labels_to_pid = JobLabelsToPID} = State) -> receive {next_label_request, Estimation, Pid} -> Pid ! {next_label_reply, NextLabel}, @@ -253,15 +234,15 @@ collect_result(#state{mode = Mode, active = Active, result = Result, {NewResult, NextLabel}; _ -> if - SCCtoPid =:= none -> ok; - true -> ets:delete(SCCtoPid) + JobLabelsToPID =:= none -> ok; + true -> ets:delete(JobLabelsToPID) end, NewResult end; N -> if - SCCtoPid =:= none -> ok; - true -> true = ets:delete(SCCtoPid, Job) + JobLabelsToPID =:= none -> ok; + true -> true = ets:delete(JobLabelsToPID, get_job_label(Mode, Job)) end, NewJobsLeft = case JobsLeft of @@ -288,6 +269,44 @@ update_result(Mode, InitData, Job, Data, Result) -> Data ++ Result end. + +-type job_label() :: integer() | module(). + +-type job_input() :: scc() | module(). + +-spec get_job_label(mode(), job()) -> job_label(). + +get_job_label(typesig, {Label, _Input}) -> Label; +get_job_label(dataflow, Job) -> Job; +get_job_label(contract_remote_types, Job) -> Job; +get_job_label(record_remote_types, Job) -> Job; +get_job_label(warnings, Job) -> Job; +get_job_label(compile, Job) -> Job. + +-spec get_job_input(mode(), job()) -> job_input(). + +get_job_input(typesig, {_Label, Input}) -> Input; +get_job_input(dataflow, Job) -> Job; +get_job_input(contract_remote_types, Job) -> Job; +get_job_input(record_remote_types, Job) -> Job; +get_job_input(warnings, Job) -> Job; +get_job_input(compile, Job) -> Job. + +-spec job_done(job(), job_result(), coordinator()) -> ok. + +job_done(Job, Result, {Collector, Regulator, _JobLabelsToPID}) -> + Regulator ! done, + Collector ! {done, Job, Result}, + ok. + +-spec get_next_label(integer(), coordinator()) -> integer(). + +get_next_label(EstimatedSize, {Collector, _Regulator, _JobLabelsToPID}) -> + Collector ! {next_label_request, EstimatedSize, self()}, + receive + {next_label_reply, NextLabel} -> NextLabel + end. + %%-------------------------------------------------------------------- %% The regulator server %% @@ -304,6 +323,12 @@ wait_activation() -> activate_pid(Pid) -> Pid ! activate. +-spec request_activation(coordinator()) -> ok. + +request_activation({_Collector, Regulator, _JobLabelsToPID}) -> + Regulator ! {req, self()}, + wait_activation(). + spawn_regulator() -> InitTickets = dialyzer_utils:parallelism(), spawn_link(fun() -> regulator_loop(InitTickets, queue:new()) end). diff --git a/lib/dialyzer/src/dialyzer_cplt.erl b/lib/dialyzer/src/dialyzer_cplt.erl new file mode 100644 index 0000000000..dcd200adb5 --- /dev/null +++ b/lib/dialyzer/src/dialyzer_cplt.erl @@ -0,0 +1,545 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%%------------------------------------------------------------------- +%%% File : dialyzer_cplt.erl +%%% Author : Tobias Lindahl <tobiasl@it.uu.se> +%%% Description : Interface to display information in the persistent +%%% lookup table files. +%%% This file handles persistent of "classic" PLT +%%% files (rather than incremental ones). It intentionally +%%% duplicates dialyzer_iplt.erl, since they are +%%% expected to diverge, and for this file to potentially +%%% be deprecated in favour of the incremental one at +%%% some point in the future. +%%% +%%% Created : 23 Jul 2004 by Tobias Lindahl <tobiasl@it.uu.se> +%%%------------------------------------------------------------------- +-module(dialyzer_cplt). + +-export([check_plt/3, + compute_md5_from_files/1, + included_files/1, + from_file/1, + get_default_cplt_filename/0, + merge_plts_or_report_conflicts/2, + plt_and_info_from_file/1, + to_file/4, + is_cplt/1 + ]). + +%% Debug utilities +-export([pp_non_returning/0, pp_mod/1]). + +-export_type([file_md5/0]). + +-include_lib("stdlib/include/ms_transform.hrl"). + +%%---------------------------------------------------------------------- + +-type deep_string() :: string() | [deep_string()]. + +%%---------------------------------------------------------------------- + + +-include("dialyzer.hrl"). + +-type file_md5() :: {file:filename(), binary()}. + +-record(file_plt, {version = "" :: string(), + file_md5_list = [] :: [file_md5()], + info = dict:new() :: dict:dict(), + contracts = dict:new() :: dict:dict(), + callbacks = dict:new() :: dict:dict(), + types = dict:new() :: dict:dict(), + exported_types = sets:new() :: sets:set(), + mod_deps :: dialyzer_callgraph:mod_deps(), + implementation_md5 = [] :: [file_md5()]}). + +%%---------------------------------------------------------------------- + +-spec get_default_cplt_filename() -> file:filename(). + +get_default_cplt_filename() -> + case os:getenv("DIALYZER_PLT") of + false -> + CacheDir = filename:basedir(user_cache, "erlang"), + filename:join(CacheDir, ".dialyzer_plt"); + UserSpecPlt -> UserSpecPlt + end. + +-spec plt_and_info_from_file(file:filename()) -> {dialyzer_plt:plt(), #plt_info{}}. + +plt_and_info_from_file(FileName) -> + from_file(FileName, true). + +-spec from_file(file:filename()) -> dialyzer_plt:plt(). + +from_file(FileName) -> + from_file(FileName, false). + +from_file(FileName, ReturnInfo) -> + Plt = dialyzer_plt:new(), + Fun = fun() -> from_file1(Plt, FileName, ReturnInfo) end, + case subproc(Fun) of + {ok, Return} -> + Return; + {error, Msg} -> + dialyzer_plt:delete(Plt), + plt_error(Msg) + end. + +from_file1(Plt, FileName, ReturnInfo) -> + case get_record_from_file(FileName) of + {ok, Rec} -> + case check_version(Rec) of + error -> + Msg = io_lib:format("Old PLT file ~ts\n", [FileName]), + {error, Msg}; + ok -> + #file_plt{info = FileInfo, + contracts = FileContracts, + callbacks = FileCallbacks, + types = FileTypes, + exported_types = FileExpTypes} = Rec, + Types = [{Mod, maps:from_list(dict:to_list(Types))} || + {Mod, Types} <- dict:to_list(FileTypes)], + CallbacksList = dict:to_list(FileCallbacks), + CallbacksByModule = + [{M, [Cb || {{M1,_,_},_} = Cb <- CallbacksList, M1 =:= M]} || + M <- lists:usort([M || {{M,_,_},_} <- CallbacksList])], + #plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes} = Plt, + [true, true, true] = + [ets:insert(ETS, Data) || + {ETS, Data} <- [{ETSInfo, dict:to_list(FileInfo)}, + {ETSTypes, Types}, + {ETSContracts, dict:to_list(FileContracts)}]], + true = ets:insert(ETSCallbacks, CallbacksByModule), + true = ets:insert(ETSExpTypes, [{ET} || + ET <- sets:to_list(FileExpTypes)]), + case ReturnInfo of + false -> {ok, Plt}; + true -> + PltInfo = #plt_info{files = Rec#file_plt.file_md5_list, + mod_deps = Rec#file_plt.mod_deps}, + {ok, {Plt, PltInfo}} + end + end; + {error, Reason} -> + Msg = io_lib:format("Could not read PLT file ~ts: ~p\n", + [FileName, Reason]), + {error, Msg} + end. + +-type err_rsn() :: 'not_valid' | 'no_such_file' | 'read_error'. + +-spec included_files(file:filename()) -> {'ok', [file:filename()]} + | {'error', err_rsn()}. + +included_files(FileName) -> + Fun = fun() -> included_files1(FileName) end, + subproc(Fun). + +included_files1(FileName) -> + case get_record_from_file(FileName) of + {ok, #file_plt{file_md5_list = Md5}} -> + {ok, [File || {File, _} <- Md5]}; + {error, _What} = Error -> + Error + end. + +check_version(#file_plt{version = ?VSN, implementation_md5 = ImplMd5}) -> + case compute_new_md5(ImplMd5, [], []) of + ok -> ok; + {differ, _, _} -> error; + {error, _} -> error + end; +check_version(#file_plt{}) -> error. + +get_record_from_file(FileName) -> + case file:read_file(FileName) of + {ok, Bin} -> + try binary_to_term(Bin) of + #file_plt{} = FilePLT -> {ok, FilePLT}; + _ -> {error, not_valid} + catch + _:_ -> {error, not_valid} + end; + {error, enoent} -> + {error, no_such_file}; + {error, _} -> + {error, read_error} + end. + +-spec is_cplt(file:filename()) -> boolean(). +is_cplt(FileName) -> + case get_record_from_file(FileName) of + {ok, _} -> true; + {error, _} -> false + end. + +-spec merge_disj_plts([#plt{}]) -> #plt{}. + +%% One of the PLTs of the list is augmented with the contents of the +%% other PLTs, and returned. The other PLTs are deleted. +%% +%% The keys are compared when checking for disjointness. Sometimes the +%% key is a module(), sometimes an mfa(). It boils down to checking if +%% any module occurs more than once. +merge_disj_plts(List) -> + {InfoList, TypesList, ExpTypesList, ContractsList, CallbacksList} = + group_fields(List), + #plt{info = table_disj_merge(InfoList), + types = table_disj_merge(TypesList), + exported_types = sets_disj_merge(ExpTypesList), + contracts = table_disj_merge(ContractsList), + callbacks = table_disj_merge(CallbacksList) + }. + +group_fields(List) -> + InfoList = [Info || #plt{info = Info} <- List], + TypesList = [Types || #plt{types = Types} <- List], + ExpTypesList = [ExpTypes || #plt{exported_types = ExpTypes} <- List], + ContractsList = [Contracts || #plt{contracts = Contracts} <- List], + CallbacksList = [Callbacks || #plt{callbacks = Callbacks} <- List], + {InfoList, TypesList, ExpTypesList, ContractsList, CallbacksList}. + +-spec merge_plts_or_report_conflicts([file:filename()], [#plt{}]) -> #plt{}. + +merge_plts_or_report_conflicts(PltFiles, Plts) -> + try + merge_disj_plts(Plts) + catch throw:{dialyzer_error, not_disjoint_plts} -> + IncFiles = lists:append([begin {ok, Fs} = included_files(F), Fs end + || F <- PltFiles]), + ConfFiles = find_duplicates(IncFiles), + Msg = io_lib:format("Could not merge PLTs since they are not disjoint\n" + "The following files are included in more than one " + "PLTs:\n~tp\n", [ConfFiles]), + plt_error(Msg) + end. + +find_duplicates(List) -> + ModList = [filename:basename(E) || E <- List], + SortedList = lists:usort(ModList), + lists:usort(ModList -- SortedList). + +-spec to_file(file:filename(), #plt{}, dialyzer_callgraph:mod_deps(), #plt_info{}) -> 'ok'. + +%% Write the PLT to file, and delete the PLT. +to_file(FileName, Plt, ModDeps, MD5_OldModDeps) -> + Fun = fun() -> to_file1(FileName, Plt, ModDeps, MD5_OldModDeps) end, + Return = subproc(Fun), + dialyzer_plt:delete(Plt), + case Return of + ok -> ok; + {error, Msg} -> plt_error(Msg) + end. + +to_file1(FileName, + #plt{info = ETSInfo, types = ETSTypes, contracts = ETSContracts, + callbacks = ETSCallbacks, exported_types = ETSExpTypes}, + ModDeps, #plt_info{files = MD5, mod_deps = OldModDeps}) -> + NewModDeps = dict:merge(fun(_Key, OldVal, NewVal) -> + ordsets:union(OldVal, NewVal) + end, + OldModDeps, ModDeps), + ImplMd5 = compute_implementation_md5(), + CallbacksList = + [Cb || + {_M, Cbs} <- dialyzer_utils:ets_tab2list(ETSCallbacks), + Cb <- Cbs], + Callbacks = dict:from_list(CallbacksList), + Info = dict:from_list(dialyzer_utils:ets_tab2list(ETSInfo)), + Types = dialyzer_utils:ets_tab2list(ETSTypes), + Contracts = dict:from_list(dialyzer_utils:ets_tab2list(ETSContracts)), + ExpTypes = sets:from_list([E || {E} <- dialyzer_utils:ets_tab2list(ETSExpTypes)]), + FileTypes = dict:from_list([{Mod, dict:from_list(maps:to_list(MTypes))} || + {Mod, MTypes} <- Types]), + Record = #file_plt{version = ?VSN, + file_md5_list = MD5, + info = Info, + contracts = Contracts, + callbacks = Callbacks, + types = FileTypes, + exported_types = ExpTypes, + mod_deps = NewModDeps, + implementation_md5 = ImplMd5}, + Bin = term_to_binary(Record, [compressed]), + case file:write_file(FileName, Bin) of + ok -> ok; + {error, Reason} -> + Msg = io_lib:format("Could not write PLT file ~ts: ~w\n", + [FileName, Reason]), + {error, Msg} + end. + +-type md5_diff() :: [{'differ', atom()} | {'removed', atom()}]. +-type check_error() :: err_rsn() | {'no_file_to_remove', file:filename()}. + +-spec check_plt(file:filename(), [file:filename()], [file:filename()]) -> + 'ok' + | {'error', check_error()} + | {'differ', [file_md5()], md5_diff(), dialyzer_callgraph:mod_deps()} + | {'old_version', [file_md5()]}. + +check_plt(FileName, RemoveFiles, AddFiles) -> + Fun = fun() -> check_plt1(FileName, RemoveFiles, AddFiles) end, + subproc(Fun). + +check_plt1(FileName, RemoveFiles, AddFiles) -> + case get_record_from_file(FileName) of + {ok, #file_plt{file_md5_list = Md5, mod_deps = ModDeps} = Rec} -> + case check_version(Rec) of + ok -> + case compute_new_md5(Md5, RemoveFiles, AddFiles) of + ok -> ok; + {differ, NewMd5, DiffMd5} -> {differ, NewMd5, DiffMd5, ModDeps}; + {error, _What} = Err -> Err + end; + error -> + case compute_new_md5(Md5, RemoveFiles, AddFiles) of + ok -> {old_version, Md5}; + {differ, NewMd5, _DiffMd5} -> {old_version, NewMd5}; + {error, _What} = Err -> Err + end + end; + Error -> Error + end. + +compute_new_md5(Md5, [], []) -> + compute_new_md5_1(Md5, [], []); +compute_new_md5(Md5, RemoveFiles0, AddFiles0) -> + %% Assume that files are first removed and then added. Files that + %% are both removed and added will be checked for consistency in the + %% normal way. If they have moved, we assume that they differ. + RemoveFiles = RemoveFiles0 -- AddFiles0, + AddFiles = AddFiles0 -- RemoveFiles0, + InitDiffList = init_diff_list(RemoveFiles, AddFiles), + case init_md5_list(Md5, RemoveFiles, AddFiles) of + {ok, NewMd5} -> compute_new_md5_1(NewMd5, [], InitDiffList); + {error, _What} = Error -> Error + end. + +compute_new_md5_1([{File, Md5} = Entry|Entries], NewList, Diff) -> + case compute_md5_from_file(File) of + Md5 -> compute_new_md5_1(Entries, [Entry|NewList], Diff); + NewMd5 -> + ModName = beam_file_to_module(File), + compute_new_md5_1(Entries, [{File, NewMd5}|NewList], [{differ, ModName}|Diff]) + end; +compute_new_md5_1([], _NewList, []) -> + ok; +compute_new_md5_1([], NewList, Diff) -> + {differ, lists:keysort(1, NewList), Diff}. + +-spec compute_implementation_md5() -> [file_md5()]. + +compute_implementation_md5() -> + Dir = code:lib_dir(dialyzer), + Files1 = ["erl_bif_types.beam", "erl_types.beam"], + Files2 = [filename:join([Dir, "ebin", F]) || F <- Files1], + compute_md5_from_files(Files2). + +-spec compute_md5_from_files([file:filename()]) -> [file_md5()]. + +compute_md5_from_files(Files) -> + lists:keysort(1, [{F, compute_md5_from_file(F)} || F <- Files]). + +compute_md5_from_file(File) -> + case beam_lib:all_chunks(File) of + {ok, _, Chunks} -> + %% We cannot use beam_lib:md5 because it does not consider + %% the debug_info chunk, where typespecs are likely stored. + %% So we consider almost all chunks except the useless ones. + Filtered = [[ID, Chunk] || {ID, Chunk} <- Chunks, ID =/= "CInf", ID =/= "Docs"], + erlang:md5(lists:sort(Filtered)); + {error, beam_lib, {file_error, _, enoent}} -> + Msg = io_lib:format("File not found: ~ts\n", [File]), + plt_error(Msg); + {error, beam_lib, _} -> + Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), + plt_error(Msg) + end. + +init_diff_list(RemoveFiles, AddFiles) -> + RemoveSet0 = sets:from_list([beam_file_to_module(F) || F <- RemoveFiles]), + AddSet0 = sets:from_list([beam_file_to_module(F) || F <- AddFiles]), + DiffSet = sets:intersection(AddSet0, RemoveSet0), + RemoveSet = sets:subtract(RemoveSet0, DiffSet), + %% Added files and diff files will appear as diff files from the md5 check. + [{removed, F} || F <- sets:to_list(RemoveSet)]. + +init_md5_list(Md5, RemoveFiles, AddFiles) -> + Files = [{remove, F} || F <- RemoveFiles] ++ [{add, F} || F <- AddFiles], + DiffFiles = lists:keysort(2, Files), + Md5Sorted = lists:keysort(1, Md5), + init_md5_list_1(Md5Sorted, DiffFiles, []). + +init_md5_list_1([{File, _Md5}|Md5Left], [{remove, File}|DiffLeft], Acc) -> + init_md5_list_1(Md5Left, DiffLeft, Acc); +init_md5_list_1([{File, _Md5} = Entry|Md5Left], [{add, File}|DiffLeft], Acc) -> + init_md5_list_1(Md5Left, DiffLeft, [Entry|Acc]); +init_md5_list_1([{File1, _Md5} = Entry|Md5Left] = Md5List, + [{Tag, File2}|DiffLeft] = DiffList, Acc) -> + case File1 < File2 of + true -> init_md5_list_1(Md5Left, DiffList, [Entry|Acc]); + false -> + %% Just an assert. + true = File1 > File2, + case Tag of + add -> init_md5_list_1(Md5List, DiffLeft, [{File2, <<>>}|Acc]); + remove -> {error, {no_file_to_remove, File2}} + end + end; +init_md5_list_1([], DiffList, Acc) -> + AddFiles = [{F, <<>>} || {add, F} <- DiffList], + {ok, lists:reverse(Acc, AddFiles)}; +init_md5_list_1(Md5List, [], Acc) -> + {ok, lists:reverse(Acc, Md5List)}. + + +subproc(Fun) -> + F = fun() -> + exit(try Fun() + catch throw:T -> + {thrown, T} + end) + end, + {Pid, Ref} = erlang:spawn_monitor(F), + receive {'DOWN', Ref, process, Pid, Return} -> + case Return of + {thrown, T} -> throw(T); + _ -> Return + end + end. + + +beam_file_to_module(Filename) -> + list_to_atom(filename:basename(Filename, ".beam")). + + +-spec plt_error(deep_string()) -> no_return(). + +plt_error(Msg) -> + throw({dialyzer_error, lists:flatten(Msg)}). + +table_disj_merge([H|T]) -> + table_disj_merge(T, H). + +table_disj_merge([], Acc) -> + Acc; +table_disj_merge([Plt|Plts], Acc) -> + case table_is_disjoint(Plt, Acc) of + true -> + NewAcc = merge_tables(Plt, Acc), + table_disj_merge(Plts, NewAcc); + false -> throw({dialyzer_error, not_disjoint_plts}) + end. + +sets_disj_merge([H|T]) -> + sets_disj_merge(T, H). + +sets_disj_merge([], Acc) -> + Acc; +sets_disj_merge([Plt|Plts], Acc) -> + case table_is_disjoint(Plt, Acc) of + true -> + NewAcc = merge_tables(Plt, Acc), + sets_disj_merge(Plts, NewAcc); + false -> throw({dialyzer_error, not_disjoint_plts}) + end. + +table_is_disjoint(T1, T2) -> + tab_is_disj(ets:first(T1), T1, T2). + +tab_is_disj('$end_of_table', _T1, _T2) -> + true; +tab_is_disj(K1, T1, T2) -> + case ets:member(T2, K1) of + false -> + tab_is_disj(ets:next(T1, K1), T1, T2); + true -> + false + end. + +merge_tables(T1, T2) -> + tab_merge(ets:first(T1), T1, T2). + +tab_merge('$end_of_table', T1, T2) -> + case ets:first(T1) of % no safe_fixtable()... + '$end_of_table' -> + true = ets:delete(T1), + T2; + Key -> + tab_merge(Key, T1, T2) + end; +tab_merge(K1, T1, T2) -> + Vs = ets:lookup(T1, K1), + NextK1 = ets:next(T1, K1), + true = ets:delete(T1, K1), + true = ets:insert(T2, Vs), + tab_merge(NextK1, T1, T2). + + +%%--------------------------------------------------------------------------- +%% Debug utilities. + +-spec pp_non_returning() -> 'ok'. + +pp_non_returning() -> + PltFile = get_default_cplt_filename(), + Plt = from_file(PltFile), + List = ets:tab2list(Plt#plt.info), + Unit = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, + erl_types:t_is_unit(Ret)], + None = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, + erl_types:t_is_none(Ret)], + io:format("=========================================\n"), + io:format("= Loops =\n"), + io:format("=========================================\n\n"), + lists:foreach(fun({{M, F, _}, Type}) -> + io:format("~w:~tw~ts.\n", + [M, F, dialyzer_utils:format_sig(Type)]) + end, lists:sort(Unit)), + io:format("\n"), + io:format("=========================================\n"), + io:format("= Errors =\n"), + io:format("=========================================\n\n"), + lists:foreach(fun({{M, F, _}, Type}) -> + io:format("~w:~w~s.\n", + [M, F, dialyzer_utils:format_sig(Type)]) + end, lists:sort(None)), + dialyzer_plt:delete(Plt). + +-spec pp_mod(atom()) -> 'ok'. + +pp_mod(Mod) when is_atom(Mod) -> + PltFile = get_default_cplt_filename(), + Plt = from_file(PltFile), + case dialyzer_plt:lookup_module(Plt, Mod) of + {value, List} -> + lists:foreach(fun({{_, F, _}, Ret, Args}) -> + T = erl_types:t_fun(Args, Ret), + S = dialyzer_utils:format_sig(T), + io:format("-spec ~tw~ts.\n", [F, S]) + end, lists:sort(List)); + none -> + io:format("dialyzer: Found no module named '~s' in the PLT\n", [Mod]) + end, + dialyzer_plt:delete(Plt). diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 8e9b32434e..68068ed67e 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -1557,8 +1557,9 @@ bind_tuple(Pat, Type, Map, State, Opaques, Rev) -> true -> Any = t_any(), [_Head|AnyTail] = [Any || _ <- Es], - UntypedRecord = t_tuple([Tag|AnyTail]), - case state__lookup_record(cerl:atom_val(Tag), length(Tags), State) of + TagAtomVal = cerl:atom_val(Tag), + UntypedRecord = t_tuple([t_atom(TagAtomVal)|AnyTail]), + case state__lookup_record(TagAtomVal, length(Tags), State) of error -> {false, UntypedRecord}; {ok, Record, _FieldNames} -> diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl index 3b04b56f80..13d0a65dbb 100644 --- a/lib/dialyzer/src/dialyzer_gui_wx.erl +++ b/lib/dialyzer/src/dialyzer_gui_wx.erl @@ -69,7 +69,7 @@ rawWarnings :: list(), backend_pid :: pid() | 'undefined', expl_pid :: pid() | 'undefined'}). - + %%------------------------------------------------------------------------ -spec start(#options{}) -> ?RET_NOTHING_SUSPICIOUS. @@ -84,7 +84,7 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> {ok, Host} = inet:gethostname(), %%---------- initializing frame --------- - Frame = wxFrame:new(Wx, -1, "Dialyzer " ++ ?VSN ++ " @ " ++ Host), + Frame = wxFrame:new(Wx, -1, "Dialyzer " ++ ?VSN ++ " @ " ++ Host), wxFrame:connect(Frame, close_window), FileMenu = createFileMenu(), WarningsMenu = createWarningsMenu(), @@ -119,7 +119,7 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> {style, ?wxTE_MULTILINE bor ?wxTE_READONLY bor ?wxHSCROLL}]), DefaultPath = code:root_dir(), - + FilePicker = wxFilePickerCtrl:new(Frame, ?FilePicker, [{path, DefaultPath}, {message, "Choose File to Analyse"}, @@ -184,7 +184,7 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> WarnButtons = wxBoxSizer:new(?wxHORIZONTAL), RunButtons = wxBoxSizer:new(?wxHORIZONTAL), Buttons = wxFlexGridSizer:new(3), - + _ = wxSizer:add(ChooseButtons, DeleteButton, ?BorderOpt), _ = wxSizer:add(ChooseButtons, DeleteAllButton, ?BorderOpt), _ = wxSizer:add(ChooseItem, Lab1, Center), @@ -232,7 +232,7 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> wxWindow:setSizer(Frame, All), wxWindow:setSizeHints(Frame, {1150,600}), wxWindow:show(Frame), - + Warnings = [{?WARN_RETURN_NO_RETURN, ?menuID_WARN_NO_RETURN_FUN}, {?WARN_RETURN_ONLY_EXIT, ?menuID_WARN_ERROR_HANDLING_FUN}, {?WARN_NOT_CALLED, ?menuID_WARN_UNUSED_FUN}, @@ -256,22 +256,22 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> case InitPltFiles of [] -> dialyzer_plt:new(); _ -> - Plts = [dialyzer_plt:from_file(F) || F <- InitPltFiles], - dialyzer_plt:merge_plts_or_report_conflicts(InitPltFiles, Plts) + Plts = [dialyzer_cplt:from_file(F) || F <- InitPltFiles], + dialyzer_cplt:merge_plts_or_report_conflicts(InitPltFiles, Plts) end, - + #gui_state{add = AddButton, add_dir = AddDirButton, add_rec = AddRecButton, - chosen_box = ChosenBox, - clear_chosen = DeleteAllButton, - clear_log = ClearLogButton, + chosen_box = ChosenBox, + clear_chosen = DeleteAllButton, + clear_log = ClearLogButton, explain_warn = ExplainWarnButton, - clear_warn = ClearWarningsButton, - del_file = DeleteButton, + clear_warn = ClearWarningsButton, + del_file = DeleteButton, doc_plt = dialyzer_plt:new(), dir_entry = DirPicker, - file_box = FilePicker, + file_box = FilePicker, files_to_analyze = ordsets:new(), gui = Wx, init_plt = InitPlt, @@ -281,7 +281,7 @@ create_window(Wx, #options{init_plts = InitPltFiles} = DialyzerOptions) -> options = DialyzerOptions, run = RunButton, stop = StopButton, - frame = Frame, + frame = Frame, warnings_box = WarningsBox, wantedWarnings = Warnings, rawWarnings = []}. @@ -318,7 +318,7 @@ createWarningsMenu() -> WarningsMenu. addCheckedItem(Menu, ItemId, Str) -> - _ = wxMenu:appendCheckItem(Menu, ItemId, Str), + _ = wxMenu:appendCheckItem(Menu, ItemId, Str), wxMenu:check(Menu, ItemId, true). createPltMenu() -> @@ -359,57 +359,57 @@ createHelpMenu() -> gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt, log = Log, frame = Frame, warnings_box = WarningsBox} = State) -> - receive + receive #wx{event = #wxClose{}} -> %% io:format("~p Closing window ~n", [self()]), ok = wxFrame:setStatusText(Frame, "Closing...",[]), wxWindow:destroy(Frame), ?RET_NOTHING_SUSPICIOUS; %% ----- Menu ----- - #wx{id = ?menuID_FILE_SAVE_LOG, obj = Frame, + #wx{id = ?menuID_FILE_SAVE_LOG, obj = Frame, event = #wxCommand{type = command_menu_selected}} -> save_file(State, log), gui_loop(State); - #wx{id=?menuID_FILE_SAVE_WARNINGS, obj=Frame, + #wx{id=?menuID_FILE_SAVE_WARNINGS, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> save_file(State, warnings), gui_loop(State); - #wx{id=?menuID_FILE_QUIT, obj=Frame, + #wx{id=?menuID_FILE_QUIT, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> case maybe_quit(State) of true -> ?RET_NOTHING_SUSPICIOUS; false -> gui_loop(State) end; - #wx{id=?menuID_PLT_SHOW_CONTENTS, obj=Frame, + #wx{id=?menuID_PLT_SHOW_CONTENTS, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> show_doc_plt(State), gui_loop(State); - #wx{id=?menuID_PLT_SEARCH_CONTENTS, obj=Frame, + #wx{id=?menuID_PLT_SEARCH_CONTENTS, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> case dialyzer_plt:get_specs(DocPlt) of "" -> error_sms(State, "No analysis has been made yet!\n"); _ -> search_doc_plt(State) end, gui_loop(State); - #wx{id=?menuID_OPTIONS_INCLUDE_DIR, obj=Frame, + #wx{id=?menuID_OPTIONS_INCLUDE_DIR, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> NewOptions = include_dialog(State), NewState = State#gui_state{options = NewOptions}, gui_loop(NewState); - #wx{id=?menuID_OPTIONS_MACRO, obj=Frame, + #wx{id=?menuID_OPTIONS_MACRO, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> NewOptions = macro_dialog(State), NewState = State#gui_state{options = NewOptions}, gui_loop(NewState); - #wx{id=?menuID_HELP_MANUAL, obj=Frame, + #wx{id=?menuID_HELP_MANUAL, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> handle_help(State, "Dialyzer Manual", "manual.txt"), gui_loop(State); - #wx{id=?menuID_HELP_WARNING_OPTIONS, obj=Frame, + #wx{id=?menuID_HELP_WARNING_OPTIONS, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> handle_help(State, "Dialyzer Warnings", "warnings.txt"), gui_loop(State); - #wx{id=?menuID_HELP_ABOUT, obj=Frame, + #wx{id=?menuID_HELP_ABOUT, obj=Frame, event=#wxCommand{type=command_menu_selected}} -> Message = " This is DIALYZER version " ++ ?VSN ++ " \n"++ "DIALYZER is a DIscrepancy AnaLYZer for ERlang programs.\n\n"++ @@ -494,8 +494,8 @@ gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt, gui_loop(NewState); {BackendPid, cserver, CServer, Plt} -> Self = self(), - Fun = - fun() -> + Fun = + fun() -> dialyzer_explanation:expl_loop(Self, CServer, Plt) end, ExplanationPid = spawn_link(Fun), @@ -527,7 +527,7 @@ maybe_quit(#gui_state{frame = Frame} = State) -> %% ------------ Yes/No Question ------------ dialog(#gui_state{frame = Frame}, Message, Title) -> MessageWin = wxMessageDialog:new(Frame, Message, [{caption, Title},{style, ?wxYES_NO bor ?wxICON_QUESTION bor ?wxNO_DEFAULT}]), - case wxDialog:showModal(MessageWin) of + case wxDialog:showModal(MessageWin) of ?wxID_YES -> true; ?wxID_NO -> @@ -535,7 +535,7 @@ dialog(#gui_state{frame = Frame}, Message, Title) -> ?wxID_CANCEL -> false end. - + search_doc_plt(#gui_state{gui = Wx} = State) -> Dialog = wxFrame:new(Wx, ?SearchPltDialog, "Search the PLT",[{size,{400,100}},{style, ?wxSTAY_ON_TOP}]), Size = {size,{120,30}}, @@ -591,8 +591,8 @@ search_plt_loop(State= #gui_state{doc_plt = DocPlt, frame = Frame}, Win, ModText M = format_search(wxTextCtrl:getValue(ModText)), F = format_search(wxTextCtrl:getValue(FunText)), A = format_search(wxTextCtrl:getValue(ArText)), - - if + + if (M =:= '_') orelse (F =:= '_') orelse (A =:= '_') -> error_sms(State, "Please give:\n Module (atom)\n Function (atom)\n Arity (integer)\n"), search_plt_loop(State, Win, ModText, FunText, ArText, Search, Cancel); @@ -606,15 +606,15 @@ search_plt_loop(State= #gui_state{doc_plt = DocPlt, frame = Frame}, Win, ModText free_editor(State, "Content of PLT", NonEmptyString) end end - end. + end. format_search([]) -> '_'; format_search(String) -> try list_to_integer(String) catch error:_ -> list_to_atom(String) - end. - + end. + show_doc_plt(#gui_state{doc_plt = DocPLT} = State) -> case dialyzer_plt:get_specs(DocPLT) of "" -> error_sms(State, "No analysis has been made yet!\n"); @@ -648,8 +648,8 @@ free_editor(#gui_state{gui = Wx, frame = Frame}, Title, Contents0) -> Width0 = LongestLine * 7 + 60, Width = if Width0 > 800 -> 800; true -> Width0 end, Size = {size,{Width, Height}}, - Win = wxFrame:new(Wx, ?Message, Title, [{size,{Width+4, Height+50}}]), - + Win = wxFrame:new(Wx, ?Message, Title, [{size,{Width+4, Height+50}}]), + Editor = wxTextCtrl:new(Win, ?Message_Info, [Size, {style, ?wxTE_MULTILINE @@ -659,7 +659,7 @@ free_editor(#gui_state{gui = Wx, frame = Frame}, Title, Contents0) -> Ok = wxButton:new(Win, ?Message_Ok, [{label, "OK"}]), wxButton:connect(Ok, command_button_clicked), Layout = wxBoxSizer:new(?wxVERTICAL), - + _ = wxSizer:add(Layout, Editor, ?BorderOpt), Flag = ?wxALIGN_CENTER bor ?wxBOTTOM bor ?wxALL, _ = wxSizer:add(Layout, Ok, [{flag, Flag}, ?Border]), @@ -686,7 +686,7 @@ handle_add_files(#gui_state{chosen_box = ChosenBox, file_box = FileBox, File -> NewFile = ordsets:new(), NewFile1 = ordsets:add_element(File,NewFile), - Ext = + Ext = case wxRadioBox:getSelection(Mode) of 0 -> ".beam"; 1-> ".erl" @@ -699,7 +699,7 @@ handle_add_dir(#gui_state{chosen_box = ChosenBox, dir_entry = DirBox, case wxDirPickerCtrl:getPath(DirBox) of "" -> State; - Dir -> + Dir -> NewDir = ordsets:new(), NewDir1 = ordsets:add_element(Dir,NewDir), Ext = case wxRadioBox:getSelection(Mode) of @@ -708,13 +708,13 @@ handle_add_dir(#gui_state{chosen_box = ChosenBox, dir_entry = DirBox, end, State#gui_state{files_to_analyze = add_files(filter_mods(NewDir1,Ext), FileList, ChosenBox, Ext)} end. - + handle_add_rec(#gui_state{chosen_box = ChosenBox, dir_entry = DirBox, files_to_analyze = FileList, mode = Mode} = State) -> case wxDirPickerCtrl:getPath(DirBox) of "" -> State; - Dir -> + Dir -> NewDir = ordsets:new(), NewDir1 = ordsets:add_element(Dir,NewDir), TargetDirs = ordsets:union(NewDir1, all_subdirs(NewDir1)), @@ -736,7 +736,7 @@ handle_file_delete(#gui_state{chosen_box = ChosenBox, handle_file_delete_all(#gui_state{chosen_box = ChosenBox} = State) -> wxListBox:clear(ChosenBox), State#gui_state{files_to_analyze = ordsets:new()}. - + add_files(File, FileList, ChosenBox, Ext) -> Set = filter_mods(FileList, Ext), Files = ordsets:union(File, Set), @@ -747,7 +747,7 @@ add_files(File, FileList, ChosenBox, Ext) -> filter_mods(Mods, Extension) -> Fun = fun(X) -> filename:extension(X) =:= Extension - orelse + orelse (filelib:is_dir(X) andalso contains_files(X, Extension)) end, @@ -781,7 +781,7 @@ start_analysis(State) -> Msg = "You must choose one or more files or dirs\n" "before starting the analysis!", error_sms(State, Msg), - config_gui_stop(State), + config_gui_stop(State), State; {ok, Files} -> Msg = "\n========== Starting Analysis ==========\n\n", @@ -825,8 +825,8 @@ run_analysis(State, Analysis) -> Self = self(), NewAnalysis = Analysis#analysis{doc_plt = dialyzer_plt:new()}, LegalWarnings = find_legal_warnings(State), - Fun = - fun() -> + Fun = + fun() -> dialyzer_analysis_callgraph:start(Self, LegalWarnings, NewAnalysis) end, BackendPid = spawn_link(Fun), @@ -834,13 +834,13 @@ run_analysis(State, Analysis) -> find_legal_warnings(#gui_state{menu = #menu{warnings = MenuWarnings}, wantedWarnings = Warnings }) -> - ordsets:from_list([Tag || {Tag, MenuItem} <- Warnings, + ordsets:from_list([Tag || {Tag, MenuItem} <- Warnings, wxMenu:isChecked(MenuWarnings, MenuItem)]). update_editor(Editor, Msg) -> wxTextCtrl:appendText(Editor,Msg). -config_gui_stop(State) -> +config_gui_stop(State) -> wxWindow:disable(State#gui_state.stop), wxWindow:enable(State#gui_state.run), wxWindow:enable(State#gui_state.del_file), @@ -904,7 +904,7 @@ save_file(#gui_state{frame = Frame, warnings_box = WBox, log = Log} = State, Typ _ -> error_sms(State, "Could not write to file:\n") end end. - + include_dialog(#gui_state{gui = Wx, frame = Frame, options = Options}) -> Size = {size,{300,480}}, Dialog = wxFrame:new(Wx, ?IncludeDir, "Include Directories",[Size]), @@ -913,11 +913,11 @@ include_dialog(#gui_state{gui = Wx, frame = Frame, options = Options}) -> DirPicker = wxDirPickerCtrl:new(Dialog, ?InclPicker, [{path, DefaultPath}, {message, "Choose Directory to Include"}, - {style,?wxDIRP_DIR_MUST_EXIST bor ?wxDIRP_USE_TEXTCTRL}]), + {style,?wxDIRP_DIR_MUST_EXIST bor ?wxDIRP_USE_TEXTCTRL}]), Box = wxListBox:new(Dialog, ?InclBox, [{size, {200,300}}, {style, ?wxLB_EXTENDED bor ?wxLB_HSCROLL - bor ?wxLB_NEEDED_SB}]), + bor ?wxLB_NEEDED_SB}]), AddButton = wxButton:new(Dialog, ?InclAdd, [{label, "Add"}]), DeleteButton = wxButton:new(Dialog, ?InclDel, [{label, "Delete"}]), DeleteAllButton = wxButton:new(Dialog, ?InclDelAll, [{label, "Delete All"}]), @@ -954,7 +954,7 @@ include_loop(Options, Win, Box, DirPicker, Frame) -> receive #wx{id = ?InclCancel, event = #wxCommand{type = command_button_clicked}} -> - wxWindow:destroy(Win), + wxWindow:destroy(Win), Options; #wx{id = ?IncludeDir, event = #wxClose{type = close_window}} -> wxWindow:destroy(Win), @@ -994,8 +994,8 @@ include_loop(Options, Win, Box, DirPicker, Frame) -> wxListBox:clear(Box), NewOptions = Options#options{include_dirs = []}, include_loop(NewOptions, Win, Box, DirPicker, Frame) - end. - + end. + macro_dialog(#gui_state{gui = Wx, frame = Frame, options = Options}) -> Size = {size,{300,480}}, Size1 = {size,{120,30}}, @@ -1020,9 +1020,9 @@ macro_dialog(#gui_state{gui = Wx, frame = Frame, options = Options}) -> wxButton:connect(Ok, command_button_clicked), wxButton:connect(Cancel, command_button_clicked), - Macros = [io_lib:format("~p = ~p", [X, Y]) + Macros = [io_lib:format("~p = ~p", [X, Y]) || {X,Y} <- Options#options.defines], - + wxListBox:set(Box, Macros), Layout = wxBoxSizer:new(?wxVERTICAL), Item = wxBoxSizer:new(?wxHORIZONTAL), @@ -1056,7 +1056,7 @@ macro_loop(Options, Win, Box, MacroText, TermText, Frame) -> receive #wx{id = ?MacroCancel, event = #wxCommand{type = command_button_clicked}} -> - wxWindow:destroy(Win), + wxWindow:destroy(Win), Options; #wx{id = ?MacroDir, event = #wxClose{type = close_window}} -> wxWindow:destroy(Win), @@ -1071,12 +1071,12 @@ macro_loop(Options, Win, Box, MacroText, TermText, Frame) -> #wx{id = ?MacroAdd, event = #wxCommand{type = command_button_clicked}} -> Defines = Options#options.defines, - NewDefines = + NewDefines = case wxTextCtrl:getValue(MacroText) of "" -> Defines; Macro -> case wxTextCtrl:getValue(TermText) of - "" -> + "" -> orddict:store(list_to_atom(Macro), true, Defines); String -> orddict:store(list_to_atom(Macro), String, Defines) @@ -1092,7 +1092,7 @@ macro_loop(Options, Win, Box, MacroText, TermText, Frame) -> case wxListBox:getSelections(Box) of {0, _} -> Options; {_, List} -> - Fun = + Fun = fun(X) -> Val = wxControlWithItems:getString(Box,X), [MacroName|_] = re:split(Val, " ", [{return, list}, unicode]), @@ -1113,19 +1113,19 @@ macro_loop(Options, Win, Box, MacroText, TermText, Frame) -> wxListBox:clear(Box), NewOptions = Options#options{defines = []}, macro_loop(NewOptions, Win, Box, MacroText, TermText, Frame) - end. + end. handle_help(State, Title, Txt) -> FileName = filename:join([code:lib_dir(dialyzer), "doc", Txt]), case file:open(FileName, [read]) of {error, Reason} -> - error_sms(State, + error_sms(State, io_lib:format("Could not find doc/~ts file!\n\n ~tp", [Txt, Reason])); {ok, _Handle} -> case file:read_file(FileName) of {error, Reason} -> - error_sms(State, + error_sms(State, io_lib:format("Could not read doc/~ts file!\n\n ~tp", [Txt, Reason])); {ok, Binary} -> @@ -1143,7 +1143,7 @@ add_warnings(#gui_state{warnings_box = WarnBox, W <- NewRawWarns], wxListBox:set(WarnBox, WarnList), State#gui_state{rawWarnings = NewRawWarns}. - + handle_explanation(#gui_state{rawWarnings = RawWarns, warnings_box = WarnBox, expl_pid = ExplPid} = State) -> @@ -1173,13 +1173,13 @@ explanation_loop(#gui_state{expl_pid = ExplPid} = State) -> show_explanation(#gui_state{gui = Wx} = State, Explanation) -> case Explanation of none -> - output_sms(State, ?DIALYZER_MESSAGE_TITLE, + output_sms(State, ?DIALYZER_MESSAGE_TITLE, "There is not any explanation for this error!\n", info); Expl -> ExplString = format_explanation(Expl), Size = {size,{700, 300}}, - Win = wxFrame:new(Wx, ?ExplWin, "Dialyzer Explanation", [{size,{740, 350}}]), - + Win = wxFrame:new(Wx, ?ExplWin, "Dialyzer Explanation", [{size,{740, 350}}]), + Editor = wxTextCtrl:new(Win, ?ExplText, [Size, {style, ?wxTE_MULTILINE @@ -1201,11 +1201,11 @@ show_explanation(#gui_state{gui = Wx} = State, Explanation) -> NewState = State#gui_state{explanation_box = Editor}, show_explanation_loop(NewState, Win, Explanation) end. - + show_explanation_loop(#gui_state{frame = Frame, expl_pid = ExplPid} = State, Win, Explanation) -> receive - {ExplPid, none, _} -> - output_sms(State, ?DIALYZER_MESSAGE_TITLE, + {ExplPid, none, _} -> + output_sms(State, ?DIALYZER_MESSAGE_TITLE, "There is not any other explanation for this error!\n", info), show_explanation_loop(State, Win, Explanation); {ExplPid, further, NewExplanation} -> diff --git a/lib/dialyzer/src/dialyzer_incremental.erl b/lib/dialyzer/src/dialyzer_incremental.erl new file mode 100644 index 0000000000..962f81d42e --- /dev/null +++ b/lib/dialyzer/src/dialyzer_incremental.erl @@ -0,0 +1,816 @@ +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(dialyzer_incremental). + +-export([start/1, start_report_modules_analyzed/1, start_report_modules_changed_and_analyzed/1]). + +-include("dialyzer.hrl"). +-include_lib("kernel/include/file.hrl"). % needed for #file_info{} + +-record(incremental_state, + {backend_pid :: pid() | 'undefined', + code_server = none :: 'none' | dialyzer_codeserver:codeserver(), + erlang_mode = false :: boolean(), + external_calls = [] :: [{mfa(), warning_info()}], + external_types = [] :: [{mfa(), warning_info()}], + legal_warnings = ordsets:new() :: [dial_warn_tag()], + mod_deps = dict:new() :: dialyzer_callgraph:mod_deps(), + output = standard_io :: io:device(), + output_format = formatted :: format(), + filename_opt = basename :: filename_opt(), + error_location = ?ERROR_LOCATION :: error_location(), + indent_opt = ?INDENT_OPT :: iopt(), + output_plt = none :: file:filename(), + plt_info = none :: 'none' | #iplt_info{}, + report_mode = normal :: rep_mode(), + return_status = ?RET_NOTHING_SUSPICIOUS :: dial_ret(), + warning_modules = [] :: [module()], + stored_warnings = [] :: [raw_warning()] + }). + +-type incrementality_reason() :: + no_stored_warnings_in_plt + | plt_built_with_different_version + | new_plt_file + | warnings_changed + | {incremental_changes, NumModulesChangedOrRemoved :: non_neg_integer()}. + +-record(incrementality_metrics, + {total_modules :: non_neg_integer(), + analysed_modules :: non_neg_integer(), + reason :: incrementality_reason() + }). + +%%-------------------------------------------------------------------- + +-spec start(#options{}) -> {dial_ret(), [dial_warning()]}. + +start(Opts) -> + {{Ret,Warns}, _ModulesAnalyzed} = start_report_modules_analyzed(Opts), + {Ret,Warns}. + +-spec start_report_modules_analyzed(#options{}) -> + {{dial_ret(), [dial_warning()]}, [module()]}. + +start_report_modules_analyzed(#options{analysis_type = incremental} = Options) -> + {{Ret, Warn}, _Changed, Analyzed} = + start_report_modules_changed_and_analyzed(Options), + {{Ret, Warn}, Analyzed}. + +-spec start_report_modules_changed_and_analyzed(#options{}) -> + {{dial_ret(), [dial_warning()]}, + Changed :: undefined | [module()], + Analyzed :: [module()]}. + +start_report_modules_changed_and_analyzed( #options{analysis_type = incremental} = Options) -> + Opts1 = init_opts_for_incremental(Options), + assert_metrics_file_valid(Opts1), + #options{init_plts = [InitPlt], legal_warnings = LegalWarnings} = Opts1, + Files = get_files_from_opts(Opts1), + case dialyzer_iplt:check_incremental_plt(InitPlt, Opts1, Files) of + {ok, #iplt_info{files = Md5, warning_map = none}, ModuleToPathLookup} -> + report_no_stored_warnings(Opts1, Md5), + PltInfo = #iplt_info{files = Md5, legal_warnings = LegalWarnings}, + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(maps:values(ModuleToPathLookup), Opts1, dialyzer_plt:new(), PltInfo), []); + {ok, #iplt_info{warning_map=WarningMap, files = Md5}, ModuleToPathLookup} -> + report_stored_warnings_no_changes(Opts1, Md5), + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(return_existing_errors(Opts1, WarningMap), []); + {old_version, Md5, ModuleToPathLookup} -> + report_different_plt_version(Opts1, Md5), + PltInfo = #iplt_info{files = Md5, legal_warnings = LegalWarnings}, + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(maps:values(ModuleToPathLookup), Opts1, dialyzer_plt:new(), PltInfo), undefined); + {new_file, Md5, ModuleToPathLookup} -> + report_new_plt_file(Opts1, InitPlt, Md5), + PltInfo = #iplt_info{files = Md5, legal_warnings = LegalWarnings}, + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(maps:values(ModuleToPathLookup), Opts1, dialyzer_plt:new(), PltInfo), undefined); + {differ, Md5, _DiffMd5, _ModDeps, none, ModuleToPathLookup} -> + report_no_stored_warnings(Opts1, Md5), + PltInfo = #iplt_info{files = Md5, legal_warnings = LegalWarnings}, + AllFiles = maps:values(ModuleToPathLookup), + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(AllFiles, Opts1, dialyzer_plt:new(), PltInfo), undefined); + {differ, Md5, DiffMd5, ModDeps, WarningMap, ModuleToPathLookup} -> + report_incremental_analysis_needed(Opts1, DiffMd5), + {AnalFiles, ModsToRemove, ModDepsInRemainingPlt} = + expand_dependent_modules(Md5, DiffMd5, ModDeps, ModuleToPathLookup), + WarningsInRemainingPlt = + sets:fold(fun(Mod, Acc) -> maps:remove(Mod, Acc) end, + WarningMap, ModsToRemove), + Plt = clean_plt(InitPlt, ModsToRemove), + + PltInfo = #iplt_info{files = Md5, + mod_deps = ModDepsInRemainingPlt, + warning_map = WarningsInRemainingPlt, + legal_warnings = LegalWarnings}, + ChangedOrRemovedMods = [ChangedOrRemovedMod || {_, ChangedOrRemovedMod} <- DiffMd5], + case AnalFiles =:= [] of + true -> + %% Only removed stuff that's unused. Just write the PLT. + report_stored_warnings_only_safe_removals(Opts1, Md5, DiffMd5), + dialyzer_iplt:to_file(Opts1#options.output_plt, Plt, ModDepsInRemainingPlt, PltInfo), + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(return_existing_errors(Opts1, WarningsInRemainingPlt), ChangedOrRemovedMods); + false -> + report_degree_of_incrementality(Opts1, Md5, DiffMd5, AnalFiles), + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(AnalFiles, Opts1, Plt, PltInfo), ChangedOrRemovedMods) + end; + {legal_warnings_changed, Md5, ModuleToPathLookup} -> + report_change_in_legal_warnings(Opts1, Md5), + PltInfo = #iplt_info{files = Md5, legal_warnings = LegalWarnings}, + AllFiles = maps:values(ModuleToPathLookup), + write_module_to_path_lookup(Opts1, ModuleToPathLookup), + enrich_with_modules_changed(do_analysis(AllFiles, Opts1, dialyzer_plt:new(), PltInfo), undefined); + {error, not_valid} -> + Msg = io_lib:format("The file: ~ts is not a valid PLT file\n~s", + [InitPlt, default_plt_error_msg()]), + cl_error(Msg); + {error, read_error} -> + Msg = io_lib:format("Could not read the PLT: ~ts\n~s", + [InitPlt, default_plt_error_msg()]), + cl_error(Msg) + end. + +-spec enrich_with_modules_changed({{Ret :: dial_ret(), Warns :: [dial_warning()]}, Analyzed :: [module()]}, Changed :: undefined | [module()]) -> + {{dial_ret(), [dial_warning()]}, Changed :: undefined | [module()], Analyzed :: [module()]}. + +enrich_with_modules_changed({{Ret,Warns}, Analyzed}, Changed) -> + {{Ret,Warns}, Changed, Analyzed}. + +default_plt_error_msg() -> + "Remove the broken PLT file or point to the correct location.\n". + +init_opts_for_incremental(Opts) -> + InitPlt = + case Opts#options.init_plts of + []-> dialyzer_iplt:get_default_iplt_filename(); + [Plt] -> Plt; + Plts -> + Msg = + io_lib:format("Incremental mode does not support multiple PLT files (~ts)\n", + [format_plts(Plts)]), + cl_error(Msg) + end, + OutputPlt = + case Opts#options.output_plt of + none -> InitPlt; + ExplicitlySetOutputPlt -> ExplicitlySetOutputPlt + end, + Opts#options{ + analysis_type = incremental, + defines = [], + from = byte_code, + init_plts = [InitPlt], + include_dirs = [], + output_plt = OutputPlt, + use_contracts = true, + get_warnings = true + }. + +assert_metrics_file_valid(#options{metrics_file = none}) -> + ok; + +assert_metrics_file_valid(#options{metrics_file = MetricsFile}) -> + case check_if_writable(MetricsFile) of + true -> ok; + false -> + Msg = io_lib:format(" The metrics file ~ts is not writable", [MetricsFile]), + cl_error(Msg) + end. + +write_metrics_file(#options{metrics_file = none}, _Format, _Args) -> + ok; + +write_metrics_file(#options{metrics_file = MetricsFile}, Format, Args) -> + case file:write_file(MetricsFile, io_lib:fwrite(Format, Args)) of + ok -> ok; + {error, Reason} -> + Msg = io_lib:format("Could not write metrics file ~ts: ~w\n", + [MetricsFile, Reason]), + throw({dialyzer_error, Msg}) + end. + +write_metrics_file( + Opts, + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumAnalysedModules, + reason = Reason} + ) -> + + ReasonDescription = + case Reason of + no_stored_warnings_in_plt -> "no_stored_warnings_in_plt"; + plt_built_with_different_version -> "plt_built_with_different_version"; + new_plt_file -> "new_plt_file"; + warnings_changed -> "warnings_changed"; + {incremental_changes, NumChangedModules} -> + io_lib:format("incremental_changes\nchanged_or_removed_modules: ~B", [NumChangedModules]) + end, + + write_metrics_file( + Opts, + "total_modules: ~B\nanalysed_modules: ~B\nreason: ~s\n", + [NumTotalModules, NumAnalysedModules, ReasonDescription]). + +write_module_to_path_lookup(#options{module_lookup_file = none}, _ModuleToPathLookup) -> + ok; + +write_module_to_path_lookup(#options{module_lookup_file = LookupFile}, ModuleToPathLookup) -> + Output = [ io_lib:fwrite("~ts, ~ts\n", [atom_to_list(ModuleName), ModulePath]) || {ModuleName, ModulePath} <- maps:to_list(ModuleToPathLookup)], + case file:write_file(LookupFile, Output) of + ok -> ok; + {error, Reason} -> + Msg = io_lib:format("Could not write module lookup file ~ts: ~w\n", + [LookupFile, Reason]), + throw({dialyzer_error, Msg}) + end. + + +report_new_plt_file(#options{report_mode = ReportMode} = Opts, InitPlt, Md5) -> + NumTotalModules = length(Md5), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumTotalModules, + reason = new_plt_file + }, + case ReportMode of + quiet -> ok; + normal -> ok; + verbose -> io:format("PLT does not yet exist at ~s, so an analysis must be run for ~w modules to populate it\n", [InitPlt, NumTotalModules]) + end, + write_metrics_file(Opts, Metrics). + +report_different_plt_version(#options{report_mode = ReportMode} = Opts, Md5) -> + NumTotalModules = length(Md5), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumTotalModules, + reason = plt_built_with_different_version + }, + case ReportMode of + quiet -> ok; + normal -> ok; + verbose -> io:format("PLT is for a different Dialyzer version, so an analysis must be run for ~w modules to rebuild it\n", [NumTotalModules]) + end, + write_metrics_file(Opts, Metrics). + +report_stored_warnings_no_changes(#options{report_mode = ReportMode} = Opts, Md5) -> + NumTotalModules = length(Md5), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = 0, + reason = {incremental_changes, 0} + }, + case ReportMode of + quiet -> ok; + normal -> ok; + verbose -> io:format("PLT has fully cached the request, so no additional analysis is needed\n", []) + end, + write_metrics_file(Opts, Metrics). + +report_stored_warnings_only_safe_removals(#options{report_mode = ReportMode} = Opts, Md5, Removed) -> + NumTotalModules = length(Md5), + NumRemovedModuled = length(Removed), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = 0, + reason = {incremental_changes, NumRemovedModuled} + }, + case ReportMode of + quiet -> ok; + normal -> ok; + verbose -> io:format("PLT has fully cached the request because nothing depended on the file removed, so no additional analysis is needed\n", []) + end, + write_metrics_file(Opts, Metrics). + +report_no_stored_warnings(#options{report_mode = ReportMode} = Opts, Md5) -> + NumTotalModules = length(Md5), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumTotalModules, + reason = no_stored_warnings_in_plt + }, + case ReportMode of + quiet -> ok; + normal -> ok; + verbose -> + io:format( + "PLT does not contain cached warnings, so an analysis must be run for ~w modules to rebuild it\n", + [length(Md5)] + ) + end, + write_metrics_file(Opts, Metrics). + +report_incremental_analysis_needed(#options{report_mode = ReportMode}, DiffMd5) -> + case ReportMode of + quiet -> ok; + normal -> io:format("There have been changes to analyze\n", []); + verbose -> report_md5_diff(DiffMd5) + end. + +report_degree_of_incrementality(#options{report_mode = ReportMode} = Opts, Md5, ChangedOrRemovedFiles, FilesThatNeedAnalysis) -> + NumTotalModules = length(Md5), + NumChangedOrRemovedModules = length(ChangedOrRemovedFiles), + NumAnalysedModules = length(FilesThatNeedAnalysis), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumAnalysedModules, + reason = {incremental_changes, NumChangedOrRemovedModules} + }, + ReportFun = fun () -> + io:format( + " Of the ~B files being tracked, ~B have been changed or removed, " + "resulting in ~B requiring analysis because they depend on those changes\n", + [NumTotalModules, NumChangedOrRemovedModules, NumAnalysedModules]) + end, + case ReportMode of + quiet -> ok; + normal -> ReportFun(); + verbose -> + ReportFun(), + io:format(" Modules which will be analysed: ~p\n", [[path_to_mod(P) || P <- FilesThatNeedAnalysis]]) + end, + write_metrics_file(Opts, Metrics). + +report_change_in_legal_warnings(#options{report_mode = ReportMode} = Opts, Md5) -> + NumTotalModules = length(Md5), + Metrics = + #incrementality_metrics{ + total_modules = NumTotalModules, + analysed_modules = NumTotalModules, + reason = warnings_changed + }, + ReportFun = fun () -> + io:format( + "PLT was built for a different set of enabled warnings, so an analysis must be run for ~w modules to rebuild it\n", + [NumTotalModules] + ) + end, + case ReportMode of + quiet -> ok; + normal -> ReportFun(); + verbose -> ReportFun() + end, + write_metrics_file(Opts, Metrics). + +report_analysis_start(#options{report_mode = quiet}) -> ok; +report_analysis_start(_) -> + io:format("Proceeding with incremental analysis..."). + +report_elapsed_time(T1, T2, #options{report_mode = ReportMode}) -> + case ReportMode of + quiet -> ok; + _ -> + ElapsedTime = T2 - T1, + Mins = ElapsedTime div 60000, + Secs = (ElapsedTime rem 60000) / 1000, + io:format(" done in ~wm~.2fs\n", [Mins, Secs]) + end. + +report_md5_diff(List) -> + io:format(" The PLT information is not up to date:\n", []), + case [Mod || {removed, Mod} <- List] of + [] -> ok; + RemovedMods -> io:format(" Removed modules: ~p\n", [RemovedMods]) + end, + case [Mod || {differ, Mod} <- List] of + [] -> ok; + ChangedMods -> io:format(" Changed modules: ~p\n", [ChangedMods]) + end. +%%-------------------------------------------------------------------- + + +format_plts([Plt]) -> Plt; +format_plts([Plt|Plts]) -> + Plt ++ ", " ++ format_plts(Plts). + +%%-------------------------------------------------------------------- + +do_analysis(Files, Options, Plt, PltInfo) -> + assert_writable(Options#options.output_plt), + report_analysis_start(Options), + State1 = init_output(Options), + State2 = State1#incremental_state{ + legal_warnings = Options#options.legal_warnings, + output_plt = Options#options.output_plt, + plt_info = PltInfo, + erlang_mode = Options#options.erlang_mode, + report_mode = Options#options.report_mode, + warning_modules = get_warning_modules_from_opts(Options) + }, + InitAnalysis = #analysis{ + type = succ_typings, + defines = Options#options.defines, + include_dirs = Options#options.include_dirs, + files = Files, + start_from = Options#options.from, + timing = Options#options.timing, + plt = Plt, + use_contracts = Options#options.use_contracts, + callgraph_file = Options#options.callgraph_file, + mod_deps_file = Options#options.mod_deps_file, + solvers = Options#options.solvers + }, + State3 = start_analysis(State2, InitAnalysis), + {T1, _} = statistics(wall_clock), + RetAndWarns = cl_loop(State3), + {T2, _} = statistics(wall_clock), + report_elapsed_time(T1, T2, Options), + {RetAndWarns, lists:usort([path_to_mod(F) || F <- Files])}. + +%%-------------------------------------------------------------------- + + +assert_writable(PltFile) -> + case check_if_writable(PltFile) of + true -> ok; + false -> + Msg = io_lib:format(" The PLT file ~ts is not writable", [PltFile]), + cl_error(Msg) + end. + +check_if_writable(PltFile) -> + case filelib:is_regular(PltFile) of + true -> is_writable_file_or_dir(PltFile); + false -> + case filelib:is_dir(PltFile) of + true -> + false; + false -> + DirName = filename:dirname(PltFile), + case filelib:is_dir(DirName) of + false -> + case filelib:ensure_dir(PltFile) of + ok -> + true; + {error, _} -> + false + end; + true -> + is_writable_file_or_dir(DirName) + end + end + end. + +is_writable_file_or_dir(File) -> + case file:read_file_info(File) of + {ok, #file_info{access = A}} -> + (A =:= write) orelse (A =:= read_write); + {error, _} -> + false + end. + +%%-------------------------------------------------------------------- + +clean_plt(PltFile, RemovedMods) -> + %% Clean the plt from the removed modules. + Plt = dialyzer_iplt:from_file(PltFile), + sets:fold(fun(M, AccPlt) -> dialyzer_plt:delete_module(AccPlt, M) end, + Plt, RemovedMods). + +expand_dependent_modules(_Md5, DiffMd5, ModDeps, ModuleToPathLookup) -> + ChangedMods = sets:from_list([M || {differ, M} <- DiffMd5]), + RemovedMods = sets:from_list([M || {removed, M} <- DiffMd5]), + BigSet = sets:union(ChangedMods, RemovedMods), + BigList = sets:to_list(BigSet), + ExpandedSet = expand_dependent_modules_1(BigList, BigSet, ModDeps), + NewModDeps = dialyzer_callgraph:strip_module_deps(ModDeps, BigSet), + AnalyzeMods = sets:subtract(ExpandedSet, RemovedMods), + FilterFun = fun(File) -> + Mod = path_to_mod(File), + sets:is_element(Mod, AnalyzeMods) + end, + {[F || F <- maps:values(ModuleToPathLookup), FilterFun(F)], ExpandedSet, NewModDeps}. + +expand_dependent_modules_1([Mod|Mods], Included, ModDeps) -> + case dict:find(Mod, ModDeps) of + {ok, Deps} -> + NewDeps = sets:subtract(sets:from_list(Deps), Included), + case sets:size(NewDeps) of + 0 -> expand_dependent_modules_1(Mods, Included, ModDeps); + _ -> + NewIncluded = sets:union(Included, NewDeps), + expand_dependent_modules_1(sets:to_list(NewDeps) ++ Mods, + NewIncluded, ModDeps) + end; + error -> + expand_dependent_modules_1(Mods, Included, ModDeps) + end; +expand_dependent_modules_1([], Included, _ModDeps) -> + Included. + +path_to_mod(File) -> + list_to_atom(filename:basename(File, ".beam")). + +init_output(#options{output_file = OutFile, + output_plt = OutPlt, + output_format = OutFormat, + filename_opt = FOpt, + indent_opt = IOpt, + error_location = EOpt} = Opts) -> + State = #incremental_state{output_format = OutFormat, + output_plt = OutPlt, + filename_opt = FOpt, + indent_opt = IOpt, + error_location = EOpt, + warning_modules = get_warning_modules_from_opts(Opts)}, + case OutFile =:= none of + true -> + State; + false -> + case file:open(OutFile, [write]) of + {ok, File} -> + %% Warnings and errors can include Unicode characters. + ok = io:setopts(File, [{encoding, unicode}]), + State#incremental_state{output = File}; + {error, Reason} -> + Msg = io_lib:format("Could not open output file ~tp, Reason: ~p\n", + [OutFile, Reason]), + cl_error(State, lists:flatten(Msg)) + end + end. + +-spec maybe_close_output_file(#incremental_state{}, boolean()) -> 'ok'. + +maybe_close_output_file(State, OutputPltInUse) -> + case State#incremental_state.output of + standard_io -> ok; + File when OutputPltInUse -> ok = file:close(File); + _File -> ok + end. + +%% ---------------------------------------------------------------- +%% +%% Main Loop +%% + +-define(LOG_CACHE_SIZE, 10). + +%%-spec cl_loop(#incremental_state{}) -> +cl_loop(State) -> + cl_loop(State, []). + +cl_loop(State, LogCache) -> + BackendPid = State#incremental_state.backend_pid, + receive + {BackendPid, log, LogMsg} -> + cl_loop(State, lists:sublist([LogMsg|LogCache], ?LOG_CACHE_SIZE)); + {BackendPid, warnings, Warnings} -> + NewState = store_warnings(State, Warnings), + cl_loop(NewState, LogCache); + {BackendPid, cserver, CodeServer, _Plt} -> % Plt is ignored + NewState = State#incremental_state{code_server = CodeServer}, + cl_loop(NewState, LogCache); + {BackendPid, done, NewPlt, _NewDocPlt} -> + return_value(State, NewPlt); + {BackendPid, ext_calls, ExtCalls} -> + cl_loop(State#incremental_state{external_calls = ExtCalls}, LogCache); + {BackendPid, ext_types, ExtTypes} -> + cl_loop(State#incremental_state{external_types = ExtTypes}, LogCache); + {BackendPid, mod_deps, ModDeps} -> + NewState = State#incremental_state{mod_deps = ModDeps}, + cl_loop(NewState, LogCache); + {'EXIT', BackendPid, {error, Reason}} -> + Msg = failed_anal_msg(Reason, LogCache), + cl_error(State, Msg); + {'EXIT', BackendPid, Reason} when Reason =/= 'normal' -> + Msg = failed_anal_msg(io_lib:format("~p", [Reason]), LogCache), + cl_error(State, Msg); + _Other -> + cl_loop(State, LogCache) + end. + +-spec failed_anal_msg(string(), [_]) -> nonempty_string(). + +failed_anal_msg(Reason, LogCache) -> + Msg = "Analysis failed with error:\n" ++ lists:flatten(Reason) ++ "\n", + case LogCache =:= [] of + true -> Msg; + false -> + Msg ++ "Last messages in the log cache:\n " ++ format_log_cache(LogCache) + end. + +%% +%% formats the log cache (treating it as a string) for pretty-printing +%% +format_log_cache(LogCache) -> + Str = lists:append(lists:reverse(LogCache)), + lists:join("\n ", string:lexemes(Str, "\n")). + +-spec store_warnings(#incremental_state{}, [raw_warning()]) -> #incremental_state{}. + +store_warnings(#incremental_state{stored_warnings = StoredWarnings} = St, Warnings) -> + St#incremental_state{stored_warnings = StoredWarnings ++ Warnings}. + +-spec cl_error(string()) -> no_return(). + +cl_error(Msg) -> + throw({dialyzer_error, lists:flatten(Msg)}). + +-spec cl_error(#incremental_state{}, string()) -> no_return(). + +cl_error(State, Msg) -> + case State#incremental_state.output of + standard_io -> ok; + Outfile -> io:format(Outfile, "\n~ts\n", [Msg]) + end, + maybe_close_output_file(State, true), + throw({dialyzer_error, lists:flatten(Msg)}). + +return_value( + State = + #incremental_state{ + code_server = CodeServer, + mod_deps = ModDeps, + output_plt = OutputPlt, + plt_info = PltInfo, + stored_warnings = NewPLTWarnings + }, + Plt) -> + %% Just for now: + case CodeServer =:= none of + true -> + ok; + false -> + dialyzer_codeserver:delete(CodeServer) + end, + OldPltWarnings = PltInfo#iplt_info.warning_map, + PLTUnknownWarnings = unknown_warnings_by_module(State), + PLTWarningMap = dialyzer_iplt:merge_warnings(NewPLTWarnings, PLTUnknownWarnings, OldPltWarnings), + PLTWarningList = + [Warn || Mod <- maps:keys(PLTWarningMap), Warn <- maps:get(Mod, PLTWarningMap, [])], + NewState = State#incremental_state{stored_warnings=PLTWarningList}, + % Write warnings for all modules to the PLT, even if we're not reporting + % them now, so subsequent runs can read them from the PLT + dialyzer_iplt:to_file(OutputPlt, Plt, ModDeps, PltInfo#iplt_info{warning_map=PLTWarningMap}), + handle_return_and_print(NewState, PLTWarningMap, true). + +handle_return_and_print(State, AllWarningsMap, OutputPltInUse) -> + WarningModules = State#incremental_state.warning_modules, + WarningsToReport = + case WarningModules =:= [] of + true -> + [Warn || Mod <- maps:keys(AllWarningsMap), Warn <- maps:get(Mod, AllWarningsMap, [])]; + false -> + [Warn || Mod <- WarningModules, Warn <- maps:get(Mod, AllWarningsMap, [])] + end, + RetValue = + case WarningsToReport =:= [] of + true -> ?RET_NOTHING_SUSPICIOUS; + false -> ?RET_DISCREPANCIES + end, + case State#incremental_state.erlang_mode of + false -> + #incremental_state{ + output = Output, + output_format = Format, + filename_opt = FOpt, + indent_opt = IOpt, + error_location = EOpt} = State, + print_warnings(WarningsToReport, Output, Format, FOpt, IOpt, EOpt), + maybe_close_output_file(State, OutputPltInUse), + {RetValue, []}; + true -> + {RetValue, set_warning_id(process_warnings(WarningsToReport), + State#incremental_state.error_location)} + end. + +return_existing_errors(Opts, PltWarnings) -> + State = init_output(Opts), + State1 = State#incremental_state{erlang_mode = Opts#options.erlang_mode}, + State2 = State1#incremental_state{warning_modules = get_warning_modules_from_opts(Opts)}, + ModulesAnalyzed = [], % No modules analyzed - we read straight from the cache + {handle_return_and_print(State2, PltWarnings, false), ModulesAnalyzed}. + +unknown_warnings_by_module(#incremental_state{legal_warnings = LegalWarnings, external_calls=Calls, external_types=Types}) -> + case ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) of + true -> + unknown_functions(Calls) ++ unknown_types(Types); + false -> [] + end. + +unknown_functions(Calls) -> + [{Mod, {?WARN_UNKNOWN, WarningInfo, {unknown_function, MFA}}} || {MFA, WarningInfo = {_, _, {Mod, _, _}}} <- Calls]. + +unknown_types(Types) -> + [{Mod, {?WARN_UNKNOWN, WarningInfo, {unknown_type, MFA}}} || + {MFA, WarningInfo = {_, _, {Mod, _, _}}} <- Types]. + +set_warning_id(Warnings, EOpt) -> + lists:map(fun({Tag, {File, Location, _MorMFA}, Msg}) -> + {Tag, {File, set_location(Location, EOpt)}, Msg} + end, Warnings). + +set_location({Line, _}, line) -> + Line; +set_location(Location, _EOpt) -> + Location. + +print_warnings([], _, _, _, _, _) -> + ok; +print_warnings(Warnings, Output, Format, FOpt, IOpt, EOpt) -> + PrWarnings = process_warnings(Warnings), + case PrWarnings of + [] -> ok; + [_|_] -> + PrWarningsId = set_warning_id(PrWarnings, EOpt), + S = case Format of + formatted -> + Opts = [{filename_opt, FOpt}, + {indent_opt, IOpt}, + {error_location, EOpt}], + [dialyzer:format_warning(W, Opts) || W <- PrWarningsId]; + raw -> + [io_lib:format("~tp. \n", [W]) || + W <- set_warning_id(PrWarningsId, EOpt)] + end, + io:format(Output, "\n~ts", [S]) + end. + +-spec process_warnings([raw_warning()]) -> [raw_warning()]. + +process_warnings(Warnings) -> + Warnings1 = lists:keysort(3, Warnings), %% First sort on Warning + Warnings2 = lists:keysort(2, Warnings1), %% Sort on file/line (and m/mfa..) + remove_duplicate_warnings(Warnings2, []). + +remove_duplicate_warnings([Duplicate, Duplicate|Left], Acc) -> + remove_duplicate_warnings([Duplicate|Left], Acc); +remove_duplicate_warnings([NotDuplicate|Left], Acc) -> + remove_duplicate_warnings(Left, [NotDuplicate|Acc]); +remove_duplicate_warnings([], Acc) -> + lists:reverse(Acc). + +get_files_from_opts(Options) -> + Files1 = add_files(Options#options.files), + Files2 = add_files_rec(Options#options.files_rec), + ordsets:union(Files1, Files2). + +get_warning_modules_from_opts(Options) -> + Files1 = add_files(Options#options.warning_files), + Files2 = add_files_rec(Options#options.warning_files_rec), + [path_to_mod(File) || File <- ordsets:union(Files1, Files2)]. + +add_files_rec(Files) -> + add_files(Files, true). + +add_files(Files) -> + add_files(Files, false). + +add_files(Files, Rec) -> + Files1 = [filename:absname(F) || F <- Files], + Files2 = ordsets:from_list(Files1), + Dirs = ordsets:filter(fun(X) -> filelib:is_dir(X) end, Files2), + Files3 = ordsets:subtract(Files2, Dirs), + Extension = ".beam", + Fun = add_file_fun(Extension), + lists:foldl( + fun(Dir, Acc) -> + filelib:fold_files(Dir, Extension, Rec, Fun, Acc) + end, + Files3, + Dirs + ). + +add_file_fun(Extension) -> + fun(File, AccFiles) -> + case filename:extension(File) =:= Extension of + true -> + AbsName = filename:absname(File), + ordsets:add_element(AbsName, AccFiles); + false -> AccFiles + end + end. + +-spec start_analysis(#incremental_state{}, #analysis{}) -> #incremental_state{}. + +start_analysis(State, Analysis) -> + Self = self(), + LegalWarnings = State#incremental_state.legal_warnings, + Fun = fun() -> + dialyzer_analysis_callgraph:start(Self, LegalWarnings, Analysis) + end, + BackendPid = spawn_link(Fun), + State#incremental_state{backend_pid = BackendPid}. diff --git a/lib/dialyzer/src/dialyzer_iplt.erl b/lib/dialyzer/src/dialyzer_iplt.erl new file mode 100644 index 0000000000..64cda8425c --- /dev/null +++ b/lib/dialyzer/src/dialyzer_iplt.erl @@ -0,0 +1,595 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%%------------------------------------------------------------------- +%%% File : dialyzer_iplt.erl +%%% Description : Interface to display information in the incremental +%%% persistent lookup tables stored in files. +%%% Incremental PLTs index files by module name, rather +%%% than absolute path, for portability, and track warnings +%%% generated per module, in order to cache them for later +%%% analyses. +%%%------------------------------------------------------------------- +-module(dialyzer_iplt). + +-export([check_incremental_plt/3, + included_modules/1, + from_file/1, + get_default_iplt_filename/0, + merge_warnings/3, + plt_and_info_from_file/1, + to_file/4, + is_iplt/1, + to_file_custom_vsn/6 % Used for testing certain kinds of backwards compatibility + ]). + +%% Debug utilities +-export([pp_non_returning/0, pp_mod/1]). + +-export_type([module_md5/0, warning_map/0]). + +-include_lib("stdlib/include/ms_transform.hrl"). + +%%---------------------------------------------------------------------- + +-type deep_string() :: string() | [deep_string()]. + +%%---------------------------------------------------------------------- + +-record(incremental_data, {mod_deps :: dialyzer_callgraph:mod_deps(), + warning_map = none :: warning_map(), + legal_warnings = none :: none | dial_warn_tags()}). + +-include("dialyzer.hrl"). + +-type warning_map() :: none | #{module() := [raw_warning()]}. +-type module_md5() :: {module(), binary()}. +-type module_file_path_lookup() :: #{module() => file:filename()}. + +-record(ifile_plt, + {version = "" :: string(), + module_md5_list = [] :: [module_md5()], + info = term_to_binary(#{}) :: binary(), %% encoded map() + contracts = term_to_binary(#{}) :: binary(), %% encoded map() + callbacks = term_to_binary(#{}) :: binary(), %% encoded map() + types = term_to_binary(#{}) :: binary(), %% encoded map() + exported_types = term_to_binary(#{}) :: binary(), %% encoded sets:set() + incremental_data = term_to_binary(#incremental_data{}) :: #incremental_data{} | binary(), %% encoded #incremental_data{} + implementation_md5 = [] :: [module_md5()]}). + +%%---------------------------------------------------------------------- + + +-spec get_default_iplt_filename() -> file:filename(). + +get_default_iplt_filename() -> + case os:getenv("DIALYZER_IPLT") of + false -> + CacheDir = filename:basedir(user_cache, "erlang"), + filename:join(CacheDir, ".dialyzer_iplt"); + UserSpecPlt -> UserSpecPlt + end. + +-spec plt_and_info_from_file(file:filename()) -> {dialyzer_plt:plt(), #iplt_info{}}. + +plt_and_info_from_file(FileName) -> + from_file(FileName, true). + +-spec from_file(file:filename()) -> dialyzer_plt:plt(). + +from_file(FileName) -> + from_file(FileName, false). + +from_file(FileName, ReturnInfo) -> + Plt = dialyzer_plt:new(), + Fun = fun() -> from_file1(Plt, FileName, ReturnInfo) end, + case subproc(Fun) of + {ok, Return} -> + Return; + {error, Msg} -> + dialyzer_plt:delete(Plt), + plt_error(Msg) + end. + +from_file1(Plt, FileName, ReturnInfo) -> + case get_record_from_file(FileName) of + {ok, Rec} -> + case check_version(Rec) of + error -> + Msg = io_lib:format("Old IPLT file ~ts\n", [FileName]), + {error, Msg}; + ok -> + #ifile_plt{info = CompressedInfo, + contracts = CompressedContracts, + callbacks = CompressedCallbacks, + types = CompressedTypes, + exported_types = CompressedExpTypes} = Rec, + FileInfo = binary_to_term(CompressedInfo), + FileContracts = binary_to_term(CompressedContracts), + FileCallbacks = binary_to_term(CompressedCallbacks), + FileTypes = binary_to_term(CompressedTypes), + FileExpTypes = binary_to_term(CompressedExpTypes), + CallbacksList = maps:to_list(FileCallbacks), + CallbacksByModule = + [{M, [Cb || {{M1,_,_},_} = Cb <- CallbacksList, M1 =:= M]} || + M <- lists:usort([M || {{M,_,_},_} <- CallbacksList])], + #plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes} = Plt, + [true, true, true] = + [ets:insert(ETS, Data) || + {ETS, Data} <- [{ETSInfo, maps:to_list(FileInfo)}, + {ETSTypes, FileTypes}, + {ETSContracts, maps:to_list(FileContracts)}]], + true = ets:insert(ETSCallbacks, CallbacksByModule), + true = ets:insert(ETSExpTypes, [{ET} || + ET <- sets:to_list(FileExpTypes)]), + case ReturnInfo of + false -> {ok, Plt}; + true -> + IncrementalData = get_incremental_data(Rec), + PltInfo = + #iplt_info{files = Rec#ifile_plt.module_md5_list, + mod_deps = IncrementalData#incremental_data.mod_deps, + warning_map = IncrementalData#incremental_data.warning_map, + legal_warnings = IncrementalData#incremental_data.legal_warnings}, + {ok, {Plt, PltInfo}} + end + end; + {error, Reason} -> + Msg = io_lib:format("Could not read IPLT file ~ts: ~p\n", + [FileName, Reason]), + {error, Msg} + end. + +-spec get_incremental_data(#ifile_plt{}) -> #incremental_data{}. +get_incremental_data(#ifile_plt{incremental_data = Data}) -> + case Data of + CompressedData when is_binary(CompressedData) -> + binary_to_term(CompressedData); + UncompressedData = #incremental_data{} -> % To support older PLTs that didn't have this field compressed + UncompressedData + end. + +-type err_rsn() :: 'not_valid' | 'no_such_file' | 'read_error'. + +-spec included_modules(file:filename()) -> {'ok', [module()]} + | {'error', err_rsn()}. + +included_modules(FileName) -> + Fun = fun() -> included_modules1(FileName) end, + subproc(Fun). + +included_modules1(FileName) -> + case get_record_from_file(FileName) of + {ok, #ifile_plt{module_md5_list = Md5}} -> + {ok, [ModuleName || {ModuleName, _} <- Md5]}; + {error, _What} = Error -> + Error + end. + +check_version(#ifile_plt{version = ?VSN, implementation_md5 = ImplMd5}) -> + case compute_new_md5(ImplMd5, [], [], implementation_module_paths()) of + ok -> ok; + {differ, _, _} -> error; + {error, _} -> error + end; +check_version(#ifile_plt{}) -> error. + +get_record_from_file(FileName) -> + case file:read_file(FileName) of + {ok, Bin} -> + try binary_to_term(Bin) of + #ifile_plt{} = FilePLT -> {ok, FilePLT}; + _ -> {error, not_valid} + catch + _:_ -> {error, not_valid} + end; + {error, enoent} -> + {error, no_such_file}; + {error, _} -> + {error, read_error} + end. + +-spec is_iplt(file:filename()) -> boolean(). +is_iplt(FileName) -> + case get_record_from_file(FileName) of + {ok, _} -> true; + {error, _} -> false + end. + +-spec to_file(file:filename(), dialyzer_plt:plt(), dialyzer_callgraph:mod_deps(), #iplt_info{}) -> 'ok'. + +%% Write the PLT to file, and deletes the PLT. +to_file(FileName, Plt, ModDeps, PLTInfo) -> + Fun = fun() -> to_file1(FileName, Plt, ModDeps, PLTInfo) end, + Return = subproc(Fun), + dialyzer_plt:delete(Plt), + case Return of + ok -> ok; + {error, Msg} -> plt_error(Msg) + end. + +to_file1(FileName, Plt, ModDeps, PltInfo) -> + to_file_custom_vsn(FileName, Plt, ModDeps, PltInfo, none, none). + +-spec to_file_custom_vsn( + file:filename(), + dialyzer_plt:plt(), + dialyzer_callgraph:mod_deps(), + #iplt_info{}, + none | string(), + none | [module_md5()]) -> 'ok'. + +to_file_custom_vsn(FileName, Plt, ModDeps, PltInfo, none, ImplMd5) -> + to_file_custom_vsn(FileName, Plt, ModDeps, PltInfo, ?VSN, ImplMd5); + +to_file_custom_vsn(FileName, Plt, ModDeps, PltInfo, Vsn, none) -> + to_file_custom_vsn(FileName, Plt, ModDeps, PltInfo, Vsn, compute_implementation_md5()); + +to_file_custom_vsn( + FileName, + #plt{info = ETSInfo, types = ETSTypes, contracts = ETSContracts, + callbacks = ETSCallbacks, exported_types = ETSExpTypes}, + NewModDeps, + #iplt_info{files = MD5, mod_deps = OldModDeps, warning_map=NewWarningMap, legal_warnings=LegalWarnings}, + Vsn, + ImplMd5) -> + CombinedModDeps = dict:merge(fun(_Key, OldVal, NewVal) -> + ordsets:union(OldVal, NewVal) + end, + OldModDeps, NewModDeps), + IncrementalData = + #incremental_data{mod_deps=CombinedModDeps, warning_map=NewWarningMap, legal_warnings=LegalWarnings}, + CallbacksList = + [Cb || + {_M, Cbs} <- dialyzer_utils:ets_tab2list(ETSCallbacks), + Cb <- Cbs], + Callbacks = maps:from_list(CallbacksList), + Info = maps:from_list(dialyzer_utils:ets_tab2list(ETSInfo)), + Types = dialyzer_utils:ets_tab2list(ETSTypes), + Contracts = maps:from_list(dialyzer_utils:ets_tab2list(ETSContracts)), + ExpTypes = sets:from_list([E || {E} <- dialyzer_utils:ets_tab2list(ETSExpTypes)], [{version, 2}]), + Record = #ifile_plt{version = Vsn, + module_md5_list = MD5, + info = term_to_binary(Info, [{compressed,9}]), + contracts = term_to_binary(Contracts, [{compressed,9}]), + callbacks = term_to_binary(Callbacks, [{compressed,9}]), + types = term_to_binary(Types, [{compressed,9}]), + exported_types = term_to_binary(ExpTypes, [{compressed,9}]), + incremental_data = term_to_binary(IncrementalData, [{compressed,9}]), + implementation_md5 = ImplMd5}, + Bin = term_to_binary(Record), + case file:write_file(FileName, Bin) of + ok -> ok; + {error, Reason} -> + Msg = io_lib:format("Could not write IPLT file ~ts: ~w\n", + [FileName, Reason]), + {error, Msg} + end. + +-spec merge_warnings(none | [raw_warning], [{module(), raw_warning()}], none | warning_map()) -> none | warning_map(). +merge_warnings(none, _, OldWarningMap) -> OldWarningMap; +merge_warnings(NewWarnings, UnknownWarnings, none) -> + convert_to_warning_map(NewWarnings, UnknownWarnings); +merge_warnings(NewWarnings, UnknownWarnings, OldWarningMap) -> + maps:merge(convert_to_warning_map(NewWarnings, UnknownWarnings), OldWarningMap). + +convert_to_warning_map(WarningList, UnknownWarnings) -> + Temp = lists:foldl( + fun({_, {_, _, MorMFA}, _} = Warn, Acc) -> + Update = fun(Old) -> [Warn|Old] end, + maps:update_with(get_module(MorMFA), Update, [Warn], Acc) + end, + #{}, + WarningList), + lists:foldl(fun({M, Warn}, Acc) -> + Update = fun(Old) -> [Warn|Old] end, + maps:update_with(M, Update, [Warn], Acc) + end, + Temp, + UnknownWarnings). + +get_module({M,_F,_A}) -> M; +get_module(M) -> M. + +-type md5_diff() :: [{'differ', atom()} | {'removed', atom()}]. +-type check_error() :: err_rsn() | {'no_file_to_remove', file:filename()}. + +-spec check_incremental_plt(file:filename(), #options{}, [file:filename()]) -> + {'ok', #iplt_info{}, module_file_path_lookup()} | + {'old_version', [module_md5()], module_file_path_lookup()} | + {'new_file', [module_md5()], module_file_path_lookup()} | + {'differ', [module_md5()], md5_diff(), dialyzer_callgraph:mod_deps(), warning_map(), module_file_path_lookup()} | + {'legal_warnings_changed', [module_md5()], module_file_path_lookup()} | + {'error', check_error()}. +check_incremental_plt(FileName, Opts, PltFiles) -> + Fun = fun() -> check_incremental_plt1(FileName, Opts, PltFiles) end, + subproc(Fun). + +check_incremental_plt1(FileName, Opts, PltFiles) -> + PltModulePathLookup = maps:from_list([ {beam_file_to_module(PltFile), PltFile} || PltFile <- PltFiles ]), + case get_record_from_file(FileName) of + {ok, #ifile_plt{module_md5_list = Md5} = Rec} -> + {RemoveModules, AddModules} = find_files_to_remove_and_add(Md5, maps:keys(PltModulePathLookup)), + IncrementalData = get_incremental_data(Rec), + PltLegalWarnings = IncrementalData#incremental_data.legal_warnings, + LegalWarnings = Opts#options.legal_warnings, + LegalWarningsMatch = PltLegalWarnings /= none andalso lists:usort(PltLegalWarnings) =:= lists:usort(LegalWarnings), + case check_version_and_compute_md5(Rec, RemoveModules, AddModules, PltModulePathLookup) of + ok when not LegalWarningsMatch -> + {legal_warnings_changed, Md5, PltModulePathLookup}; + {differ, NewMd5, _, _, _} when not LegalWarningsMatch -> + {legal_warnings_changed, NewMd5, PltModulePathLookup}; + ok -> + {ok, #iplt_info{files = Md5, + mod_deps = IncrementalData#incremental_data.mod_deps, + warning_map = IncrementalData#incremental_data.warning_map, + legal_warnings = IncrementalData#incremental_data.legal_warnings}, + PltModulePathLookup}; + {old_version, Md5} -> + {old_version, Md5}; + {differ, NewMd5, DiffMd5, ModDeps, _} -> + {differ, NewMd5, DiffMd5, ModDeps, IncrementalData#incremental_data.warning_map, PltModulePathLookup}; + {old_version, NewMd5} -> + {old_version, NewMd5, PltModulePathLookup}; + {error, Error} -> + {error, Error} + end; + {error, no_such_file} -> + {new_file, compute_md5_from_files(PltModulePathLookup), PltModulePathLookup}; + Error -> Error + end. + +find_files_to_remove_and_add(Md5, PltModules) -> + OldPltFiles = gb_sets:from_list([Name || {Name,_Md5Bin} <- Md5]), + NewPltFiles = gb_sets:from_list(PltModules), + {gb_sets:to_list(gb_sets:subtract(OldPltFiles, NewPltFiles)), + gb_sets:to_list(gb_sets:subtract(NewPltFiles, OldPltFiles))}. + +check_version_and_compute_md5(Rec, RemoveFiles, AddFiles, ModuleToPathLookup) -> + Md5 = Rec#ifile_plt.module_md5_list, + case check_version(Rec) of + ok -> + case compute_new_md5(Md5, RemoveFiles, AddFiles, ModuleToPathLookup) of + ok -> ok; + {differ, NewMd5, DiffMd5} -> + IncrementalData = get_incremental_data(Rec), + {differ, + NewMd5, + DiffMd5, + IncrementalData#incremental_data.mod_deps, + IncrementalData#incremental_data.warning_map}; + {error, _What} = Err -> Err + end; + error -> + case compute_new_md5(Md5, RemoveFiles, AddFiles, ModuleToPathLookup) of + ok -> {old_version, Md5}; + {differ, NewMd5, _DiffMd5} -> {old_version, NewMd5}; + {error, _What} = Err -> Err + end + end. + +compute_new_md5(Md5, [], [], ModuleToPathLookup) -> + compute_new_md5_1(Md5, [], ModuleToPathLookup); +compute_new_md5(Md5, RemoveFiles0, AddFiles0, ModuleToPathLookup) -> + %% Assume that files are first removed and then added. Files that + %% are both removed and added will be checked for consistency in the + %% normal way. + RemoveFiles = RemoveFiles0 -- AddFiles0, + AddFiles = AddFiles0 -- RemoveFiles0, + InitDiffList = init_diff_list(RemoveFiles, AddFiles), + case init_md5_list(Md5, RemoveFiles, AddFiles) of + {ok, NewMd5} -> compute_new_md5_1(NewMd5, InitDiffList, ModuleToPathLookup); + {error, _What} = Error -> Error + end. + +compute_new_md5_1(Entries, InitDiffs, ModuleToPathLookup) -> + Modules = [Module || {Module, _Md5} <- Entries], + ExistingHashes = [Md5 || {_Module, Md5} <- Entries], + Files = [maps:get(Module, ModuleToPathLookup) || Module <- Modules], + NewHashes = dialyzer_utils:p_map(fun compute_md5_from_file/1, Files), + Diffs = + lists:zipwith3( + fun (Module, BeforeHash, AfterHash) -> + case BeforeHash of + AfterHash -> + none; + _ -> + {differ, Module} + end + end, + Modules, + ExistingHashes, + NewHashes), + Diffs1 = InitDiffs ++ lists:filter(fun ({differ,_}) -> true; (none) -> false end, Diffs), + case Diffs1 of + [] -> + ok; + _ -> + ModuleHashes = lists:zip(Modules, NewHashes), + {differ, lists:keysort(1, ModuleHashes), Diffs1} + end. + +-spec implementation_module_paths() -> module_file_path_lookup(). +implementation_module_paths() -> + Dir = code:lib_dir(dialyzer), + Files1 = ["erl_bif_types.beam", "erl_types.beam"], + Files2 = [filename:join([Dir, "ebin", F]) || F <- Files1], + maps:from_list([{beam_file_to_module(File), File} || File <- Files2]). + +-spec compute_implementation_md5() -> [module_md5()]. + +compute_implementation_md5() -> + Modules = implementation_module_paths(), + compute_md5_from_files(Modules). + +-spec compute_md5_from_files(module_file_path_lookup()) -> [module_md5()]. + +compute_md5_from_files(ModuleToPathLookup) -> + {Modules,Files} = lists:unzip(maps:to_list(ModuleToPathLookup)), + Hashes = dialyzer_utils:p_map(fun compute_md5_from_file/1, Files), + lists:keysort(1, lists:zip(Modules, Hashes)). + +compute_md5_from_file(File) -> + case beam_lib:chunks(File, [debug_info]) of + {ok, {ModuleName, [{debug_info, {debug_info_v1, Backend, Data}}]}} -> + %% We cannot use beam_lib:md5 because it includes + %% non-portable or otherwise irrelvant data that would + %% cause the PLT to be invalidated needlessly too often + case Backend:debug_info(erlang_v1, ModuleName, Data, []) of + {ok, Code} -> + StabilisedCode = lists:filtermap(fun (Form) -> make_stable(ModuleName, Form) end, Code), + StabilisedCodeBin = erlang:term_to_binary(StabilisedCode), + erlang:md5(StabilisedCodeBin); + {error, Reason} -> + Msg = io_lib:format("Could not compute MD5 for .beam (debug_info error) - did you forget to set the debug_info compilation option? ~ts ~tw\n", [File, Reason]), + throw({dialyzer_error, Msg}) + end; + {ok, {_, [{debug_info, no_debug_info}]}} -> + Msg = io_lib:format("Could not compute MD5 for .beam (debug_info missing): ~ts\n", [File]), + throw({dialyzer_error, Msg}); + {error, beam_lib, {file_error, _, enoent}} -> + Msg = io_lib:format("File not found: ~ts\n", [File]), + plt_error(Msg); + {error, beam_lib, _} -> + Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), + plt_error(Msg) + end. + +%% Absolute paths in -file attributes make beam file hashes brittle, since the +%% same beam built elsewhere will contain a different absolute path, despite +%% being semantically identical. +%% +%% Here, we replace the full path with just the basename. This is very similar +%% to the effect of the +deterministic option, by by doing this rewriting here, +%% we gain some of those determinism benefits even when the build is not run +%% with +deterministic. +make_stable(_, {attribute, Anno, file, {SrcFilePath, Line}}) -> + {true, {attribute, Anno, file, {filename:basename(SrcFilePath), Line}}}; + +make_stable(_, Attr) -> + {true, Attr}. + +init_diff_list(RemoveFiles, AddFiles) -> + RemoveSet0 = sets:from_list([beam_file_to_module(F) || F <- RemoveFiles]), + AddSet0 = sets:from_list([beam_file_to_module(F) || F <- AddFiles]), + DiffSet = sets:intersection(AddSet0, RemoveSet0), + RemoveSet = sets:subtract(RemoveSet0, DiffSet), + %% Added files and diff files will appear as diff files from the md5 check. + [{removed, F} || F <- sets:to_list(RemoveSet)]. + +init_md5_list(Md5, RemoveFiles, AddFiles) -> + Mods = [{remove, beam_file_to_module(F)} || F <- RemoveFiles] ++ [{add, beam_file_to_module(F)} || F <- AddFiles], + DiffMods = lists:keysort(2, Mods), + Md5Sorted = lists:keysort(1, Md5), + init_md5_list_1(Md5Sorted, DiffMods, []). + +init_md5_list_1([{Mod, _Md5}|Md5Left], [{remove, Mod}|DiffLeft], Acc) -> + init_md5_list_1(Md5Left, DiffLeft, Acc); +init_md5_list_1([{Mod, _Md5} = Entry|Md5Left], [{add, Mod}|DiffLeft], Acc) -> + init_md5_list_1(Md5Left, DiffLeft, [Entry|Acc]); +init_md5_list_1([{Mod1, _Md5} = Entry|Md5Left] = Md5List, + [{Tag, Mod2}|DiffLeft] = DiffList, Acc) -> + case Mod1 < Mod2 of + true -> init_md5_list_1(Md5Left, DiffList, [Entry|Acc]); + false -> + %% Just an assert. + true = Mod1 > Mod2, + case Tag of + add -> init_md5_list_1(Md5List, DiffLeft, [{Mod2, <<>>}|Acc]); + remove -> {error, {no_file_to_remove, Mod2}} + end + end; +init_md5_list_1([], DiffList, Acc) -> + AddMods = [{M, <<>>} || {add, M} <- DiffList], + {ok, lists:reverse(Acc, AddMods)}; +init_md5_list_1(Md5List, [], Acc) -> + {ok, lists:reverse(Acc, Md5List)}. + + +subproc(Fun) -> + F = fun() -> + exit(try Fun() + catch throw:T -> + {thrown, T} + end) + end, + {Pid, Ref} = erlang:spawn_monitor(F), + receive {'DOWN', Ref, process, Pid, Return} -> + case Return of + {thrown, T} -> throw(T); + _ -> Return + end + end. + + +beam_file_to_module(Filename) -> + list_to_atom(filename:basename(Filename, ".beam")). + + +-spec plt_error(deep_string()) -> no_return(). + +plt_error(Msg) -> + throw({dialyzer_error, lists:flatten(Msg)}). + + +%%--------------------------------------------------------------------------- +%% Debug utilities. + +-spec pp_non_returning() -> 'ok'. + +pp_non_returning() -> + PltFile = get_default_iplt_filename(), + Plt = from_file(PltFile), + List = ets:tab2list(Plt#plt.info), + Unit = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, + erl_types:t_is_unit(Ret)], + None = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, + erl_types:t_is_none(Ret)], + io:format("=========================================\n"), + io:format("= Loops =\n"), + io:format("=========================================\n\n"), + lists:foreach(fun({{M, F, _}, Type}) -> + io:format("~w:~tw~ts.\n", + [M, F, dialyzer_utils:format_sig(Type)]) + end, lists:sort(Unit)), + io:format("\n"), + io:format("=========================================\n"), + io:format("= Errors =\n"), + io:format("=========================================\n\n"), + lists:foreach(fun({{M, F, _}, Type}) -> + io:format("~w:~w~s.\n", + [M, F, dialyzer_utils:format_sig(Type)]) + end, lists:sort(None)), + dialyzer_plt:delete(Plt). + +-spec pp_mod(atom()) -> 'ok'. + +pp_mod(Mod) when is_atom(Mod) -> + PltFile = get_default_iplt_filename(), + Plt = from_file(PltFile), + case dialyzer_plt:lookup_module(Plt, Mod) of + {value, List} -> + lists:foreach(fun({{_, F, _}, Ret, Args}) -> + T = erl_types:t_fun(Args, Ret), + S = dialyzer_utils:format_sig(T), + io:format("-spec ~tw~ts.\n", [F, S]) + end, lists:sort(List)); + none -> + io:format("dialyzer: Found no module named '~s' in the IPLT\n", [Mod]) + end, + dialyzer_plt:delete(Plt). diff --git a/lib/dialyzer/src/dialyzer_options.erl b/lib/dialyzer/src/dialyzer_options.erl index 5a56e0b0cb..bcac905fc1 100644 --- a/lib/dialyzer/src/dialyzer_options.erl +++ b/lib/dialyzer/src/dialyzer_options.erl @@ -47,10 +47,8 @@ build(Opts) -> ?WARN_BEHAVIOUR, ?WARN_UNDEFINED_CALLBACK], DefaultWarns1 = ordsets:from_list(DefaultWarns), - InitPlt = dialyzer_plt:get_default_plt(), DefaultOpts = #options{}, - DefaultOpts1 = DefaultOpts#options{legal_warnings = DefaultWarns1, - init_plts = [InitPlt]}, + DefaultOpts1 = DefaultOpts#options{legal_warnings = DefaultWarns1}, try Opts1 = preprocess_opts(Opts), Env = env_default_opts(), @@ -69,14 +67,45 @@ preprocess_opts([Opt|Opts]) -> [Opt|preprocess_opts(Opts)]. postprocess_opts(Opts = #options{}) -> - check_file_existence(Opts), - Opts1 = check_output_plt(Opts), - adapt_get_warnings(Opts1). + Opts1 = + case {Opts#options.init_plts, Opts#options.analysis_type} of + {[],incremental} -> Opts#options{init_plts=[dialyzer_iplt:get_default_iplt_filename()]}; + {[],_} -> Opts#options{init_plts=[dialyzer_cplt:get_default_cplt_filename()]}; + {[_|_],_} -> Opts + end, + check_file_existence(Opts1), + check_metrics_file_validity(Opts1), + check_module_lookup_file_validity(Opts1), + Opts2 = check_output_plt(Opts1), + check_init_plt_kind(Opts2), + Opts3 = manage_default_apps(Opts2), + adapt_get_warnings(Opts3). + +check_metrics_file_validity(#options{analysis_type = incremental, metrics_file = none}) -> + ok; +check_metrics_file_validity(#options{analysis_type = incremental, metrics_file = FileName}) -> + assert_filename(FileName); +check_metrics_file_validity(#options{analysis_type = _NotIncremental, metrics_file = none}) -> + ok; +check_metrics_file_validity(#options{analysis_type = _NotIncremental, metrics_file = FileName}) -> + bad_option("A metrics filename may only be given when in incremental mode", {metrics_file, FileName}). + +check_module_lookup_file_validity(#options{analysis_type = incremental, module_lookup_file = none}) -> + ok; +check_module_lookup_file_validity(#options{analysis_type = incremental, module_lookup_file = FileName}) -> + assert_filename(FileName); +check_module_lookup_file_validity(#options{analysis_type = _NotIncremental, module_lookup_file = none}) -> + ok; +check_module_lookup_file_validity(#options{analysis_type = _NotIncremental, module_lookup_file = FileName}) -> + bad_option("A module lookup filename may only be given when in incremental mode", {module_lookup_file, FileName}). check_file_existence(#options{analysis_type = plt_remove}) -> ok; -check_file_existence(#options{files = Files, files_rec = FilesRec}) -> +check_file_existence(#options{files = Files, files_rec = FilesRec, + warning_files = WarningFiles, warning_files_rec = WarningFilesRec}) -> assert_filenames_exist(Files), - assert_filenames_exist(FilesRec). + assert_filenames_exist(FilesRec), + assert_filenames_exist(WarningFiles), + assert_filenames_exist(WarningFilesRec). check_output_plt(Opts = #options{analysis_type = Mode, from = From, output_plt = OutPLT}) -> @@ -98,6 +127,71 @@ check_output_plt(Opts = #options{analysis_type = Mode, from = From, end end. +check_init_plt_kind(#options{analysis_type = incremental, init_plts = InitPlts}) -> + RunCheck = fun(FileName) -> + case dialyzer_plt:plt_kind(FileName) of + no_file -> ok; + iplt -> ok; + cplt -> + bad_option("Given file is a classic PLT file, " + "but in incremental mode, " + "an incremental PLT file is expected", + {init_plt_file, FileName}); + bad_file -> + bad_option("Given file is not a PLT file", {init_plt_file, FileName}) + end + end, + lists:foreach(RunCheck, InitPlts); +check_init_plt_kind(#options{analysis_type = _NotIncremental, init_plts = InitPlts}) -> + RunCheck = fun(FileName) -> + case dialyzer_plt:plt_kind(FileName) of + no_file -> ok; + cplt -> ok; + iplt -> + bad_option("Given file is an incremental PLT file, " + "but outside of incremental mode, " + "a classic PLT file is expected", + {init_plt_file, FileName}); + bad_file -> + bad_option("Given file is not a PLT file", {init_plt_file, FileName}) + end + end, + lists:foreach(RunCheck, InitPlts). + +%% If no apps are set explicitly, we fall back to config +manage_default_apps(Opts = #options{analysis_type = incremental, files = [], files_rec = [], warning_files = [], warning_files_rec = []}) -> + DefaultConfig = get_default_config_filename(), + case file:consult(DefaultConfig) of + {ok, [{incremental, {default_apps, DefaultApps}=Term}]} when + is_list(DefaultApps) -> + AppDirs = get_app_dirs(DefaultApps), + assert_filenames_form(Term, AppDirs), + Opts#options{files_rec = AppDirs}; + {ok, [{incremental, {default_apps, DefaultApps}=TermApps, + {default_warning_apps, DefaultWarningApps}=TermWarns}]} when + is_list(DefaultApps), is_list(DefaultWarningApps) -> + AppDirs = get_app_dirs(DefaultApps), + assert_filenames_form(TermApps, AppDirs), + WarningAppDirs = get_app_dirs(DefaultWarningApps), + assert_filenames_form(TermWarns, WarningAppDirs), + Opts#options{files_rec = AppDirs, warning_files_rec = WarningAppDirs}; + {ok, _Terms} -> + bad_option("Given Erlang terms could not be understood as Dialyzer config", DefaultConfig); + {error, Reason} -> + bad_option(file:format_error(Reason), DefaultConfig) + end; +manage_default_apps(Opts) -> + Opts. + +% Intended to work like dialyzer_iplt:get_default_iplt_filename() +get_default_config_filename() -> + case os:getenv("DIALYZER_CONFIG") of + false -> + CacheDir = filename:basedir(user_config, "erlang"), + filename:join(CacheDir, "dialyzer.config"); + UserSpecConfig -> UserSpecConfig + end. + adapt_get_warnings(Opts = #options{analysis_type = Mode, get_warnings = Warns}) -> %% Warnings are off by default in plt mode, and on by default in @@ -138,6 +232,18 @@ build_options([{OptionName, Value} = Term|Rest], Options) -> OldValues = Options#options.files_rec, assert_filenames_form(Term, Value), build_options(Rest, Options#options{files_rec = Value ++ OldValues}); + warning_apps -> + OldValues = Options#options.warning_files_rec, + AppDirs = get_app_dirs(Value), + assert_filenames_form(Term, AppDirs), + build_options(Rest, Options#options{warning_files_rec = AppDirs ++ OldValues}); + warning_files -> + assert_filenames_form(Term, Value), + build_options(Rest, Options#options{warning_files = Value}); + warning_files_rec -> + OldValues = Options#options.warning_files_rec, + assert_filenames_form(Term, Value), + build_options(Rest, Options#options{warning_files_rec = Value ++ OldValues}); analysis_type -> NewOptions = case Value of @@ -146,6 +252,7 @@ build_options([{OptionName, Value} = Term|Rest], Options) -> plt_build -> Options#options{analysis_type = Value}; plt_check -> Options#options{analysis_type = Value}; plt_remove -> Options#options{analysis_type = Value}; + incremental -> Options#options{analysis_type = Value}; dataflow -> bad_option("Analysis type is no longer supported", Term); old_style -> bad_option("Analysis type is no longer supported", Term); Other -> bad_option("Unknown analysis type", Other) @@ -164,7 +271,7 @@ build_options([{OptionName, Value} = Term|Rest], Options) -> get_warnings -> build_options(Rest, Options#options{get_warnings = Value}); plts -> - assert_filenames(Term, Value), + %assert_filenames(Term, Value), build_options(Rest, Options#options{init_plts = Value}); include_dirs -> assert_filenames(Term, Value), @@ -178,6 +285,12 @@ build_options([{OptionName, Value} = Term|Rest], Options) -> output_file -> assert_filename(Value), build_options(Rest, Options#options{output_file = Value}); + metrics_file -> + assert_filename(Value), + build_options(Rest, Options#options{metrics_file = Value}); + module_lookup_file -> + assert_filename(Value), + build_options(Rest, Options#options{module_lookup_file = Value}); output_format -> assert_output_format(Value), build_options(Rest, Options#options{output_format = Value}); @@ -199,6 +312,9 @@ build_options([{OptionName, Value} = Term|Rest], Options) -> callgraph_file -> assert_filename(Value), build_options(Rest, Options#options{callgraph_file = Value}); + mod_deps_file -> + assert_filename(Value), + build_options(Rest, Options#options{mod_deps_file = Value}); error_location -> assert_error_location(Value), build_options(Rest, Options#options{error_location = Value}); @@ -271,7 +387,7 @@ assert_filename_opt(fullpath) -> assert_filename_opt(Term) -> bad_option("Illegal value for filename_opt", Term). -assert_plt_op(#options{analysis_type = OldVal}, +assert_plt_op(#options{analysis_type = OldVal}, #options{analysis_type = NewVal}) -> case is_plt_mode(OldVal) andalso is_plt_mode(NewVal) of true -> bad_option("Options cannot be combined", [OldVal, NewVal]); @@ -282,6 +398,7 @@ is_plt_mode(plt_add) -> true; is_plt_mode(plt_build) -> true; is_plt_mode(plt_remove) -> true; is_plt_mode(plt_check) -> true; +is_plt_mode(incremental) -> true; is_plt_mode(succ_typings) -> false. assert_error_location(column) -> diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 409d1f65c5..2492d4e32e 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -16,21 +16,17 @@ %%% File : dialyzer_plt.erl %%% Author : Tobias Lindahl <tobiasl@it.uu.se> %%% Description : Interface to display information in the persistent -%%% lookup tables. +%%% lookup tables stored in memory, and other commonality +%%% between the various kinds of persisted PLT files. %%% %%% Created : 23 Jul 2004 by Tobias Lindahl <tobiasl@it.uu.se> %%%------------------------------------------------------------------- -module(dialyzer_plt). --export([check_plt/3, - compute_md5_from_files/1, - contains_mfa/2, +-export([contains_mfa/2, all_modules/1, delete_list/2, delete_module/2, - included_files/1, - from_file/1, - get_default_plt/0, get_module_types/2, get_exported_types/1, insert_list/2, @@ -43,32 +39,23 @@ lookup_contract/2, lookup_callbacks/2, lookup_module/2, - merge_plts/1, - merge_plts_or_report_conflicts/2, + merge_plts/1, new/0, - plt_and_info_from_file/1, get_specs/1, get_specs/4, - to_file/4, - delete/1, - get_all_types/1, - get_all_contracts/1, - get_all_callbacks/1 + delete/1, + get_all_types/1, + get_all_contracts/1, + get_all_callbacks/1, + plt_kind/1 ]). -%% Debug utilities --export([pp_non_returning/0, pp_mod/1]). - --export_type([plt/0, file_md5/0]). +-export_type([plt/0]). -include_lib("stdlib/include/ms_transform.hrl"). %%---------------------------------------------------------------------- --type mod_deps() :: dialyzer_callgraph:mod_deps(). - --type deep_string() :: string() | [deep_string()]. - %% The following are used for searching the PLT when using the GUI %% (e.g. in show or search PLT contents). The user might be searching %% with a partial specification, in which case the missing items @@ -78,30 +65,9 @@ %%---------------------------------------------------------------------- --record(plt, {info :: ets:tid(), %% {mfa() | integer(), ret_args_types()} - types :: ets:tid(), %% {module(), erl_types:type_table()} - contracts :: ets:tid(), %% {mfa(), #contract{}} - callbacks :: ets:tid(), %% {module(), - %% [{mfa(), - %% dialyzer_contracts:file_contract()}] - exported_types :: ets:tid() %% {module(), sets:set()} - }). - --opaque plt() :: #plt{}. - -include("dialyzer.hrl"). --type file_md5() :: {file:filename(), binary()}. - --record(file_plt, {version = "" :: string(), - file_md5_list = [] :: [file_md5()], - info = dict:new() :: dict:dict(), - contracts = dict:new() :: dict:dict(), - callbacks = dict:new() :: dict:dict(), - types = dict:new() :: dict:dict(), - exported_types = sets:new() :: sets:set(), - mod_deps :: mod_deps(), - implementation_md5 = [] :: [file_md5()]}). +-type plt() :: #plt{}. %%---------------------------------------------------------------------- @@ -232,123 +198,6 @@ all_modules(#plt{info = Info, contracts = Cs}) -> contains_mfa(#plt{info = Info, contracts = Contracts}, MFA) -> ets:member(Info, MFA) orelse ets:member(Contracts, MFA). --spec get_default_plt() -> file:filename(). - -get_default_plt() -> - case os:getenv("DIALYZER_PLT") of - false -> - CacheDir = filename:basedir(user_cache, "erlang"), - filename:join(CacheDir, ".dialyzer_plt"); - UserSpecPlt -> UserSpecPlt - end. - --spec plt_and_info_from_file(file:filename()) -> {plt(), #plt_info{}}. - -plt_and_info_from_file(FileName) -> - from_file(FileName, true). - --spec from_file(file:filename()) -> plt(). - -from_file(FileName) -> - from_file(FileName, false). - -from_file(FileName, ReturnInfo) -> - Plt = new(), - Fun = fun() -> from_file1(Plt, FileName, ReturnInfo) end, - case subproc(Fun) of - {ok, Return} -> - Return; - {error, Msg} -> - delete(Plt), - plt_error(Msg) - end. - -from_file1(Plt, FileName, ReturnInfo) -> - case get_record_from_file(FileName) of - {ok, Rec} -> - case check_version(Rec) of - error -> - Msg = io_lib:format("Old PLT file ~ts\n", [FileName]), - {error, Msg}; - ok -> - #file_plt{info = FileInfo, - contracts = FileContracts, - callbacks = FileCallbacks, - types = FileTypes, - exported_types = FileExpTypes} = Rec, - Types = [{Mod, maps:from_list(dict:to_list(Types))} || - {Mod, Types} <- dict:to_list(FileTypes)], - CallbacksList = dict:to_list(FileCallbacks), - CallbacksByModule = - [{M, [Cb || {{M1,_,_},_} = Cb <- CallbacksList, M1 =:= M]} || - M <- lists:usort([M || {{M,_,_},_} <- CallbacksList])], - #plt{info = ETSInfo, - types = ETSTypes, - contracts = ETSContracts, - callbacks = ETSCallbacks, - exported_types = ETSExpTypes} = Plt, - [true, true, true] = - [ets:insert(ETS, Data) || - {ETS, Data} <- [{ETSInfo, dict:to_list(FileInfo)}, - {ETSTypes, Types}, - {ETSContracts, dict:to_list(FileContracts)}]], - true = ets:insert(ETSCallbacks, CallbacksByModule), - true = ets:insert(ETSExpTypes, [{ET} || - ET <- sets:to_list(FileExpTypes)]), - case ReturnInfo of - false -> {ok, Plt}; - true -> - PltInfo = #plt_info{files = Rec#file_plt.file_md5_list, - mod_deps = Rec#file_plt.mod_deps}, - {ok, {Plt, PltInfo}} - end - end; - {error, Reason} -> - Msg = io_lib:format("Could not read PLT file ~ts: ~p\n", - [FileName, Reason]), - {error, Msg} - end. - --type err_rsn() :: 'not_valid' | 'no_such_file' | 'read_error'. - --spec included_files(file:filename()) -> {'ok', [file:filename()]} - | {'error', err_rsn()}. - -included_files(FileName) -> - Fun = fun() -> included_files1(FileName) end, - subproc(Fun). - -included_files1(FileName) -> - case get_record_from_file(FileName) of - {ok, #file_plt{file_md5_list = Md5}} -> - {ok, [File || {File, _} <- Md5]}; - {error, _What} = Error -> - Error - end. - -check_version(#file_plt{version = ?VSN, implementation_md5 = ImplMd5}) -> - case compute_new_md5(ImplMd5, [], []) of - ok -> ok; - {differ, _, _} -> error; - {error, _} -> error - end; -check_version(#file_plt{}) -> error. - -get_record_from_file(FileName) -> - case file:read_file(FileName) of - {ok, Bin} -> - try binary_to_term(Bin) of - #file_plt{} = FilePLT -> {ok, FilePLT}; - _ -> {error, not_valid} - catch - _:_ -> {error, not_valid} - end; - {error, enoent} -> - {error, no_such_file}; - {error, _} -> - {error, read_error} - end. - -spec merge_plts([plt()]) -> plt(). %% One of the PLTs of the list is augmented with the contents of the @@ -364,23 +213,6 @@ merge_plts(List) -> callbacks = table_merge(CallbacksList) }. --spec merge_disj_plts([plt()]) -> plt(). - -%% One of the PLTs of the list is augmented with the contents of the -%% other PLTs, and returned. The other PLTs are deleted. -%% -%% The keys are compared when checking for disjointness. Sometimes the -%% key is a module(), sometimes an mfa(). It boils down to checking if -%% any module occurs more than once. -merge_disj_plts(List) -> - {InfoList, TypesList, ExpTypesList, ContractsList, CallbacksList} = - group_fields(List), - #plt{info = table_disj_merge(InfoList), - types = table_disj_merge(TypesList), - exported_types = sets_disj_merge(ExpTypesList), - contracts = table_disj_merge(ContractsList), - callbacks = table_disj_merge(CallbacksList) - }. group_fields(List) -> InfoList = [Info || #plt{info = Info} <- List], @@ -390,199 +222,6 @@ group_fields(List) -> CallbacksList = [Callbacks || #plt{callbacks = Callbacks} <- List], {InfoList, TypesList, ExpTypesList, ContractsList, CallbacksList}. --spec merge_plts_or_report_conflicts([file:filename()], [plt()]) -> plt(). - -merge_plts_or_report_conflicts(PltFiles, Plts) -> - try - merge_disj_plts(Plts) - catch throw:{dialyzer_error, not_disjoint_plts} -> - IncFiles = lists:append([begin {ok, Fs} = included_files(F), Fs end - || F <- PltFiles]), - ConfFiles = find_duplicates(IncFiles), - Msg = io_lib:format("Could not merge PLTs since they are not disjoint\n" - "The following files are included in more than one " - "PLTs:\n~tp\n", [ConfFiles]), - plt_error(Msg) - end. - -find_duplicates(List) -> - ModList = [filename:basename(E) || E <- List], - SortedList = lists:usort(ModList), - lists:usort(ModList -- SortedList). - --spec to_file(file:filename(), plt(), mod_deps(), #plt_info{}) -> 'ok'. - -%% Write the PLT to file, and delete the PLT. -to_file(FileName, Plt, ModDeps, MD5_OldModDeps) -> - Fun = fun() -> to_file1(FileName, Plt, ModDeps, MD5_OldModDeps) end, - Return = subproc(Fun), - delete(Plt), - case Return of - ok -> ok; - {error, Msg} -> plt_error(Msg) - end. - -to_file1(FileName, - #plt{info = ETSInfo, types = ETSTypes, contracts = ETSContracts, - callbacks = ETSCallbacks, exported_types = ETSExpTypes}, - ModDeps, #plt_info{files = MD5, mod_deps = OldModDeps}) -> - NewModDeps = dict:merge(fun(_Key, OldVal, NewVal) -> - ordsets:union(OldVal, NewVal) - end, - OldModDeps, ModDeps), - ImplMd5 = compute_implementation_md5(), - CallbacksList = - [Cb || - {_M, Cbs} <- tab2list(ETSCallbacks), - Cb <- Cbs], - Callbacks = dict:from_list(CallbacksList), - Info = dict:from_list(tab2list(ETSInfo)), - Types = tab2list(ETSTypes), - Contracts = dict:from_list(tab2list(ETSContracts)), - ExpTypes = sets:from_list([E || {E} <- tab2list(ETSExpTypes)]), - FileTypes = dict:from_list([{Mod, dict:from_list(maps:to_list(MTypes))} || - {Mod, MTypes} <- Types]), - Record = #file_plt{version = ?VSN, - file_md5_list = MD5, - info = Info, - contracts = Contracts, - callbacks = Callbacks, - types = FileTypes, - exported_types = ExpTypes, - mod_deps = NewModDeps, - implementation_md5 = ImplMd5}, - Bin = term_to_binary(Record, [compressed]), - case file:write_file(FileName, Bin) of - ok -> ok; - {error, Reason} -> - Msg = io_lib:format("Could not write PLT file ~ts: ~w\n", - [FileName, Reason]), - {error, Msg} - end. - --type md5_diff() :: [{'differ', atom()} | {'removed', atom()}]. --type check_error() :: err_rsn() | {'no_file_to_remove', file:filename()}. - --spec check_plt(file:filename(), [file:filename()], [file:filename()]) -> - 'ok' - | {'error', check_error()} - | {'differ', [file_md5()], md5_diff(), mod_deps()} - | {'old_version', [file_md5()]}. - -check_plt(FileName, RemoveFiles, AddFiles) -> - Fun = fun() -> check_plt1(FileName, RemoveFiles, AddFiles) end, - subproc(Fun). - -check_plt1(FileName, RemoveFiles, AddFiles) -> - case get_record_from_file(FileName) of - {ok, #file_plt{file_md5_list = Md5, mod_deps = ModDeps} = Rec} -> - case check_version(Rec) of - ok -> - case compute_new_md5(Md5, RemoveFiles, AddFiles) of - ok -> ok; - {differ, NewMd5, DiffMd5} -> {differ, NewMd5, DiffMd5, ModDeps}; - {error, _What} = Err -> Err - end; - error -> - case compute_new_md5(Md5, RemoveFiles, AddFiles) of - ok -> {old_version, Md5}; - {differ, NewMd5, _DiffMd5} -> {old_version, NewMd5}; - {error, _What} = Err -> Err - end - end; - Error -> Error - end. - -compute_new_md5(Md5, [], []) -> - compute_new_md5_1(Md5, [], []); -compute_new_md5(Md5, RemoveFiles0, AddFiles0) -> - %% Assume that files are first removed and then added. Files that - %% are both removed and added will be checked for consistency in the - %% normal way. If they have moved, we assume that they differ. - RemoveFiles = RemoveFiles0 -- AddFiles0, - AddFiles = AddFiles0 -- RemoveFiles0, - InitDiffList = init_diff_list(RemoveFiles, AddFiles), - case init_md5_list(Md5, RemoveFiles, AddFiles) of - {ok, NewMd5} -> compute_new_md5_1(NewMd5, [], InitDiffList); - {error, _What} = Error -> Error - end. - -compute_new_md5_1([{File, Md5} = Entry|Entries], NewList, Diff) -> - case compute_md5_from_file(File) of - Md5 -> compute_new_md5_1(Entries, [Entry|NewList], Diff); - NewMd5 -> - ModName = beam_file_to_module(File), - compute_new_md5_1(Entries, [{File, NewMd5}|NewList], [{differ, ModName}|Diff]) - end; -compute_new_md5_1([], _NewList, []) -> - ok; -compute_new_md5_1([], NewList, Diff) -> - {differ, lists:keysort(1, NewList), Diff}. - --spec compute_implementation_md5() -> [file_md5()]. - -compute_implementation_md5() -> - Dir = code:lib_dir(dialyzer), - Files1 = ["erl_bif_types.beam", "erl_types.beam"], - Files2 = [filename:join([Dir, "ebin", F]) || F <- Files1], - compute_md5_from_files(Files2). - --spec compute_md5_from_files([file:filename()]) -> [file_md5()]. - -compute_md5_from_files(Files) -> - lists:keysort(1, [{F, compute_md5_from_file(F)} || F <- Files]). - -compute_md5_from_file(File) -> - case beam_lib:all_chunks(File) of - {ok, _, Chunks} -> - %% We cannot use beam_lib:md5 because it does not consider - %% the debug_info chunk, where typespecs are likely stored. - %% So we consider almost all chunks except the useless ones. - Filtered = [[ID, Chunk] || {ID, Chunk} <- Chunks, ID =/= "CInf", ID =/= "Docs"], - erlang:md5(lists:sort(Filtered)); - {error, beam_lib, {file_error, _, enoent}} -> - Msg = io_lib:format("File not found: ~ts\n", [File]), - plt_error(Msg); - {error, beam_lib, _} -> - Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), - plt_error(Msg) - end. - -init_diff_list(RemoveFiles, AddFiles) -> - RemoveSet0 = sets:from_list([beam_file_to_module(F) || F <- RemoveFiles]), - AddSet0 = sets:from_list([beam_file_to_module(F) || F <- AddFiles]), - DiffSet = sets:intersection(AddSet0, RemoveSet0), - RemoveSet = sets:subtract(RemoveSet0, DiffSet), - %% Added files and diff files will appear as diff files from the md5 check. - [{removed, F} || F <- sets:to_list(RemoveSet)]. - -init_md5_list(Md5, RemoveFiles, AddFiles) -> - Files = [{remove, F} || F <- RemoveFiles] ++ [{add, F} || F <- AddFiles], - DiffFiles = lists:keysort(2, Files), - Md5Sorted = lists:keysort(1, Md5), - init_md5_list_1(Md5Sorted, DiffFiles, []). - -init_md5_list_1([{File, _Md5}|Md5Left], [{remove, File}|DiffLeft], Acc) -> - init_md5_list_1(Md5Left, DiffLeft, Acc); -init_md5_list_1([{File, _Md5} = Entry|Md5Left], [{add, File}|DiffLeft], Acc) -> - init_md5_list_1(Md5Left, DiffLeft, [Entry|Acc]); -init_md5_list_1([{File1, _Md5} = Entry|Md5Left] = Md5List, - [{Tag, File2}|DiffLeft] = DiffList, Acc) -> - case File1 < File2 of - true -> init_md5_list_1(Md5Left, DiffList, [Entry|Acc]); - false -> - %% Just an assert. - true = File1 > File2, - case Tag of - add -> init_md5_list_1(Md5List, DiffLeft, [{File2, <<>>}|Acc]); - remove -> {error, {no_file_to_remove, File2}} - end - end; -init_md5_list_1([], DiffList, Acc) -> - AddFiles = [{F, <<>>} || {add, F} <- DiffList], - {ok, lists:reverse(Acc, AddFiles)}; -init_md5_list_1(Md5List, [], Acc) -> - {ok, lists:reverse(Acc, Md5List)}. -spec delete(plt()) -> 'ok'. @@ -598,24 +237,6 @@ delete(#plt{info = ETSInfo, true = ets:delete(ETSExpTypes), ok. -tab2list(Tab) -> - dialyzer_utils:ets_tab2list(Tab). - -subproc(Fun) -> - F = fun() -> - exit(try Fun() - catch throw:T -> - {thrown, T} - end) - end, - {Pid, Ref} = erlang:spawn_monitor(F), - receive {'DOWN', Ref, process, Pid, Return} -> - case Return of - {thrown, T} -> throw(T); - _ -> Return - end - end. - %%--------------------------------------------------------------------------- %% Edoc @@ -627,9 +248,6 @@ get_specs(#plt{info = Info}) -> {{_,_,_} = MFA, Val} <- table_to_list(Info)]), lists:flatten(create_specs(L, [])). -beam_file_to_module(Filename) -> - list_to_atom(filename:basename(Filename, ".beam")). - -spec get_specs(plt(), atom(), atom(), arity_patt()) -> 'none' | string(). get_specs(#plt{info = Info}, M, F, A) when is_atom(M), is_atom(F) -> @@ -663,10 +281,6 @@ expand_args([ArgType|Left]) -> end ++ ","|expand_args(Left)]. --spec plt_error(deep_string()) -> no_return(). - -plt_error(Msg) -> - throw({dialyzer_error, lists:flatten(Msg)}). %%--------------------------------------------------------------------------- %% Ets table @@ -735,19 +349,6 @@ table_merge([Plt|Plts], Acc) -> NewAcc = merge_tables(Plt, Acc), table_merge(Plts, NewAcc). -table_disj_merge([H|T]) -> - table_disj_merge(T, H). - -table_disj_merge([], Acc) -> - Acc; -table_disj_merge([Plt|Plts], Acc) -> - case table_is_disjoint(Plt, Acc) of - true -> - NewAcc = merge_tables(Plt, Acc), - table_disj_merge(Plts, NewAcc); - false -> throw({dialyzer_error, not_disjoint_plts}) - end. - sets_merge([H|T]) -> sets_merge(T, H). @@ -757,31 +358,6 @@ sets_merge([Plt|Plts], Acc) -> NewAcc = merge_tables(Plt, Acc), sets_merge(Plts, NewAcc). -sets_disj_merge([H|T]) -> - sets_disj_merge(T, H). - -sets_disj_merge([], Acc) -> - Acc; -sets_disj_merge([Plt|Plts], Acc) -> - case table_is_disjoint(Plt, Acc) of - true -> - NewAcc = merge_tables(Plt, Acc), - sets_disj_merge(Plts, NewAcc); - false -> throw({dialyzer_error, not_disjoint_plts}) - end. - -table_is_disjoint(T1, T2) -> - tab_is_disj(ets:first(T1), T1, T2). - -tab_is_disj('$end_of_table', _T1, _T2) -> - true; -tab_is_disj(K1, T1, T2) -> - case ets:member(T2, K1) of - false -> - tab_is_disj(ets:next(T1, K1), T1, T2); - true -> - false - end. merge_tables(T1, T2) -> tab_merge(ets:first(T1), T1, T2). @@ -801,53 +377,6 @@ tab_merge(K1, T1, T2) -> true = ets:insert(T2, Vs), tab_merge(NextK1, T1, T2). -%%--------------------------------------------------------------------------- -%% Debug utilities. - --spec pp_non_returning() -> 'ok'. - -pp_non_returning() -> - PltFile = get_default_plt(), - Plt = from_file(PltFile), - List = table_to_list(Plt#plt.info), - Unit = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, - erl_types:t_is_unit(Ret)], - None = [{MFA, erl_types:t_fun(Args, Ret)} || {MFA, {Ret, Args}} <- List, - erl_types:t_is_none(Ret)], - io:format("=========================================\n"), - io:format("= Loops =\n"), - io:format("=========================================\n\n"), - lists:foreach(fun({{M, F, _}, Type}) -> - io:format("~w:~tw~ts.\n", - [M, F, dialyzer_utils:format_sig(Type)]) - end, lists:sort(Unit)), - io:format("\n"), - io:format("=========================================\n"), - io:format("= Errors =\n"), - io:format("=========================================\n\n"), - lists:foreach(fun({{M, F, _}, Type}) -> - io:format("~w:~w~s.\n", - [M, F, dialyzer_utils:format_sig(Type)]) - end, lists:sort(None)), - delete(Plt). - --spec pp_mod(atom()) -> 'ok'. - -pp_mod(Mod) when is_atom(Mod) -> - PltFile = get_default_plt(), - Plt = from_file(PltFile), - case lookup_module(Plt, Mod) of - {value, List} -> - lists:foreach(fun({{_, F, _}, Ret, Args}) -> - T = erl_types:t_fun(Args, Ret), - S = dialyzer_utils:format_sig(T), - io:format("-spec ~tw~ts.\n", [F, S]) - end, lists:sort(List)); - none -> - io:format("dialyzer: Found no module named '~s' in the PLT\n", [Mod]) - end, - delete(Plt). - %% Returns all contracts stored in the PLT -spec get_all_contracts(plt()) -> #{mfa() => #contract{}}. @@ -868,3 +397,18 @@ get_all_callbacks(#plt{callbacks = ETSCallbacks}) -> get_all_types(#plt{types = ETSTypes}) -> Types = ets:tab2list(ETSTypes), maps:from_list(Types). + +-spec plt_kind(file:filename()) -> 'iplt' | 'cplt' | 'bad_file' | 'no_file'. +plt_kind(FileName) -> + case filelib:is_regular(FileName) of + true -> + case dialyzer_iplt:is_iplt(FileName) of + true -> iplt; + false -> + case dialyzer_cplt:is_cplt(FileName) of + true -> cplt; + false -> bad_file + end + end; + false -> no_file + end. diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl index 248f78a43f..18b402f54f 100644 --- a/lib/dialyzer/src/dialyzer_succ_typings.erl +++ b/lib/dialyzer/src/dialyzer_succ_typings.erl @@ -212,17 +212,17 @@ add_to_result(Labels, Result, {_Codeserver, Callgraph, _Plt, _Solver}) -> get_success_typings(Callgraph, Plt, Codeserver, TimingServer, Solvers) -> %% Condense the call graph to its strongly connected components (SCCs). - {SCCs, Callgraph1} = + {LabelledSCCs, Callgraph1} = ?timing(TimingServer, "order", dialyzer_callgraph:finalize(Callgraph)), State = #st{callgraph = Callgraph1, plt = Plt, codeserver = Codeserver, timing_server = TimingServer, solvers = Solvers}, - get_refined_success_typings(SCCs, State). + get_refined_success_typings(LabelledSCCs, State). -get_refined_success_typings(SCCs, #st{callgraph = Callgraph, +get_refined_success_typings(LabelledSCCs, #st{callgraph = Callgraph, timing_server = TimingServer} = State) -> %% Find the success types for the SCCs. - case find_succ_typings(SCCs, State) of + case find_succ_typings(LabelledSCCs, State) of [] -> %% No new type information was discovered. We are done. State; @@ -233,7 +233,6 @@ get_refined_success_typings(SCCs, #st{callgraph = Callgraph, ?timing(TimingServer, "order", _C1, dialyzer_callgraph:module_postorder_from_funs(NotFixpoint1, Callgraph)), - ?debug("Module postorder: ~p\n", [Modules]), ModState = State#st{callgraph = ModCallgraph}, case refine_succ_typings(ModulePostorder, ModState) of @@ -242,20 +241,20 @@ get_refined_success_typings(SCCs, #st{callgraph = Callgraph, ModState; NotFixpoint2 -> %% Need to reset the callgraph before repeating. - {NewSCCs, Callgraph2} = + {NewLabelledSCCs, Callgraph2} = ?timing(TimingServer, "order", _C2, dialyzer_callgraph:reset_from_funs(NotFixpoint2, ModCallgraph)), NewState = ModState#st{callgraph = Callgraph2}, - get_refined_success_typings(NewSCCs, NewState) + get_refined_success_typings(NewLabelledSCCs, NewState) end end. -find_succ_typings(SCCs, State) -> +find_succ_typings(LabelledSCCs, State) -> {Init, Timing} = init_pass_data(State), Updated = ?timing(Timing, "typesig", - dialyzer_coordinator:parallel_job(typesig, SCCs, Init, Timing)), + dialyzer_coordinator:parallel_job(typesig, LabelledSCCs, Init, Timing)), ?debug("==================== Typesig done ====================\n\n", []), Updated. diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index 082dbdd521..d49a07dfd0 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -21,6 +21,7 @@ %%%------------------------------------------------------------------- -module(dialyzer_typesig). +-feature(maybe_expr, enable). -export([analyze_scc/7]). -export([get_safe_underapprox/2]). @@ -471,21 +472,19 @@ traverse(Tree, DefinedVars, State) -> {NewEvars, TmpState} = lists:mapfoldl(Fun, State1, EVars), {TmpState, t_tuple(NewEvars)} end, - case Elements of - [Tag|Fields] -> - case cerl:is_c_atom(Tag) andalso is_literal_record(Tree) of - true -> - %% Check if a record is constructed. - Arity = length(Fields), - case lookup_record(State2, cerl:atom_val(Tag), Arity) of - {error, State3} -> {State3, TupleType}; - {ok, RecType, State3} -> - State4 = state__store_conj(TupleType, sub, RecType, State3), - {State4, TupleType} - end; - false -> {State2, TupleType} - end; - [] -> {State2, TupleType} + maybe + [Tag|Fields] ?= Elements, + true ?= cerl:is_c_atom(Tag) andalso is_literal_record(Tree), + %% Check if a record is constructed. + Arity = length(Fields), + case lookup_record(State2, cerl:atom_val(Tag), Arity) of + {error, State3} -> {State3, TupleType}; + {ok, RecType, State3} -> + State4 = state__store_conj(TupleType, sub, RecType, State3), + {State4, TupleType} + end + else + _ -> {State2, TupleType} end; map -> Entries = cerl:map_es(Tree), diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 292c01f7aa..c0bbf49625 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -45,7 +45,9 @@ ets_tab2list/1, ets_move/2, parallelism/0, - family/1 + family/1, + p_foreach/2, + p_map/2 ]). %% For dialyzer_worker. @@ -550,14 +552,53 @@ get_spec_info([], SpecMap, CallbackMap, {ok, SpecMap, CallbackMap}. core_to_attr_tuples(Core) -> - [{cerl:concrete(Key), get_core_location(cerl:get_ann(Key)), cerl:concrete(Value)} || - {Key, Value} <- cerl:module_attrs(Core)]. + As = [{cerl:concrete(Key), get_core_location(cerl:get_ann(Key)), cerl:concrete(Value)} || + {Key, Value} <- cerl:module_attrs(Core)], + case cerl:concrete(cerl:module_name(Core)) of + erlang -> + As; + _ -> + %% Starting from Erlang/OTP 26, locally defining a type having + %% the same name as a built-in type is allowed. Change the tag + %% from `type` to `user_type` for all such redefinitions. + massage_forms(As, sets:new([{version, 2}])) + end. get_core_location([L | _As]) when is_integer(L) -> L; get_core_location([{L, C} | _As]) when is_integer(L), is_integer(C) -> {L, C}; get_core_location([_ | As]) -> get_core_location(As); get_core_location([]) -> undefined. +massage_forms([{type, Loc, [{Name, Type0, Args}]} | T], Defs0) -> + Type = massage_type(Type0, Defs0), + Defs = sets:add_element({Name, length(Args)}, Defs0), + [{type, Loc, [{Name, Type, Args}]} | massage_forms(T, Defs)]; +massage_forms([{spec, Loc, [{Name, [Type0]}]} | T], Defs) -> + Type = massage_type(Type0, Defs), + [{spec, Loc, [{Name, [Type]}]} | massage_forms(T, Defs)]; +massage_forms([H | T], Defs) -> + [H | massage_forms(T, Defs)]; +massage_forms([], _Defs) -> + []. + +massage_type({type, Loc, Name, Args0}, Defs) when is_list(Args0) -> + case sets:is_element({Name, length(Args0)}, Defs) of + true -> + %% This name for a built-in type has been overriden locally + %% with a new definition. + {user_type, Loc, Name, Args0}; + false -> + Args = massage_type_list(Args0, Defs), + {type, Loc, Name, Args} + end; +massage_type(Type, _Defs) -> + Type. + +massage_type_list([H|T], Defs) -> + [massage_type(H, Defs) | massage_type_list(T, Defs)]; +massage_type_list([], _Defs) -> + []. + -spec get_fun_meta_info(module(), cerl:c_module(), [dial_warn_tag()]) -> dialyzer_codeserver:fun_meta_info() | {'error', string()}. @@ -1146,3 +1187,79 @@ parallelism() -> family(L) -> sofs:to_external(sofs:rel2fam(sofs:relation(L))). + +-spec p_foreach(fun((X) -> any()), [X]) -> ok. +p_foreach(Fun, List) -> + N = dialyzer_utils:parallelism(), + Ref = make_ref(), + start(Fun, List, Ref, N, gb_sets:new()). + +start(Fun, [Arg|Rest], Ref, N, Outstanding) when N > 0 -> + Self = self(), + Pid = spawn_link( + fun() -> + try Fun(Arg) of + _Val -> Self ! {done, Ref, self()} + catch + throw:Throw -> Self ! {throw, Throw, Ref, self()} + end + end), + start(Fun, Rest, Ref, N-1, gb_sets:add_element(Pid, Outstanding)); +start(Fun, Args, Ref, N, Outstanding) when N >= 0 -> + case {gb_sets:is_empty(Outstanding), Args} of + {true, []} -> ok; + {true, Args} -> start(Fun, Args, Ref, 1, Outstanding); + {false, _} -> + receive + {done, Ref, Pid} -> + start(Fun, Args, Ref, N+1, gb_sets:delete(Pid, Outstanding)); + {throw, Throw, Ref, Pid} -> + clean_up(Throw, Ref, gb_sets:delete(Pid, Outstanding)) + end + end. + +-spec p_map(fun((X) -> Y), [X]) -> [Y]. +p_map(Fun, List) -> + Parent = self(), + Batches = batch(List, dialyzer_utils:parallelism()), + BatchJobs = + [spawn_link( + fun() -> + try + Result = lists:map(Fun,Batch), + Parent ! {done, self(), Result} + catch + throw:Throw -> Parent ! {throw, self(), Throw} + end + end) + || Batch <- Batches], + lists:append([ + receive + {done, Pid, BatchResult} -> BatchResult; + {throw, Pid, Throw} -> throw(Throw) + end + || Pid <- BatchJobs]). + +-spec batch([X], non_neg_integer()) -> [[X]]. +batch(List, BatchSize) -> + batch(BatchSize, 0, List, []). +batch(_, _, [], Acc) -> + [lists:reverse(Acc)]; +batch(BatchSize, BatchSize, List, Acc) -> + [lists:reverse(Acc) | batch(BatchSize, 0, List, [])]; +batch(BatchSize, PartialBatchSize, [H|T], Acc) -> + batch(BatchSize, PartialBatchSize+1, T, [H|Acc]). + + +clean_up(ThrowVal, Ref, Outstanding) -> + case gb_sets:is_empty(Outstanding) of + true -> + throw(ThrowVal); + false -> + receive + {done, Ref, Pid} -> + clean_up(ThrowVal, Ref, gb_sets:delete(Pid, Outstanding)); + {throw, _Throw, Ref, Pid} -> + clean_up(ThrowVal, Ref, gb_sets:delete(Pid, Outstanding)) + end + end. diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl index c9ceb75a40..db33ea0e13 100644 --- a/lib/dialyzer/src/dialyzer_worker.erl +++ b/lib/dialyzer/src/dialyzer_worker.erl @@ -56,9 +56,9 @@ launch(Mode, Job, InitData, Coordinator) -> %%-------------------------------------------------------------------- %% Local functions. -init(#state{job = SCC, mode = Mode, init_data = InitData} = State) +init(#state{job = Job, mode = Mode, init_data = InitData} = State) when Mode =:= 'typesig'; Mode =:= 'dataflow' -> - wait_for_success_typings(SCC, InitData, State), + wait_for_success_typings(Mode, Job, InitData, State), run(State); init(#state{mode = Mode} = State) when Mode =:= 'compile'; Mode =:= 'warnings'; @@ -73,6 +73,7 @@ run(#state{coordinator = Coordinator, job = Job} = State) -> run_job(#state{mode = Mode, job = Job, init_data = InitData} = State) -> ?debug("~w: ~p: ~p\n", [self(), Mode, Job]), + StartableJob = dialyzer_coordinator:get_job_input(Mode, Job), case Mode of compile -> case start_compilation(State) of @@ -82,16 +83,18 @@ run_job(#state{mode = Mode, job = Job, init_data = InitData} = State) -> {error, _Reason} = Error -> Error end; - typesig -> - dialyzer_succ_typings:find_succ_types_for_scc(Job, InitData); - dataflow -> - dialyzer_succ_typings:refine_one_module(Job, InitData); - contract_remote_types -> - dialyzer_contracts:process_contract_remote_types_module(Job, InitData); - record_remote_types -> - dialyzer_utils:process_record_remote_types_module(Job, InitData); - warnings -> - dialyzer_succ_typings:collect_warnings(Job, InitData) + _ -> + StartableJob = dialyzer_coordinator:get_job_input(Mode, Job), + case Mode of + typesig -> dialyzer_succ_typings:find_succ_types_for_scc(StartableJob, InitData); + dataflow -> dialyzer_succ_typings:refine_one_module(StartableJob, InitData); + contract_remote_types -> + dialyzer_contracts:process_contract_remote_types_module(StartableJob, InitData); + record_remote_types -> + dialyzer_utils:process_record_remote_types_module(StartableJob, InitData); + warnings -> + dialyzer_succ_typings:collect_warnings(StartableJob, InitData) + end end. start_compilation(#state{job = Job, init_data = InitData}) -> @@ -105,6 +108,8 @@ continue_compilation(Label, Data) -> %% Wait for the results of success typings of modules or SCCs that we %% depend on. ('typesig' or 'dataflow' mode) -wait_for_success_typings(SCC, InitData, #state{coordinator = Coordinator}) -> - DependsOnSCCs = dialyzer_succ_typings:find_depends_on(SCC, InitData), - dialyzer_coordinator:wait_for_success_typings(DependsOnSCCs, Coordinator). +wait_for_success_typings(Mode, Job, InitData, #state{coordinator = Coordinator}) -> + JobLabel = dialyzer_coordinator:get_job_label(Mode, Job), + DependsOnJobLabels = dialyzer_succ_typings:find_depends_on(JobLabel, InitData), + ?debug("~w: Deps ~p: ~p\n", [self(), Job, DependsOnJobLabels]), + dialyzer_coordinator:wait_for_success_typings(DependsOnJobLabels, Coordinator). diff --git a/lib/dialyzer/src/erl_types.erl b/lib/dialyzer/src/erl_types.erl index 6fe8111482..9abd967cd1 100644 --- a/lib/dialyzer/src/erl_types.erl +++ b/lib/dialyzer/src/erl_types.erl @@ -4875,7 +4875,7 @@ remote_from_form(Anno, RemMod, Name, Args, S, D, L, C) -> end. ext_types_message(MFA, Anno, Site) -> - {MFA, {site_file(Site), erl_anno:location(Anno)}}. + {MFA, {site_file(Site), erl_anno:location(Anno), site_mfa(Site)}}. remote_from_form1(RemMod, Name, Args, ArgsLen, RemDict, RemType, TypeNames, Site, S, D, L, C) -> @@ -5206,6 +5206,9 @@ list_check_record_fields([H|Tail], S, C) -> site_module({_, {Module, _, _}, _}) -> Module. +site_mfa({_, {M, F, A}, _}) -> + {M, F, A}. + site_file({_, _, File}) -> File. diff --git a/lib/dialyzer/src/typer.erl b/lib/dialyzer/src/typer.erl index 9fa2cd78a2..3b88ca6282 100644 --- a/lib/dialyzer/src/typer.erl +++ b/lib/dialyzer/src/typer.erl @@ -215,6 +215,7 @@ help_message() -> Prints type information as Edoc @spec comments, not as type specs --plt PLT Use the specified dialyzer PLT file rather than the default one + (Incremental and non-incremental PLT files are supported) -T file* The specified file(s) already contain type specifications and these are to be trusted in order to print specs for the rest of the files diff --git a/lib/dialyzer/src/typer_core.erl b/lib/dialyzer/src/typer_core.erl index c9051f825e..1556d0b09b 100644 --- a/lib/dialyzer/src/typer_core.erl +++ b/lib/dialyzer/src/typer_core.erl @@ -438,7 +438,7 @@ get_type({{M, F, A} = MFA, Range, Arg}, CodeServer, Records, Analysis) -> {{F, A}, {contract, Contract}}; {error, {overlapping_contract, []}} -> {{F, A}, {contract, Contract}}; - {error, invalid_contract} -> + {error, {invalid_contract, _}} -> CString = dialyzer_contracts:contract_to_string(Contract), SigString = dialyzer_utils:format_sig(Sig, Records), Msg = io_lib:format("Error in contract of function ~w:~tw/~w\n" @@ -960,13 +960,32 @@ get_file([]) -> "no_file". % should not happen -spec get_dialyzer_plt(analysis()) -> plt(). -get_dialyzer_plt(#analysis{plt = PltFile0}) -> +get_dialyzer_plt(#analysis{plt = PltFile0}=Analysis) -> PltFile = case PltFile0 =:= none of - true -> dialyzer_plt:get_default_plt(); + true -> + case filelib:is_regular(dialyzer_cplt:get_default_cplt_filename()) of + true -> dialyzer_cplt:get_default_cplt_filename(); + false -> + case filelib:is_regular(dialyzer_iplt:get_default_iplt_filename()) of + true -> dialyzer_iplt:get_default_iplt_filename(); + false -> + fatal_error( + "No PLT file given, and no existing PLT was found at default locations " ++ + dialyzer_cplt:get_default_cplt_filename() ++ + " and " ++ + dialyzer_iplt:get_default_iplt_filename(), + Analysis) + end + end; false -> PltFile0 end, - dialyzer_plt:from_file(PltFile). + case dialyzer_plt:plt_kind(PltFile) of + cplt -> dialyzer_cplt:from_file(PltFile); + iplt -> dialyzer_iplt:from_file(PltFile); + bad_file -> fatal_error("Invalid PLT file at path " ++ PltFile, Analysis); + no_file -> fatal_error("No PLT file found at path " ++ PltFile, Analysis) + end. %% Exported Types diff --git a/lib/dialyzer/test/Makefile b/lib/dialyzer/test/Makefile index 7308c20400..4c79fe0581 100644 --- a/lib/dialyzer/test/Makefile +++ b/lib/dialyzer/test/Makefile @@ -12,9 +12,12 @@ AUXILIARY_FILES=\ dialyzer_common.erl\ file_utils.erl\ dialyzer_SUITE.erl\ + dialyzer_utils_SUITE.erl\ dialyzer_cl_SUITE.erl\ abstract_SUITE.erl\ - plt_SUITE.erl\ + iplt_SUITE.erl\ + cplt_SUITE.erl\ + incremental_SUITE.erl\ typer_SUITE.erl\ erl_types_SUITE.erl diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/callbacks_and_specs b/lib/dialyzer/test/behaviour_SUITE_data/results/callbacks_and_specs index d1bfc7295c..77ebf486c9 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/results/callbacks_and_specs +++ b/lib/dialyzer/test/behaviour_SUITE_data/results/callbacks_and_specs @@ -1,5 +1,6 @@ -my_callbacks_wrong.erl:26:2: The return type #state{parent::pid(),status::'closed' | 'init' | 'open',subscribe::[{pid(),integer()}],counter::integer()} in the specification of callback_init/1 is not a subtype of {'ok',_}, which is the expected return type for the callback of the my_behaviour behaviour +my_callbacks_wrong.erl:26:2: The return type #state{parent::pid(),status::'closed' | 'init' | 'open',subscribe::[{pid(),integer()}],counter::integer()} in the specification of callback_init/1 has nothing in common with {'ok',_}, which is the expected return type for the callback of the my_behaviour behaviour my_callbacks_wrong.erl:28:1: The inferred return type of callback_init/1 (#state{parent::pid(),status::'init',subscribe::[],counter::1}) has nothing in common with {'ok',_}, which is the expected return type for the callback of the my_behaviour behaviour -my_callbacks_wrong.erl:30:2: The return type {'reply',#state{parent::pid(),status::'closed' | 'init' | 'open',subscribe::[{pid(),integer()}],counter::integer()}} in the specification of callback_cast/3 is not a subtype of {'noreply',_}, which is the expected return type for the callback of the my_behaviour behaviour -my_callbacks_wrong.erl:39:2: The specified type for the 2nd argument of callback_call/3 (atom()) is not a supertype of pid(), which is expected type for this argument in the callback of the my_behaviour behaviour +my_callbacks_wrong.erl:30:2: The return type {'reply',#state{parent::pid(),status::'closed' | 'init' | 'open',subscribe::[{pid(),integer()}],counter::integer()}} in the specification of callback_cast/3 has nothing in common with {'noreply',_}, which is the expected return type for the callback of the my_behaviour behaviour +my_callbacks_wrong.erl:33:1: The inferred return type of callback_cast/3 ({'reply',_}) has nothing in common with {'noreply',_}, which is the expected return type for the callback of the my_behaviour behaviour +my_callbacks_wrong.erl:39:2: The specified type for the 2nd argument of callback_call/3 (atom()) has nothing in common with pid(), which is expected type for this argument in the callback of the my_behaviour behaviour diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args b/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args index a1412f29e6..6a9aacc77e 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args +++ b/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args @@ -2,4 +2,4 @@ gen_server_incorrect_args.erl:3:2: Undefined callback function handle_cast/2 (behaviour gen_server) gen_server_incorrect_args.erl:3:2: Undefined callback function init/1 (behaviour gen_server) gen_server_incorrect_args.erl:7:1: The inferred return type of handle_call/3 ({'no'} | {'ok'}) has nothing in common with {'noreply',_} | {'noreply',_,'hibernate' | 'infinity' | non_neg_integer() | {'continue',_}} | {'reply',_,_} | {'stop',_,_} | {'reply',_,_,'hibernate' | 'infinity' | non_neg_integer() | {'continue',_}} | {'stop',_,_,_}, which is the expected return type for the callback of the gen_server behaviour -gen_server_incorrect_args.erl:7:1: The inferred type for the 2nd argument of handle_call/3 ('boo' | 'foo') is not a supertype of {pid(),gen_server:reply_tag()}, which is expected type for this argument in the callback of the gen_server behaviour +gen_server_incorrect_args.erl:7:1: The inferred type for the 2nd argument of handle_call/3 ('boo' | 'foo') has nothing in common with {pid(),gen_server:reply_tag()}, which is expected type for this argument in the callback of the gen_server behaviour diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/otp_6221 b/lib/dialyzer/test/behaviour_SUITE_data/results/otp_6221 new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/dialyzer/test/behaviour_SUITE_data/results/otp_6221 diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour b/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour index ab69a698c5..347500277b 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour +++ b/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour @@ -3,7 +3,7 @@ sample_callback_wrong.erl:16:1: The inferred return type of sample_callback_2/0 sample_callback_wrong.erl:17:1: The inferred return type of sample_callback_3/0 ('fair') has nothing in common with 'fail' | {'ok',1..255}, which is the expected return type for the callback of the sample_behaviour behaviour sample_callback_wrong.erl:18:1: The inferred return type of sample_callback_4/1 ('fail') has nothing in common with 'ok', which is the expected return type for the callback of the sample_behaviour behaviour sample_callback_wrong.erl:20:1: The inferred return type of sample_callback_5/1 (string()) has nothing in common with 'fail' | 'ok', which is the expected return type for the callback of the sample_behaviour behaviour -sample_callback_wrong.erl:20:1: The inferred type for the 1st argument of sample_callback_5/1 (atom()) is not a supertype of 1..255, which is expected type for this argument in the callback of the sample_behaviour behaviour +sample_callback_wrong.erl:20:1: The inferred type for the 1st argument of sample_callback_5/1 (atom()) has nothing in common with 1..255, which is expected type for this argument in the callback of the sample_behaviour behaviour sample_callback_wrong.erl:22:1: The inferred return type of sample_callback_6/3 ({'okk',number()}) has nothing in common with 'fail' | {'ok',1..255}, which is the expected return type for the callback of the sample_behaviour behaviour -sample_callback_wrong.erl:22:1: The inferred type for the 3rd argument of sample_callback_6/3 (atom()) is not a supertype of string(), which is expected type for this argument in the callback of the sample_behaviour behaviour +sample_callback_wrong.erl:22:1: The inferred type for the 3rd argument of sample_callback_6/3 (atom()) has nothing in common with string(), which is expected type for this argument in the callback of the sample_behaviour behaviour sample_callback_wrong.erl:4:2: Undefined callback function sample_callback_1/0 (behaviour sample_behaviour) diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour_old b/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour_old index 6d8145a8ca..5f2b8d2ad4 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour_old +++ b/lib/dialyzer/test/behaviour_SUITE_data/results/sample_behaviour_old @@ -1,4 +1,4 @@ -incorrect_args_callback.erl:12:1: The inferred type for the 2nd argument of bar/2 ('yes') is not a supertype of [any()], which is expected type for this argument in the callback of the correct_behaviour behaviour +incorrect_args_callback.erl:12:1: The inferred type for the 2nd argument of bar/2 ('yes') has nothing in common with [any()], which is expected type for this argument in the callback of the correct_behaviour behaviour incorrect_return_callback.erl:9:1: The inferred return type of foo/0 ('error') has nothing in common with 'no' | 'yes', which is the expected return type for the callback of the correct_behaviour behaviour missing_callback.erl:5:2: Undefined callback function foo/0 (behaviour correct_behaviour) diff --git a/lib/dialyzer/test/behaviour_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl b/lib/dialyzer/test/behaviour_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl index 0459622dc1..0f485096ff 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl +++ b/lib/dialyzer/test/behaviour_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl @@ -28,13 +28,13 @@ callback_init(Parent) -> #state{parent = Parent}. %% Wrong return -spec callback_cast(state(), pid() | atom(), cast_message()) -> - {'noreply' | 'reply', state()}. %% More generic spec + {'reply', state()}. %% Non-overlapping spec callback_cast(#state{parent = Pid} = State, Pid, Message) when Message =:= 'open'; Message =:= 'close' -> - {noreply, State#state{status = Message}}; + {reply, State#state{status = Message}}; callback_cast(State, _Pid, _Message) -> - {noreply, State}. + {reply, State}. -spec callback_call(state(), atom(), call_message()) -> %% Wrong arg spec {'reply', state(), call_reply()}. diff --git a/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_behaviour.erl b/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_behaviour.erl new file mode 100644 index 0000000000..347ea403c2 --- /dev/null +++ b/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_behaviour.erl @@ -0,0 +1,3 @@ +-module(my_behaviour). + +-callback foo() -> #{ {{{f,f}, f}, f} => x }. diff --git a/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_callbacks_correct.erl b/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_callbacks_correct.erl new file mode 100644 index 0000000000..4a08017c84 --- /dev/null +++ b/lib/dialyzer/test/behaviour_SUITE_data/src/otp_6221/my_callbacks_correct.erl @@ -0,0 +1,17 @@ +-module(my_callbacks_correct). + +-behaviour(my_behaviour). + +-export([foo/0]). + +-type pair(A,B) :: {A,B}. + +-type nested() :: pair(pair(pair(f,f),f),f). + +%% This is correctly implemented, but a combination of Dialyzer +%% "simplification" logic and subtyping rules for behaviours means +%% this implementation has historically been erroneously rejected +-spec foo() -> #{ nested() => x }. +foo() -> + Ret = #{ {{{f,f}, f}, f} => x }, + Ret. diff --git a/lib/dialyzer/test/plt_SUITE.erl b/lib/dialyzer/test/cplt_SUITE.erl index e1c5ecdd95..b4b7870ef4 100644 --- a/lib/dialyzer/test/plt_SUITE.erl +++ b/lib/dialyzer/test/cplt_SUITE.erl @@ -1,7 +1,4 @@ -%% This suite is the only hand made and simply -%% checks if we can build and update a plt. - --module(plt_SUITE). +-module(cplt_SUITE). -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). @@ -39,8 +36,12 @@ mod_dep_from_exported_fun_spec_args/1, mod_dep_from_unexported_fun_spec_return/1, mod_dep_from_exported_fun_spec_return/1, - mod_dep_from_unexported_type/1 - ]). + mod_dep_from_unexported_type/1, + adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported/1, + removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported/1, + removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed/1, + adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed/1 + ]). suite() -> [{timetrap, ?plt_timeout}]. @@ -73,7 +74,11 @@ all() -> [build_plt, build_xdg_plt, beam_tests, update_plt, run_plt_check, mod_dep_from_exported_fun_spec_args, mod_dep_from_unexported_fun_spec_return, mod_dep_from_exported_fun_spec_return, - mod_dep_from_unexported_type + mod_dep_from_unexported_type, + adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported, + removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported, + removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed, + adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed ]. build_plt(Config) -> @@ -252,7 +257,7 @@ update_plt(Config) -> test() -> plt_gc:one().">>, {ok, TestBeam} = compile(Config, Test, test, []), - [{warn_callgraph, _, {call_to_missing, [plt_gc, one, 0]}}] = + [{warn_callgraph, {_Filename, {5,19}}, {call_to_missing, [plt_gc,one,0]}}] = dialyzer:run([{analysis_type, succ_typings}, {files, [TestBeam]}, {init_plt, Plt}] ++ Opts), @@ -550,7 +555,7 @@ missing_plt_file_2(Config, PltFile, BeamFile2) -> end. missing_plt_file_3() -> - try dialyzer_plt:from_file("no_such_file"), false + try dialyzer_cplt:from_file("no_such_file"), false catch throw:{dialyzer_error, _} -> true end. @@ -872,7 +877,7 @@ check_plt_deps(Config, TestName, DependerSrc, ExpectedTypeDepsInPltUnsorted) -> {ok, DepsBeamFile} = compile(Config, type_deps, []), {ok, DependerBeamFile} = compile(Config, DependerSrc, depender, []), [] = run_dialyzer(plt_build, [DependerBeamFile, DepsBeamFile], [{output_plt, PltFile}]), - {_ResPlt, #plt_info{mod_deps = DepsByModule}} = dialyzer_plt:plt_and_info_from_file(PltFile), + {_ResPlt, #plt_info{mod_deps = DepsByModule}} = dialyzer_cplt:plt_and_info_from_file(PltFile), ActualTypeDepsInPlt = lists:sort(dict:to_list(dict:erase(erlang, DepsByModule))), @@ -895,7 +900,7 @@ erlang_beam() -> EBeam end. -%% Builds the named module using the source in the plt_SUITE_data dir +%% Builds the named module using the source in the cplt_SUITE_data dir compile(Config, Module, CompileOpts) -> Source = lists:concat([Module, ".erl"]), PrivDir = proplists:get_value(priv_dir,Config), @@ -920,3 +925,81 @@ run_dialyzer(Analysis, Files, Opts) -> {files, Files}, {from, byte_code} | Opts]). + + +m_src_without_warning() -> <<" + -module(m). + -export([updt/3]). + + updt(X, K, V) -> X#{ K => V }. + ">>. + +m_src_with_warning() -> <<" + -module(m). + -export([updt/3]). + + -spec updt(list(), term(), term()) -> list(). % Warning: Spec is wrong! Function takes a map, not a list + updt(X, K, V) -> X#{ K => V }. + ">>. + +adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".plt"), + Opts = [{init_plt, [PltFile]}], + OptsNoContractWarn = [{init_plt, [PltFile]}, {warnings, [no_contracts]}], + {ok, Beam} = compile(Config, m_src_with_warning(), m, []), + [] = run_dialyzer(incremental, [Beam], OptsNoContractWarn), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [Beam], Opts)). + +removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".plt"), + Opts = [{init_plt, [PltFile]}], + OptsNoContractWarn = [{init_plt, [PltFile]}, {warnings, [no_contracts]}], + {ok, Beam} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [Beam], Opts)), + ?assertEqual( + [], + run_dialyzer(incremental, [Beam], OptsNoContractWarn)). + +removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".plt"), + Opts = [{init_plt, [PltFile]}], + {ok, BeamFileBefore} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [BeamFileBefore], Opts)), + {ok, BeamFileAfter} = compile(Config, m_src_without_warning(), m, []), + ?assertEqual( + [], + run_dialyzer(incremental, [BeamFileAfter], Opts)). + +adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".plt"), + Opts = [{init_plt, [PltFile]}], + {ok, BeamFileAfter} = compile(Config, m_src_without_warning(), m, []), + ?assertEqual( + [], + run_dialyzer(incremental, [BeamFileAfter], Opts)), + {ok, BeamFileBefore} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [BeamFileBefore], Opts)). diff --git a/lib/dialyzer/test/plt_SUITE_data/type_deps.erl b/lib/dialyzer/test/cplt_SUITE_data/type_deps.erl index f6fcfc3d23..f6fcfc3d23 100644 --- a/lib/dialyzer/test/plt_SUITE_data/type_deps.erl +++ b/lib/dialyzer/test/cplt_SUITE_data/type_deps.erl diff --git a/lib/dialyzer/test/dialyzer_SUITE.erl b/lib/dialyzer/test/dialyzer_SUITE.erl index 058fdc8a50..fe6987192a 100644 --- a/lib/dialyzer/test/dialyzer_SUITE.erl +++ b/lib/dialyzer/test/dialyzer_SUITE.erl @@ -20,6 +20,7 @@ -module(dialyzer_SUITE). -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). %% Default timetrap timeout (set in init_per_testcase). -define(default_timeout, test_server:minutes(10)). @@ -31,12 +32,21 @@ -export([init_per_testcase/2, end_per_testcase/2]). %% Test cases must be exported. --export([app_test/1, appup_test/1, file_list/1]). +-export([app_test/1, appup_test/1]). +-export([cplt_info/1, iplt_info/1, + incremental_plt_given_to_classic_mode/1, + classic_plt_given_to_incremental_mode/1, + if_output_plt_is_missing_incremental_mode_makes_it/1, + file_list/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test, appup_test, file_list]. + [app_test, appup_test, cplt_info, iplt_info, + incremental_plt_given_to_classic_mode, + classic_plt_given_to_incremental_mode, + if_output_plt_is_missing_incremental_mode_makes_it, + file_list]. groups() -> []. @@ -62,6 +72,15 @@ end_per_testcase(_Case, Config) -> test_server:timetrap_cancel(Dog), ok. +compile(Config, Prog, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = proplists:get_value(priv_dir,Config), + Filename = filename:join([PrivDir, Source]), + ok = file:write_file(Filename, Prog), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(Filename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + %%% %%% Test cases starts here. %%% @@ -77,6 +96,99 @@ app_test(Config) when is_list(Config) -> appup_test(Config) when is_list(Config) -> ok = test_server:appup_test(dialyzer). +cplt_info(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "cplt_info.plt"), + _ = dialyzer:run([{analysis_type, plt_build}, + {files, [Beam1]}, + {init_plt, Plt1}, + {from, byte_code}]), + + {ok, [{files, [Beam1]}]} = dialyzer:plt_info(Plt1). + +iplt_info(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "iplt_info.plt"), + _ = dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt1}, + {from, byte_code}]), + + {ok, {incremental, [{modules, [foo]}]}} = dialyzer:plt_info(Plt1). + +incremental_plt_given_to_classic_mode(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "my_incremental.iplt"), + _ = dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt1}, + {output_plt, Plt1}, + {from, byte_code}]), + + ?assertException(throw, {dialyzer_error, "Given file is an incremental PLT file, but outside of incremental mode, a classic PLT file is expected: {init_plt_file," ++ _ }, + dialyzer:run([{analysis_type, plt_check}, + {files, [Beam1]}, + {init_plt, Plt1}, + {from, byte_code}])). + +classic_plt_given_to_incremental_mode(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "my_classic.plt"), + _ = dialyzer:run([{analysis_type, plt_build}, + {files, [Beam1]}, + {init_plt, Plt1}, + {output_plt, Plt1}, + {from, byte_code}]), + + ?assertException(throw, {dialyzer_error, "Given file is a classic PLT file, but in incremental mode, an incremental PLT file is expected: {init_plt_file," ++ _ }, + dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt1}, + {from, byte_code}])). + +if_output_plt_is_missing_incremental_mode_makes_it(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "brand_new.iplt"), + ?assertMatch(false, filelib:is_regular(Plt1)), + + _ = dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt1}, + {output_plt, Plt1}, + {from, byte_code}]), + + ?assertMatch(true, filelib:is_regular(Plt1)). + file_list(Config) -> PrivDir = proplists:get_value(priv_dir, Config), @@ -129,3 +241,4 @@ expected(Files0) -> ":6:5: The variable _ can never match since previous clauses completely covered the type \n" " atom()\n" || F <- Files], iolist_to_binary(S). + diff --git a/lib/dialyzer/test/dialyzer_common.erl b/lib/dialyzer/test/dialyzer_common.erl index ca1fc1993a..f7c00e3389 100644 --- a/lib/dialyzer/test/dialyzer_common.erl +++ b/lib/dialyzer/test/dialyzer_common.erl @@ -89,7 +89,7 @@ explain_fail_with_lock() -> obtain_plt(PltFilename) -> io:format("Obtaining plt:"), - InitPlt = dialyzer_plt:get_default_plt(), + InitPlt = dialyzer_cplt:get_default_cplt_filename(), io:format("Will try to use ~s as a starting point and add otp apps ~w.", [InitPlt, ?required_modules]), try dialyzer:run([{analysis_type, plt_add}, @@ -120,7 +120,7 @@ build_plt(PltFilename) -> end. -spec check(atom(), dialyzer:dial_options(), string(), string()) -> - 'same' | {differ, [term()]}. + 'same' | {differ, TestCase :: atom(), [term()]}. check(TestCase, Opts, Dir, OutDir) -> PltFilename = plt_file(OutDir), @@ -161,10 +161,10 @@ check(TestCase, Opts, Dir, OutDir) -> case file_utils:diff(NewResFile, OldResFile) of 'same' -> file:delete(NewResFile), 'same'; - Any -> escape_strings(Any) + {'differ', List} -> escape_strings({'differ', TestCase, List}) end catch - Kind:Error -> {'dialyzer crashed', Kind, Error} + Kind:Error:Stacktrace -> {'dialyzer crashed', Kind, Error, Stacktrace} end. fix_options(Opts, Dir) -> @@ -203,9 +203,9 @@ create_all_suites() -> Suites = get_suites(Cwd), lists:foreach(fun create_suite/1, Suites). -escape_strings({differ,List}) -> +escape_strings({differ, TestCase, List}) -> Map = fun({T,L,S}) -> {T,L,xmerl_lib:export_text(S)} end, - {differ, lists:keysort(3, lists:map(Map, List))}. + {differ, TestCase, lists:keysort(3, lists:map(Map, List))}. -spec get_suites(file:filename()) -> [string()]. @@ -216,7 +216,8 @@ get_suites(Dir) -> FullFilenames = [filename:join(Dir, F) || F <-Filenames ], Dirs = [is_suite_data(filename:basename(F), ?suite_data) || F <- FullFilenames, - file_utils:file_type(F) =:= {ok, 'directory'}], + file_utils:file_type(F) =:= {ok, 'directory'}, + file_utils:file_type(filename:join(F, ?input_files_directory)) =:= {ok, 'directory'}], [S || {yes, S} <- Dirs] end. diff --git a/lib/dialyzer/test/dialyzer_utils_SUITE.erl b/lib/dialyzer/test/dialyzer_utils_SUITE.erl new file mode 100644 index 0000000000..312b3bd589 --- /dev/null +++ b/lib/dialyzer/test/dialyzer_utils_SUITE.erl @@ -0,0 +1,52 @@ +-module(dialyzer_utils_SUITE). + +-export([all/0, + p_map_implements_map/1, + p_map_handles_errors_like_map_does/1, + p_map_preserves_ordering/1 + ]). + +all() -> + [p_map_implements_map, + p_map_handles_errors_like_map_does, + p_map_preserves_ordering + ]. + +p_map_implements_map(_Config) -> + Fun = fun (N) -> N + 2 end, + List = [2,1,3,2], + Expected = lists:map(Fun, List), + Expected = dialyzer_utils:p_map(Fun, List). + +p_map_handles_errors_like_map_does(_Config) -> + Fun = fun (3) -> throw("an error"); (N) -> N + 2 end, + List = [2,1,3,2], + ListsOk = + try + lists:map(Fun, List), + false + catch _:_ -> + true + end, + case ListsOk of + true -> ok; + false -> ct:fail("Expected lists:map/2 to throw") + end, + UtilsOk = + try + dialyzer_utils:p_map(Fun, List), + false + catch _:_ -> + true + end, + case UtilsOk of + true -> ok; + false -> ct:fail("Expected dialyzer_utils:p_map/2 to throw") + end. + +p_map_preserves_ordering(_Config) -> + Fun = fun (N) -> timer:sleep(N * 50), N + 2 end, + List = [2,1,3,2,1,5,2,1,3,2,9,4,2,1,3,2,5,4], + Expected = lists:map(Fun, List), + Expected = dialyzer_utils:p_map(Fun, List). + diff --git a/lib/dialyzer/test/incremental_SUITE.erl b/lib/dialyzer/test/incremental_SUITE.erl new file mode 100644 index 0000000000..486e1d07be --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE.erl @@ -0,0 +1,962 @@ +%% This suite tests the incremental mode which for now woirks as follows: +%% You can specify which files you want in your plt, incremental mode will +%% do the analysis needed to get the PLT into a good state and report the +%% warnings from the modules which it re-analyzed. + +-module(incremental_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include("dialyzer_test_constants.hrl"). + +-export([suite/0, + all/0, + report_new_plt_test/1, + report_degree_of_incrementality_test/1, + report_no_stored_warnings_test/1, + report_stored_warnings_no_files_changed_test/1, + report_stored_warnings_only_files_safely_removed_test/1, + report_old_plt_version_number_test/1, + report_old_plt_version_hash_test/1, + report_legal_warnings_added/1, + report_legal_warnings_removed/1, + incremental_test/1, + incremental_select_warnings_test/1, + add_and_remove_test/1, + add_and_remove_test_with_unknown_warnings/1, + verify_plt_info/1, + verify_plt_info_with_unknown_warnings/1, + fixing_all_warnings/1, + default_apps_config_xdg/1, + default_apps_config_env_var/1, + default_apps_config_env_var_prioritised_over_xdg/1, + multiple_plts_unsupported_in_incremental_mode/1]). + +suite() -> + [{timetrap, ?plt_timeout}]. + +all() -> [report_new_plt_test, + report_degree_of_incrementality_test, + report_no_stored_warnings_test, + report_stored_warnings_no_files_changed_test, + report_stored_warnings_only_files_safely_removed_test, + report_old_plt_version_number_test, + report_old_plt_version_hash_test, + report_legal_warnings_added, + report_legal_warnings_removed, + incremental_test, + incremental_select_warnings_test, + add_and_remove_test, + add_and_remove_test_with_unknown_warnings, + verify_plt_info, + verify_plt_info_with_unknown_warnings, + fixing_all_warnings, + default_apps_config_xdg, + default_apps_config_env_var, + default_apps_config_env_var_prioritised_over_xdg, + multiple_plts_unsupported_in_incremental_mode]. + +erlang_module() -> + case code:where_is_file("erlang.beam") of + non_existing -> + filename:join([code:root_dir(), + "erts", "preloaded", "ebin", + "erlang.beam"]); + EBeam -> + EBeam + end. + +verify_plt_info_with_unknown_warnings(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || + Module <- Modules] end, + Plt = filename:join(PrivDir, "verify_plt_info.iplt"), + compile_all(DataDir, PrivDir), + ErlangModule = erlang_module(), + Opts = [{init_plt, [Plt]}, {warnings, [unknown]}], + AllSubSets = all_subsets(all_mods()), + Run = fun(Mods) -> run_dialyzer(incremental, [ErlangModule | ToPath(Mods)], Opts) end, + [run_and_verify_plt_info_with_unknown_warnings(Subset, Run, Plt) || + Subset <- AllSubSets]. + +run_and_verify_plt_info_with_unknown_warnings(Subset, Run, PltFile) -> + _ = Run(Subset), + {Plt, {iplt_info, _Md5, _ModDeps, Warnings, _LegalWarnings}} = + dialyzer_iplt:plt_and_info_from_file(PltFile), + dialyzer_plt:delete(Plt), + verify_unknown_warnings(Subset, Warnings). + +verify_unknown_warnings(Subset, Warnings) -> + Unused = all_mods() -- Subset, + lists:foreach(fun(Mod) -> check_unknown_warnings(Mod, Unused, Warnings) end, Subset), + lists:foreach(fun(Mod) -> check_unused_mod_warnings(Mod, Warnings) end, Unused). + +check_unknown_warnings(Mod, Unused, Warnings) -> + Unknown = [unknown_function_warning(M) || + M <- Unused, lists:member(Mod, depends_on(M))], + Other = [warning_for_module(Mod)], + Expected = lists:sort(Unknown ++ Other), + Stored = lists:sort(replace_location(maps:get(Mod, Warnings))), + Expected = Stored. + +unknown_function_warning(M) -> + {warn_unknown, loc, {unknown_function, {M, id, 1}}}. + +verify_plt_info(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || + Module <- Modules] end, + Plt = filename:join(PrivDir, "verify_plt_info.iplt"), + compile_all(DataDir, PrivDir), + ErlangModule = erlang_module(), + Opts = [{init_plt, [Plt]}], + AllSubSets = all_subsets(all_mods()), + Run = fun(Mods) -> run_dialyzer(incremental, [ErlangModule | ToPath(Mods)], Opts) end, + [run_and_verify_plt_info(Subset, Run, Plt) || Subset <- AllSubSets]. + +run_and_verify_plt_info(Subset, Run, PltFile) -> + _ = Run(Subset), + {Plt, {iplt_info, Md5, ModDeps, Warnings, _LegalWarnings}} = + dialyzer_iplt:plt_and_info_from_file(PltFile), + dialyzer_plt:delete(Plt), + verify_md5_list([erlang|Subset], Md5), + verify_mod_deps(Subset, ModDeps), + verify_warnings(Subset, Warnings). + +verify_md5_list(AllModules, Md5) -> + ExpectedModules = lists:sort(AllModules), + ActualModules = lists:sort([ModuleName || {ModuleName, _CheckSum} <- Md5]), + case ExpectedModules =:= ActualModules of + true -> + ok; + false -> + ct:pal("Expected modules: ~p~nActual Modules:~p~n", + [ExpectedModules, ActualModules]), + ExpectedModules = ActualModules + end. + +verify_mod_deps(Subset, ModDeps) -> + Unused = all_mods() -- Subset, + DependsOn = fun(Mod) -> depends_on(Mod) -- Unused end, + lists:foreach(fun(Mod) -> check_one_mod(Mod, ModDeps, DependsOn) end, Subset). + +check_one_mod(Mod, ModDeps, DependsOn) -> + StoredDeps = lists:sort(rewrite_ans(dict:find(Mod, ModDeps))), + ExpectedDeps = lists:sort(DependsOn(Mod)), + case ExpectedDeps =:= StoredDeps of + true -> + ok; + false -> + ct:pal("Expected deps: ~p~nStored deps:~p~n", + [ExpectedDeps, StoredDeps]), + ExpectedDeps = StoredDeps + end. + +rewrite_ans({ok, Value}) -> Value; +rewrite_ans(error) -> []. + +verify_warnings(Subset, Warnings) -> + Unused = all_mods() -- Subset, + lists:foreach(fun(Mod) -> check_warnings(Mod, Warnings) end, Subset), + lists:foreach(fun(Mod) -> check_unused_mod_warnings(Mod, Warnings) end, Unused). + +check_warnings(Mod, Warnings) -> + StoredWarnings = lists:sort(replace_location(maps:get(Mod, Warnings))), + ExpectedWarnings = [warning_for_module(Mod)], + ExpectedWarnings = StoredWarnings. + +check_unused_mod_warnings(Mod, Warnings) -> + false = maps:is_key(Mod, Warnings). + +add_and_remove_test_with_unknown_warnings(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || + Module <- Modules] end, + Plt = filename:join(PrivDir, "add_and_remove.iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {warnings, [unknown]}, {report_mode, verbose}], + Run = fun(Mods) -> + run_dialyzer_for_modules_analyzed(incremental, + [erlang_module() | ToPath(Mods)], + [{warning_files, ToPath(Mods)} | Opts]) + end, + %% We ignore empty Mods, since no warning modules modules is + %% interpreted as a request to show all warning modules (for + %% convenience). However, if all module warnings are shown and + %% 'unknown' warnings are enabled, we get a load of 'unknown' + %% warnings from the built-in Erlang module, so we exclude that + %% case here + AllSubSets = [Mods || Mods <- all_subsets(all_mods()), Mods =/= []], + %Pairs = [{I, N} || I <- AllSubSets, N <- AllSubSets], % All pairs takes ~ 10 mins to run + Pairs = some_pairs(AllSubSets), + [run_one_add_and_remove_test(Initial, New, Run, true) || {Initial, New} <- Pairs], + ok. + +%% Run the tests to add and remove modules for some combinations +add_and_remove_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, "add_and_remove.iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}], + Run = fun(Mods) -> run_dialyzer_for_modules_analyzed(incremental, [erlang_module() | ToPath(Mods)], Opts) end, + AllSubSets = [Mods || Mods <- all_subsets(all_mods())], + %Pairs = [{I, N} || I <- AllSubSets, N <- AllSubSets], % All pairs takes ~ 10 mins to run + Pairs = some_pairs(AllSubSets), + [run_one_add_and_remove_test(Initial, New, Run, false) || {Initial, New} <- Pairs], + ok. + + +%% Runs a test where we first run the analysis on the InitialSet, and +%% then run the analysis on the newset, checking that we get the +%% warnings we expect given this change of modules. Supports running +%% with unknown function and types and also with getting warnings for +%% all modules, not just the re-analyzed ones. +run_one_add_and_remove_test(InitialSet, NewSet, Run, WithUnknown) -> + {ExpectedErrors, ExpectedAnalyzed} = + expected_warnings_and_analyzed_add_and_remove(InitialSet, NewSet, WithUnknown), + _ = Run(InitialSet), + {ErrorsRaw, Analyzed} = Run(NewSet), + Errors = ordsets:from_list(replace_location(ErrorsRaw)), + + if Errors =/= ExpectedErrors -> + ct:pal("Initial modules: ~p~nNew Modules:~p~nModules analyzed:~p~nModules expected to be analyzed:~p~n" + "Returned Warnings:~p~nExpected Warnings:~p~n", + [InitialSet, NewSet, Analyzed, ExpectedAnalyzed, Errors, ExpectedErrors]), + {Errors, Analyzed} = {ExpectedErrors, ExpectedAnalyzed}; + true -> + ok + end. + +%% Just create some pairs. +some_pairs([A, B | Rest]) -> + [{A, B} | some_pairs([B|Rest])]; +some_pairs(_) -> []. + + +%% We expect incrementality to minimise the number of modules +%% re-analysed, but still report warnings for modules which didn't +%% need to be re-analysed because their warnings were cached. We +%% expect a module to be re-analyzed when it is freshly added or when +%% one of its transistive dependencies have been added or removed, +%% using the dependency graph from the previous run. Adds unknown +%% function warnings when they are supported, and generates all +%% warnings when that feature is turned on. +expected_warnings_and_analyzed_add_and_remove(Initial, New, WithUnknown) -> + InitialSet = ordsets:from_list(Initial), + NewSet = ordsets:from_list(New), + Removed = ordsets:from_list(InitialSet -- NewSet), + Added = ordsets:from_list(NewSet -- InitialSet), + Changed = ordsets:union(Removed, Added), + All = ordsets:from_list(all_mods()), + Missing = ordsets:subtract(All, NewSet), + DependsOn = fun(Mod) -> depends_on(Mod, fun(X) -> lists:member(X, InitialSet) end) end, + Dependencies = ordsets:from_list([Mod || C <- Changed, Mod <- depends_on_transitively(C, DependsOn)]), + Analyzed = ordsets:intersection(ordsets:union(Added, Dependencies), NewSet), + WarnModules = NewSet, + UnknownWarnings = + [unknown_function_warning(M) || + WithUnknown, + Mod <- WarnModules, + M <- Missing, + lists:member(Mod, depends_on(M))], + {ordsets:from_list([warning_for_module(Mod) || + Mod <- WarnModules] ++ UnknownWarnings), Analyzed}. + +%% Run the incremental test for all combination of the 6 modules, +%% checking that we get the expected level of incrementality, i.e. we +%% only reanalyze what we need to given what has changed +incremental_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || + Module <- Modules] end, + Change = fun(M) -> change_module(DataDir, PrivDir, M) end, + Plt = filename:join(PrivDir, "incremental.iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}], + AllWarnings = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + Run = fun() -> + run_dialyzer_for_modules_analyzed(incremental, + [erlang_module() | ToPath(all_mods())], + Opts) + end, + [run_one_incremental_test(SubSet, Change, Run, AllWarnings) || + SubSet <- lists:reverse(all_subsets(all_mods()))], + ok. + +%% Run the incremental test for all combination of the 6 modules, +%% checking that with or without touching a file, we get the expected +%% warnings +incremental_select_warnings_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || + Module <- Modules] end, + Plt = filename:join(PrivDir, "select_warnings.iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}], + _ = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + RunWithoutChange = + fun(ExtraOpt) -> + run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], [ExtraOpt|Opts]) + end, + RunWithChange = + fun(ExtraOpt) -> + change_module(DataDir, PrivDir, m4), + run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], [ExtraOpt|Opts]) + end, + TestSets = [Mods || Mods <- all_subsets(all_mods()), Mods =/= []], + [run_one_select_warnings_test(SubSet, RunWithoutChange, ToPath) || SubSet <- TestSets], + [run_one_select_warnings_test(SubSet, RunWithChange, ToPath) || SubSet <- TestSets]. + +run_one_select_warnings_test(WarningMods, Run, ToPath) -> + ExpectedWarnings = lists:sort([warning_for_module(M) || M <- WarningMods]), + ActualWarnings = lists:sort(replace_location(Run({warning_files, ToPath(WarningMods)}))), + case ExpectedWarnings =/= ActualWarnings of + true -> + ct:pal("Warning modules: ~p~nReturned Warnings:~p~nExpected Warnings:~p~n", + [WarningMods, ActualWarnings, ExpectedWarnings]), + ExpectedWarnings = ActualWarnings; + false -> + ok + end. + +%% Recompiles a set of modules and then runs the analysis verifying that we re-analyze +%% the changed modules and their transitive dependencies. +run_one_incremental_test(ChangedMods, Change, Run, ExpectedWarnings) -> + lists:foreach(Change, ChangedMods), + ExpectedModulesAnalyzed = expected_modules_reanalyzed_incremental(ChangedMods), + {Warnings, ModulesAnalyzed} = Run(), + if Warnings =/= ExpectedWarnings -> + ct:pal("Changed modules: ~p~nModules reanalyzed:~p~nExpected modules reanalyzed:~p~nReturned Warnings:~p~nExpected Warnings:~p~n", + [ChangedMods, ModulesAnalyzed, ExpectedModulesAnalyzed, Warnings, ExpectedWarnings]), + {Warnings, ModulesAnalyzed} = {ExpectedWarnings, ExpectedModulesAnalyzed}; + true -> + ok + end. + +check_metrics_file_contents(MetricsFileName, ExpectedMetricsFileStr) -> + {ok, MetricsFileBin} = file:read_file(MetricsFileName), + MetricsFileStr = unicode:characters_to_list(MetricsFileBin), + if MetricsFileStr =/= ExpectedMetricsFileStr -> + ct:pal("Metrics file didn't contain expected contents. Expected:~n~p~nActual:~n~p~n", + [ExpectedMetricsFileStr, MetricsFileStr]), + ExpectedMetricsFileStr = MetricsFileStr; + true -> + ok + end. + +report_new_plt_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + {_Warnings, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + CandidateLines = + lists:filter( + fun (Line) -> + case string:prefix(Line, "PLT does not yet exist at") of + nomatch -> false; + _ -> true + end + end, + Stdout), + ExpectedLine = "PLT does not yet exist at " ++ Plt ++ ", so an analysis must be run for 7 modules to populate it\n", + if CandidateLines =/= [ExpectedLine] -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nLines that look close: ~n~p~n", + [ExpectedLine, CandidateLines]), + CandidateLines = [ExpectedLine]; + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: new_plt_file\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_degree_of_incrementality_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + _WarningsBeforeTouchingFile = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + change_module(DataDir, PrivDir, m1), + change_module(DataDir, PrivDir, m5), + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + + ExpectedLine = + " Of the 7 files being tracked, 2 have been changed or removed, " + "resulting in 5 requiring analysis because they depend on those changes\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 5\n" + "reason: incremental_changes\n" + "changed_or_removed_modules: 2\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_stored_warnings_only_files_safely_removed_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + _WarningsBeforeTouchingFile = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + delete_module_beam_file(PrivDir, m1), + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods() -- [m1])], Opts), + + ExpectedLine = + "PLT has fully cached the request because nothing depended on the file removed, so no additional analysis is needed\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 6\n" + "analysed_modules: 0\n" + "reason: incremental_changes\n" + "changed_or_removed_modules: 1\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_stored_warnings_no_files_changed_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + _WarningsBeforeTouchingFile = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + + ExpectedLine = + "PLT has fully cached the request, so no additional analysis is needed\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 0\n" + "reason: incremental_changes\n" + "changed_or_removed_modules: 0\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_old_plt_version_number_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + PltFileName = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [PltFileName]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + + % Set up "old" PLT file + Plt = dialyzer_plt:new(), + ModDeps = dict:new(), + ValidButEmptyPltInfo = + {iplt_info, [], dict:new(), #{}, ordsets:new()}, + OldVsn = "0.0.1", + dialyzer_iplt:to_file_custom_vsn(PltFileName, Plt, ModDeps, ValidButEmptyPltInfo, OldVsn, none), + + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + ExpectedLine = + "PLT is for a different Dialyzer version, so an analysis must be run for 7 modules to rebuild it\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: plt_built_with_different_version\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_old_plt_version_hash_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + PltFileName = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [PltFileName]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + + %% Set up "old" PLT file + Plt = dialyzer_plt:new(), + ModDeps = dict:new(), + ValidButEmptyPltInfo = + {iplt_info, [], dict:new(), #{}, ordsets:new()}, + FakeImplMd5 = [ + {erl_types, erlang:md5([$f, $o, $o])}, + {erl_bif_types, erlang:md5([$b, $a, $r])} + ], + dialyzer_iplt:to_file_custom_vsn(PltFileName, Plt, ModDeps, ValidButEmptyPltInfo, none, FakeImplMd5), + + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + ExpectedLine = + "PLT is for a different Dialyzer version, so an analysis must be run for 7 modules to rebuild it\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: plt_built_with_different_version\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_no_stored_warnings_test(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + PltFileName = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [PltFileName]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + + % Run Dialyzer, read the PLT, blank the warning map, then write it back + _ = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + {Plt, PltInfo} = dialyzer_iplt:plt_and_info_from_file(PltFileName), + {iplt_info, Md5, ModDeps, _Warnings, LegalWarnings} = PltInfo, + MissingWarningMap = none, + ValidPltInfoWithNoWarningMap = + {iplt_info, Md5, ModDeps, MissingWarningMap, LegalWarnings}, + dialyzer_iplt:to_file(PltFileName, Plt, ModDeps, ValidPltInfoWithNoWarningMap), + + {_, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts), + ExpectedLine = + "PLT does not contain cached warnings, so an analysis must be run for 7 modules to rebuild it\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: no_stored_warnings_in_plt\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_legal_warnings_added(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}], + _WarningsBeforeTouchingFile = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + Opts1 = [{init_plt, [Plt]}, {report_mode, verbose}, {warnings, [unknown]}, {metrics_file, MetricsFile}], + {_WarningsAfterTouchingFile, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts1), + + ExpectedLine = + "PLT was built for a different set of enabled warnings, so an analysis " + "must be run for 7 modules to rebuild it\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: warnings_changed\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +report_legal_warnings_removed(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + MetricsFile = PrivDir ++ "my_incrementality_metrics.log", + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + compile_all(DataDir, PrivDir), + Opts = [{init_plt, [Plt]}, {report_mode, verbose}, {warnings, [unknown]}], + _ = run_dialyzer(incremental, [erlang_module() | ToPath(all_mods())], Opts), + Opts1 = [{init_plt, [Plt]}, {report_mode, verbose}, {metrics_file, MetricsFile}], + {_, Stdout} = run_dialyzer_capture(incremental, [erlang_module() | ToPath(all_mods())], Opts1), + + ExpectedLine = + "PLT was built for a different set of enabled warnings, so an analysis " + "must be run for 7 modules to rebuild it\n", + case lists:any(fun (Line) -> Line =:= ExpectedLine end, Stdout) of + false -> + ct:pal("Failed to find expected log line in stdout.~nExpected:~n ~p~nStdout: ~n~p~n", + [ExpectedLine, Stdout]), + ct:fail("Dialyzer failed to print the expected line to stdout"); + true -> + ok + end, + ExpectedMetricsFileStr = + "total_modules: 7\n" + "analysed_modules: 7\n" + "reason: warnings_changed\n", + check_metrics_file_contents(MetricsFile, ExpectedMetricsFileStr). + +fixing_all_warnings(Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + ToPath = fun(Modules) -> [PrivDir ++ atom_to_list(Module) ++ ".beam" || Module <- Modules] end, + Plt = filename:join(PrivDir, "verify_plt_info.iplt"), + compile_all(DataDir, PrivDir), + ErlangModule = erlang_module(), + Opts = [{init_plt, [Plt]}, {warning_files, ToPath([fix, got_fixed])}], + [] = run_dialyzer(incremental, [ErlangModule | ToPath([fix, got_fixed])], Opts), + break_module(DataDir, PrivDir), + [_Error1, _Error2] = run_dialyzer(incremental, [ErlangModule | ToPath([fix, got_fixed])], Opts), + fix_module(DataDir, PrivDir), + [] = run_dialyzer(incremental, [ErlangModule | ToPath([fix, got_fixed])], Opts). + +break_module(DataDir, PrivDir) -> + compile:file(filename:join(DataDir, "fix.erl"), + [{outdir, PrivDir}, + debug_info, + {d, error, true}]). + +fix_module(DataDir, PrivDir) -> + compile:file(filename:join(DataDir, "fix.erl"), + [{outdir, PrivDir}, + debug_info]). + +%% There is one warning per module which will be reported when that module is re-analyzed +%% We expect a module to be re-analyzed when it has changed, or when one of its transitive +%% dependencies have changed. +expected_modules_reanalyzed_incremental(ChangedMods) -> + DependsOn = fun depends_on/1, + ordsets:from_list([Mod || C <- ChangedMods, + Mod <- [C|depends_on_transitively(C, DependsOn)]]). + +change_module(DataDir, PrivDir, Mod) -> + compile:file(filename:join(DataDir, atom_to_list(Mod)++".erl"), + [{outdir, PrivDir}, debug_info, {d, Mod, erlang:monotonic_time()}]). + +delete_module_beam_file(PrivDir, Mod) -> + ModuleBeamPath = PrivDir ++ atom_to_list(Mod) ++ ".beam", + ok = file:delete(ModuleBeamPath). + +%% Common utility functions + +replace_location(Warnings) -> + [{Type, loc, Warning} || {Type, _Loc, Warning} <- Warnings]. + +warning_for_module(Mod) -> {warn_contract_types,loc, + {invalid_contract, + [Mod,wrong,1,{[1],true},"(float()) -> float()","(integer()) -> integer()"]}}. + +compile_all(DataDir, PrivDir) -> + [{ok, _} = compile:file(File, [{outdir, PrivDir}, debug_info]) || File <- filelib:wildcard(DataDir ++ "*.erl")]. + +%% This describes the dependencies between modules m1->m6 in +%% incremental_SUITE_data/ If you change thos deps update here. +%% Don't add cycles (yet) or at least without changing depends on transitively +depends_on(m1) -> []; +depends_on(m2) -> [m1]; +depends_on(m3) -> []; +depends_on(m4) -> [m2,m3]; +depends_on(m5) -> [m4]; +depends_on(m6) -> [m4]. + +depends_on(X, IsInPlt) -> + [D || D <- depends_on(X), IsInPlt(D)]. + +depends_on_transitively(M, DependsOn) -> + Val = DependsOn(M), + Val ++ [Dep || V <- Val, Dep <- [V|depends_on_transitively(V, DependsOn)]]. + +all_subsets([X|Rest]) -> + [Res || SubSet <- all_subsets(Rest), Res <- [[X|SubSet], SubSet]]; +all_subsets([]) -> [[]]. + +all_mods() -> [m1,m2,m3,m4,m5,m6]. + +run_dialyzer(Analysis, Files, Opts) -> + dialyzer:run([{analysis_type, Analysis}, + {files, Files}, + {from, byte_code}| + Opts]). + +run_dialyzer_for_modules_analyzed(Analysis, Files, Opts) -> + dialyzer:run_report_modules_analyzed([{analysis_type, Analysis}, + {files, Files}, + {from, byte_code}| + Opts]). + +default_apps_config_xdg(Config) -> + TestHome = filename:join(?config(priv_dir, Config), ?FUNCTION_NAME), + + %% We change the $HOME of the emulator to run this test + HomeEnv = + case os:type() of + {win32, _} -> + [Drive | Path] = filename:split(TestHome), + [{"APPDATA", filename:join(TestHome,"AppData")}, + {"HOMEDRIVE", Drive}, {"HOMEPATH", Path}]; + _ -> + [{"HOME", TestHome}] + end, + + %% We add in the default subdirs used for each system + %% to store config within the home directory + ConfigSubDirs = + case os:type() of + {unix, darwin} -> + ["Library","Application Support"]; + {win32, _} -> + []; + _ -> + [".config"] + end, + + HomeConfigFilename = + filename:join([TestHome] ++ ConfigSubDirs ++ ["erlang", "dialyzer.config"]), + ok = filelib:ensure_dir(HomeConfigFilename), + + HomeConfig = + {incremental, {default_apps, [stdlib, kernel, erts, compiler, mnesia, ftp]}}, + + ok = file:write_file(HomeConfigFilename, io_lib:format("~p.~n", [HomeConfig])), + + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + + {ok, Peer, Node} = ?CT_PEER(#{ env => HomeEnv }), + + erpc:call( + Node, + fun() -> + _ = dialyzer:run([{analysis_type, incremental}, + {init_plt,PltFile}, + {output_plt, PltFile}]), + {ok, {incremental, [{modules, Modules}]}} = dialyzer:plt_info(PltFile), + + ExpectedModules = [gb_sets, erlang, compile, mnesia, ftp], + + % Assert PLT info contains modules from the apps given in the config + ?assertMatch([], sets:to_list(sets:subtract( + sets:from_list(ExpectedModules), + sets:from_list(Modules)))) + end), + + peer:stop(Peer). + + +default_apps_config_env_var(Config) -> + TestDir = filename:join(?config(priv_dir, Config), ?FUNCTION_NAME), + + ConfigFilename = + filename:join( + [TestDir, "some_custom_location", "dialyzer.config"]), + ok = filelib:ensure_dir(ConfigFilename), + + Env = [{"DIALYZER_CONFIG", ConfigFilename}], + + DialyzerConfig = + {incremental, {default_apps, [stdlib, kernel, erts, compiler, mnesia, ftp]}}, + + ok = file:write_file(ConfigFilename, io_lib:format("~p.~n", [DialyzerConfig])), + + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + + {ok, Peer, Node} = ?CT_PEER(#{ env => Env }), + + erpc:call( + Node, + fun() -> + _ = dialyzer:run([{analysis_type, incremental}, + {init_plt,PltFile}, + {output_plt, PltFile}]), + {ok, {incremental, [{modules, Modules}]}} = dialyzer:plt_info(PltFile), + + ExpectedModules = [gb_sets, erlang, compile, mnesia, ftp], + + % Assert PLT info contains modules from the apps given in the config + ?assertMatch([], sets:to_list(sets:subtract( + sets:from_list(ExpectedModules), + sets:from_list(Modules)))) + end), + + peer:stop(Peer). + +default_apps_config_env_var_prioritised_over_xdg(Config) -> + TestHome = filename:join(?config(priv_dir, Config), ?FUNCTION_NAME), + + ConfigSubDirs = + case os:type() of + {unix, darwin} -> + ["Library","Application Support"]; + {win32, _} -> + []; + _ -> + [".config"] + end, + + HomeConfigFilename = + filename:join([TestHome] ++ ConfigSubDirs ++ ["erlang", "dialyzer.config"]), + ok = filelib:ensure_dir(HomeConfigFilename), + + HomeConfig = + {incremental, {default_apps, [stdlib, kernel, erts]}}, + + ok = file:write_file(HomeConfigFilename, io_lib:format("~p.~n", [HomeConfig])), + + ConfigFilename = + filename:join( + [TestHome, "some_custom_location", "dialyzer.config"]), + ok = filelib:ensure_dir(ConfigFilename), + + + %% We change the $HOME of the emulator to run this test + HomeEnv = case os:type() of + {win32, _} -> + [Drive | Path] = filename:split(TestHome), + [{"APPDATA", filename:join(TestHome,"AppData")}, + {"HOMEDRIVE", Drive}, {"HOMEPATH", Path}]; + _ -> + [{"HOME", TestHome}] + end, + + Env = HomeEnv ++ [{"DIALYZER_CONFIG", ConfigFilename}], + + EnvVarConfig = + {incremental, {default_apps, [compiler, mnesia, ftp]}}, + + ok = file:write_file(ConfigFilename, io_lib:format("~p.~n", [EnvVarConfig])), + + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + + {ok, Peer, Node} = ?CT_PEER(#{ env => Env }), + + erpc:call( + Node, + fun() -> + _ = dialyzer:run([{analysis_type, incremental}, + {init_plt,PltFile}, + {output_plt, PltFile}]), + {ok, {incremental, [{modules, Modules}]}} = dialyzer:plt_info(PltFile), + + ExpectedModules = [compile, mnesia, ftp], + UnexpectedModules = [gb_sets], + + % Assert PLT info contains modules from the apps given in the env var config + ?assertMatch([], sets:to_list(sets:subtract( + sets:from_list(ExpectedModules), + sets:from_list(Modules)))), + + % Assert PLT info does not contain modules from the apps given in the xdg config + ?assertMatch([], sets:to_list(sets:intersection( + sets:from_list(UnexpectedModules), + sets:from_list(Modules)))) + end), + + peer:stop(Peer). + +multiple_plts_unsupported_in_incremental_mode(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + BazPltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ "-baz.iplt"), + QuuxPltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ "-quux.iplt"), + + BazSrc = <<" + -module(baz). + + f() -> ok. + ">>, + QuuxSrc = <<" + -module(quux). + + g() -> undefined. + ">>, + + {ok, BazBeamFile} = compile(Config, BazSrc, baz, []), + {ok, QuuxBeamFile} = compile(Config, QuuxSrc, quux, []), + _ = run_dialyzer(incremental, [BazBeamFile], [{output_plt, BazPltFile}]), + _ = run_dialyzer(incremental, [QuuxBeamFile], [{output_plt, QuuxPltFile}]), + + ?assertThrow( + {dialyzer_error,"Incremental mode does not support multiple PLT files (" ++ _}, + run_dialyzer(incremental, + [BazBeamFile, QuuxBeamFile], + [{plts, [BazPltFile, QuuxPltFile]}])). + +compile(Config, Prog, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = proplists:get_value(priv_dir,Config), + Filename = filename:join([PrivDir, Source]), + ok = file:write_file(Filename, Prog), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(Filename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +%% https://erlang.org/doc/man/ct.html#capture_get-1 seems to work by redirecting +%% writes to stdout to be messages to the current pid. Those messages get queued up, +%% with the intent to read them back later to get stdout. +%% +%% At various points, Dialyzer dequeues messages from its worker child processes, +%% sometimes throwing them away knowing it doesn't need certain results from them. +%% +%% Sadly, these two things interact badly and Dialyzer ends up dequeuing (and hence, +%% deleting) some of what was written to stdout. The solution is to run Dialyzer in +%% a sub-process, so the messages go to the parent process's mailbox and Dialyzer is +%% free to do what it wants with its own messages. +%% +%% Fundamentally this is an issue with ct:capture_get, since it seems to presume it's +%% fine to send messages to the test case's pid and they'll never be read by the code +%% under test! +run_dialyzer_capture(Analysis, Files, Opts) -> + ct:capture_start(), + TestCasePid = self(), + RunDialyzer = fun() -> + Ret = run_dialyzer(Analysis, Files, Opts), + TestCasePid ! {dialyzer_done, Ret} + end, + _DialyzerPid = spawn_link(RunDialyzer), + DialyzerRet = + receive + {dialyzer_done, Ret} -> Ret + end, + ct:capture_stop(), + Stdout = ct:capture_get([]), + {DialyzerRet, Stdout}. diff --git a/lib/dialyzer/test/incremental_SUITE_data/fix.erl b/lib/dialyzer/test/incremental_SUITE_data/fix.erl new file mode 100644 index 0000000000..6c1e85eda5 --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/fix.erl @@ -0,0 +1,10 @@ +-module(fix). + +-export([m/0]). + +-spec m() -> integer(). +-ifdef(error). +m() -> 3.14. +-else. +m() -> 3. +-endif. diff --git a/lib/dialyzer/test/incremental_SUITE_data/got_fixed.erl b/lib/dialyzer/test/incremental_SUITE_data/got_fixed.erl new file mode 100644 index 0000000000..e6db525b90 --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/got_fixed.erl @@ -0,0 +1,6 @@ +-module(got_fixed). + +-export([m/0]). + +-spec m() -> integer(). +m() -> fix:m(). diff --git a/lib/dialyzer/test/incremental_SUITE_data/m1.erl b/lib/dialyzer/test/incremental_SUITE_data/m1.erl new file mode 100644 index 0000000000..1c49a2714a --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m1.erl @@ -0,0 +1,14 @@ +-module(m1). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m1). +m() -> ?m1. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> m2:id(X). diff --git a/lib/dialyzer/test/incremental_SUITE_data/m2.erl b/lib/dialyzer/test/incremental_SUITE_data/m2.erl new file mode 100644 index 0000000000..83c4f2b55e --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m2.erl @@ -0,0 +1,14 @@ +-module(m2). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m2). +m() -> ?m2. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> m4:id(X). diff --git a/lib/dialyzer/test/incremental_SUITE_data/m3.erl b/lib/dialyzer/test/incremental_SUITE_data/m3.erl new file mode 100644 index 0000000000..8ea8b8b45b --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m3.erl @@ -0,0 +1,14 @@ +-module(m3). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m3). +m() -> ?m3. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> m4:id(X). diff --git a/lib/dialyzer/test/incremental_SUITE_data/m4.erl b/lib/dialyzer/test/incremental_SUITE_data/m4.erl new file mode 100644 index 0000000000..43da7b80c7 --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m4.erl @@ -0,0 +1,14 @@ +-module(m4). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m4). +m() -> ?m4. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> m5:id(X), m6:id(X). diff --git a/lib/dialyzer/test/incremental_SUITE_data/m5.erl b/lib/dialyzer/test/incremental_SUITE_data/m5.erl new file mode 100644 index 0000000000..0e8e3b2d1b --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m5.erl @@ -0,0 +1,14 @@ +-module(m5). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m5). +m() -> ?m5. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> X. diff --git a/lib/dialyzer/test/incremental_SUITE_data/m6.erl b/lib/dialyzer/test/incremental_SUITE_data/m6.erl new file mode 100644 index 0000000000..9d24db5378 --- /dev/null +++ b/lib/dialyzer/test/incremental_SUITE_data/m6.erl @@ -0,0 +1,14 @@ +-module(m6). + +-export([id/1, m/0, wrong/1]). + +-ifdef(m6). +m() -> ?m6. +-else. +m() -> false. +-endif. + +-spec wrong(float()) -> float(). +wrong(X) when is_integer(X) -> X. + +id(X) -> X. diff --git a/lib/dialyzer/test/indent_SUITE_data/results/callbacks_and_specs b/lib/dialyzer/test/indent_SUITE_data/results/callbacks_and_specs index 0d513775a1..6e34494e2a 100644 --- a/lib/dialyzer/test/indent_SUITE_data/results/callbacks_and_specs +++ b/lib/dialyzer/test/indent_SUITE_data/results/callbacks_and_specs @@ -3,7 +3,7 @@ my_callbacks_wrong.erl:26:2: The return type #state{parent :: pid(), status :: 'closed' | 'init' | 'open', subscribe :: [{pid(), integer()}], - counter :: integer()} in the specification of callback_init/1 is not a subtype of + counter :: integer()} in the specification of callback_init/1 has nothing in common with {'ok', _}, which is the expected return type for the callback of the my_behaviour behaviour my_callbacks_wrong.erl:28:1: The inferred return type of callback_init/1 (#state{parent :: pid(), @@ -11,13 +11,9 @@ my_callbacks_wrong.erl:28:1: The inferred return type of callback_init/1 subscribe :: [], counter :: 1}) has nothing in common with {'ok', _}, which is the expected return type for the callback of the my_behaviour behaviour -my_callbacks_wrong.erl:30:2: The return type - {'reply', - #state{parent :: pid(), - status :: 'closed' | 'init' | 'open', - subscribe :: [{pid(), integer()}], - counter :: integer()}} in the specification of callback_cast/3 is not a subtype of +my_callbacks_wrong.erl:33:1: The inferred return type of callback_cast/3 + ({'reply', _}) has nothing in common with {'noreply', _}, which is the expected return type for the callback of the my_behaviour behaviour my_callbacks_wrong.erl:39:2: The specified type for the 2nd argument of callback_call/3 ( - atom()) is not a supertype of + atom()) has nothing in common with pid(), which is expected type for this argument in the callback of the my_behaviour behaviour diff --git a/lib/dialyzer/test/indent_SUITE_data/results/contracts_with_subtypes b/lib/dialyzer/test/indent_SUITE_data/results/contracts_with_subtypes index 039e5e23f6..9294602211 100644 --- a/lib/dialyzer/test/indent_SUITE_data/results/contracts_with_subtypes +++ b/lib/dialyzer/test/indent_SUITE_data/results/contracts_with_subtypes @@ -74,8 +74,12 @@ contracts_with_subtypes.erl:238:2: The pattern contracts_with_subtypes.erl:239:2: The pattern 'alpha' can never match the type {'ok', _, string()} -contracts_with_subtypes.erl:23:2: Invalid type specification for function contracts_with_subtypes:extract2/0. The success typing is +contracts_with_subtypes.erl:23:2: Invalid type specification for function contracts_with_subtypes:extract2/0. + The success typing is contracts_with_subtypes:extract2 () -> 'something' + But the spec is contracts_with_subtypes:extract2 + () -> 'ok' + The return types do not overlap contracts_with_subtypes.erl:240:2: The pattern {'ok', 42} can never match the type {'ok', _, string()} @@ -129,8 +133,12 @@ contracts_with_subtypes.erl:78:16: The call contracts_with_subtypes:foo2 contracts_with_subtypes.erl:79:16: The call contracts_with_subtypes:foo3 (5) breaks the contract (Arg1) -> Res when Arg2 :: atom(), Arg1 :: Arg2, Res :: atom() -contracts_with_subtypes.erl:7:2: Invalid type specification for function contracts_with_subtypes:extract/0. The success typing is +contracts_with_subtypes.erl:7:2: Invalid type specification for function contracts_with_subtypes:extract/0. + The success typing is contracts_with_subtypes:extract () -> 'something' + But the spec is contracts_with_subtypes:extract + () -> 'ok' + The return types do not overlap contracts_with_subtypes.erl:80:16: The call contracts_with_subtypes:foo4 (5) breaks the contract (Type) -> Type when Type :: atom() diff --git a/lib/dialyzer/test/indent_SUITE_data/results/record_update b/lib/dialyzer/test/indent_SUITE_data/results/record_update index 997b3ecb96..9ab8d478b6 100644 --- a/lib/dialyzer/test/indent_SUITE_data/results/record_update +++ b/lib/dialyzer/test/indent_SUITE_data/results/record_update @@ -1,3 +1,7 @@ -record_update.erl:7:2: Invalid type specification for function record_update:quux/2. The success typing is +record_update.erl:7:2: Invalid type specification for function record_update:quux/2. + The success typing is record_update:quux (#foo{bar :: atom()}, atom()) -> #foo{bar :: atom()} + But the spec is record_update:quux + (#foo{}, string()) -> #foo{} + They do not overlap in the 2nd argument diff --git a/lib/dialyzer/test/indent_SUITE_data/results/sample_behaviour b/lib/dialyzer/test/indent_SUITE_data/results/sample_behaviour index e7ae7505f6..393d428576 100644 --- a/lib/dialyzer/test/indent_SUITE_data/results/sample_behaviour +++ b/lib/dialyzer/test/indent_SUITE_data/results/sample_behaviour @@ -12,12 +12,12 @@ sample_callback_wrong.erl:20:1: The inferred return type of sample_callback_5/1 (string()) has nothing in common with 'fail' | 'ok', which is the expected return type for the callback of the sample_behaviour behaviour sample_callback_wrong.erl:20:1: The inferred type for the 1st argument of sample_callback_5/1 ( - atom()) is not a supertype of + atom()) has nothing in common with 1..255, which is expected type for this argument in the callback of the sample_behaviour behaviour sample_callback_wrong.erl:22:1: The inferred return type of sample_callback_6/3 ({'okk', number()}) has nothing in common with 'fail' | {'ok', 1..255}, which is the expected return type for the callback of the sample_behaviour behaviour sample_callback_wrong.erl:22:1: The inferred type for the 3rd argument of sample_callback_6/3 ( - atom()) is not a supertype of + atom()) has nothing in common with string(), which is expected type for this argument in the callback of the sample_behaviour behaviour sample_callback_wrong.erl:4:2: Undefined callback function sample_callback_1/0 (behaviour sample_behaviour) diff --git a/lib/dialyzer/test/indent_SUITE_data/results/simple b/lib/dialyzer/test/indent_SUITE_data/results/simple index f33392d5bc..7fea96c502 100644 --- a/lib/dialyzer/test/indent_SUITE_data/results/simple +++ b/lib/dialyzer/test/indent_SUITE_data/results/simple @@ -83,8 +83,12 @@ rec_api.erl:29:5: Matching of pattern rec_api.erl:33:5: The attempt to match a term of type rec_adt:r1() against the pattern {'r1', 'a'} breaks the opacity of the term -rec_api.erl:35:2: Invalid type specification for function rec_api:adt_t1/1. The success typing is +rec_api.erl:35:2: Invalid type specification for function rec_api:adt_t1/1. + The success typing is rec_api:adt_t1 (#r1{f1 :: 'a'}) -> #r1{f1 :: 'a'} + But the spec is rec_api:adt_t1 + (rec_adt:r1()) -> rec_adt:r1() + They do not overlap in the 1st argument, and the return types do not overlap rec_api.erl:40:2: The specification for rec_api:adt_r1/0 has an opaque subtype rec_adt:r1() which is violated by the success typing () -> #r1{f1 :: 'a'} @@ -182,14 +186,26 @@ simple1_api.erl:342:8: Guard test simple1_api.erl:347:8: Guard test A :: simple1_adt:b1() =:= 'true' contains an opaque term as 1st argument -simple1_api.erl:355:2: Invalid type specification for function simple1_api:bool_adt_t6/1. The success typing is +simple1_api.erl:355:2: Invalid type specification for function simple1_api:bool_adt_t6/1. + The success typing is simple1_api:bool_adt_t6 ('true') -> 1 + But the spec is simple1_api:bool_adt_t6 + (simple1_adt:b1()) -> integer() + They do not overlap in the 1st argument simple1_api.erl:365:8: Clause guard cannot succeed. -simple1_api.erl:368:2: Invalid type specification for function simple1_api:bool_adt_t8/2. The success typing is +simple1_api.erl:368:2: Invalid type specification for function simple1_api:bool_adt_t8/2. + The success typing is simple1_api:bool_adt_t8 (boolean(), boolean()) -> 1 + But the spec is simple1_api:bool_adt_t8 + (simple1_adt:b1(), simple1_adt:b2()) -> integer() + They do not overlap in the 1st and 2nd arguments simple1_api.erl:378:8: Clause guard cannot succeed. -simple1_api.erl:381:2: Invalid type specification for function simple1_api:bool_adt_t9/2. The success typing is +simple1_api.erl:381:2: Invalid type specification for function simple1_api:bool_adt_t9/2. + The success typing is simple1_api:bool_adt_t9 ('false', 'false') -> 1 + But the spec is simple1_api:bool_adt_t9 + (simple1_adt:b1(), simple1_adt:b2()) -> integer() + They do not overlap in the 1st and 2nd arguments simple1_api.erl:407:12: The size simple1_adt:i1() breaks the opacity of A simple1_api.erl:418:9: The attempt to match a term of type diff --git a/lib/dialyzer/test/indent_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl b/lib/dialyzer/test/indent_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl index 0459622dc1..31613ca06f 100644 --- a/lib/dialyzer/test/indent_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl +++ b/lib/dialyzer/test/indent_SUITE_data/src/callbacks_and_specs/my_callbacks_wrong.erl @@ -32,9 +32,9 @@ callback_init(Parent) -> #state{parent = Parent}. %% Wrong return callback_cast(#state{parent = Pid} = State, Pid, Message) when Message =:= 'open'; Message =:= 'close' -> - {noreply, State#state{status = Message}}; + {reply, State#state{status = Message}}; % Wrong return callback_cast(State, _Pid, _Message) -> - {noreply, State}. + {reply, State}. % Wrong return -spec callback_call(state(), atom(), call_message()) -> %% Wrong arg spec {'reply', state(), call_reply()}. diff --git a/lib/dialyzer/test/iplt_SUITE.erl b/lib/dialyzer/test/iplt_SUITE.erl new file mode 100644 index 0000000000..88120a99b8 --- /dev/null +++ b/lib/dialyzer/test/iplt_SUITE.erl @@ -0,0 +1,833 @@ +-module(iplt_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("dialyzer/src/dialyzer.hrl"). +-include("dialyzer_test_constants.hrl"). + +-export([suite/0, all/0, + init_per_testcase/2, + end_per_testcase/2, + beam_tests/1, + local_fun_same_as_callback/1, + letrec_rvals/1, + missing_plt_file/1, + build_xdg_plt/1, + mod_dep_from_behaviour/1, + mod_dep_from_record_definition_field_value_default_used/1, + mod_dep_from_record_definition_field_value_default_unused/1, + mod_dep_from_record_definition_field_type/1, + mod_dep_from_overloaded_callback/1, + mod_dep_from_exported_overloaded_fun_spec/1, + mod_dep_from_unexported_overloaded_fun_spec/1, + mod_dep_from_callback_constraint/1, + mod_dep_from_unexported_fun_spec_constraint/1, + mod_dep_from_exported_fun_spec_constraint/1, + mod_dep_from_exported_type/1, + mod_dep_from_callback_return/1, + mod_dep_from_callback_args/1, + mod_dep_from_unexported_opaque_type_args/1, + mod_dep_from_exported_opaque_type_args/1, + mod_dep_from_unexported_opaque_type/1, + mod_dep_from_exported_opaque_type/1, + mod_dep_from_unexported_type_args/1, + mod_dep_from_exported_type_args/1, + mod_dep_from_unexported_fun_spec_args/1, + mod_dep_from_exported_fun_spec_args/1, + mod_dep_from_unexported_fun_spec_return/1, + mod_dep_from_exported_fun_spec_return/1, + mod_dep_from_unexported_type/1, + adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported/1, + removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported/1, + removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed/1, + adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed/1, + reading_from_one_plt_and_writing_to_another_does_not_mutate_the_input_plt/1, + reading_from_and_writing_to_one_plt_mutates_it/1, + beams_with_no_debug_info_are_rejected/1 + ]). + +suite() -> + [{timetrap, ?plt_timeout}]. + +all() -> [build_xdg_plt, beam_tests, + local_fun_same_as_callback, + letrec_rvals, + missing_plt_file, + mod_dep_from_behaviour, + mod_dep_from_record_definition_field_value_default_used, + mod_dep_from_record_definition_field_value_default_unused, + mod_dep_from_record_definition_field_type, + mod_dep_from_overloaded_callback, + mod_dep_from_exported_overloaded_fun_spec, + mod_dep_from_unexported_overloaded_fun_spec, + mod_dep_from_callback_constraint, + mod_dep_from_unexported_fun_spec_constraint, + mod_dep_from_exported_fun_spec_constraint, + mod_dep_from_exported_type, + mod_dep_from_callback_return, + mod_dep_from_callback_args, + mod_dep_from_unexported_opaque_type_args, + mod_dep_from_exported_opaque_type_args, + mod_dep_from_unexported_opaque_type, + mod_dep_from_exported_opaque_type, + mod_dep_from_unexported_type_args, + mod_dep_from_exported_type_args, + mod_dep_from_unexported_fun_spec_args, + mod_dep_from_exported_fun_spec_args, + mod_dep_from_unexported_fun_spec_return, + mod_dep_from_exported_fun_spec_return, + mod_dep_from_unexported_type, + adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported, + removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported, + removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed, + adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed, + reading_from_one_plt_and_writing_to_another_does_not_mutate_the_input_plt, + reading_from_and_writing_to_one_plt_mutates_it, + beams_with_no_debug_info_are_rejected + ]. + +init_per_testcase(_TestCase, Config) -> + % Always run from a clean default PLT, so tests aren't dependent on what + % happens to be in the default user cache when they start + _ = file:delete(dialyzer_iplt:get_default_iplt_filename()), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +build_xdg_plt(Config) -> + TestHome = filename:join(?config(priv_dir, Config), ?FUNCTION_NAME), + + %% We change the $HOME of the emulator to run this test + HomeEnv = case os:type() of + {win32, _} -> + [Drive | Path] = filename:split(TestHome), + [{"APPDATA", filename:join(TestHome,"AppData")}, + {"HOMEDRIVE", Drive}, {"HOMEPATH", Path}]; + _ -> + [{"HOME", TestHome}] + end, + + {ok, Peer, Node} = ?CT_PEER(#{ env => HomeEnv }), + + erpc:call( + Node, + fun() -> + ?assertMatch([], dialyzer:run( + [{analysis_type, incremental}, + {apps, [erts]}])), + ?assertMatch( + {ok,_}, file:read_file( + filename:join( + filename:basedir(user_cache, "erlang"), + ".dialyzer_iplt"))) + end), + + peer:stop(Peer). + +beam_tests(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Plt = filename:join(PrivDir, "beam_tests.iplt"), + Src = <<" + -module(no_auto_import). + + %% Copied from erl_lint_SUITE.erl, clash6 + + -export([size/1]). + + size([]) -> + 0; + size({N,_}) -> + N; + size([_|T]) -> + 1+size(T). + ">>, + Opts = [no_auto_import], + {ok, BeamFile} = compile(Config, Src, no_auto_import, Opts), + [] = run_dialyzer(incremental, [BeamFile], [{output_plt, Plt}]), + ok. + + + +%%% If a behaviour module contains an non-exported function with the same name +%%% as one of the behaviour's callbacks, the callback info was inadvertently +%%% deleted from the PLT as the dialyzer_plt:delete_list/2 function was cleaning +%%% up the callback table. This bug was reported by Brujo Benavides. + +local_fun_same_as_callback(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(bad_behaviour). + -callback bad() -> bad. + -export([publicly_bad/0]). + + %% @doc This function is here just to avoid the 'unused' warning for bad/0 + publicly_bad() -> bad(). + + %% @doc This function overlaps with the callback with the same name, and + %% that was an issue for dialyzer since it's a private function. + bad() -> bad.">>, + {ok, Beam} = compile(Config, Prog1, bad_behaviour, []), + + ErlangBeam = case code:where_is_file("erlang.beam") of + non_existing -> + filename:join([code:root_dir(), + "erts", "preloaded", "ebin", + "erlang.beam"]); + EBeam -> + EBeam + end, + Plt = filename:join(PrivDir, "plt_bad_behaviour.iplt"), + Opts = [{from, byte_code}], + [] = dialyzer:run([{analysis_type, incremental}, + {files, [Beam, ErlangBeam]}, + {output_plt, Plt}] ++ Opts), + + Prog2 = + <<"-module(bad_child). + -behaviour(bad_behaviour). + + -export([bad/0]). + + %% @doc This function incorrectly implements bad_behaviour. + bad() -> not_bad.">>, + {ok, TestBeam} = compile(Config, Prog2, bad_child, []), + + [{warn_behaviour, _, + {callback_type_mismatch, + [bad_behaviour,bad,0,"'not_bad'","'bad'"]}}] = + dialyzer:run([{analysis_type, incremental}, + {files, [TestBeam, Beam, ErlangBeam]}, + {init_plt, Plt}] ++ Opts), + ok. + + + +letrec_rvals(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Plt = filename:join(PrivDir, "letrec_rvals.iplt"), + Prog = <<" +-module(letrec_rvals). + +-export([demo_fun/1]). + +demo_fun(_Arg) -> + case ok of + _ -> + _Res = _Arg, + [ ok || _ <- [] ] + end, + _Res. + +handle_info() -> + case chids_to_audit() of + {ChIds, St2} -> + [ ChId || ChId <- ChIds ], + ok + end, + check_done(St2). + +chids_to_audit() -> + some_module:get_audit_list(). + +check_done(_) -> + ok. + ">>, + {ok, BeamFile} = compile(Config, Prog, letrec_rvals, []), + _ = run_dialyzer(incremental, [BeamFile], [{output_plt, Plt}]), + ok. + +missing_plt_file(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + PltFile = filename:join(PrivDir, "missing_plt_file.iplt"), + Prog2 = <<"-module(missing_plt_file2). + t() -> foo.">>, + {ok, BeamFile2} = compile(Config, Prog2, missing_plt_file2, []), + + true = missing_plt_file_1(Config, PltFile, BeamFile2), + true = missing_plt_file_2(Config, PltFile, BeamFile2), + ok. + +missing_plt_file_1(Config, PltFile, BeamFile2) -> + BeamFile = create1(Config, PltFile), + ok = file:delete(BeamFile), + try incr(PltFile, BeamFile2), true + catch throw:{dialyzer_error, _} -> false + end. + +create1(Config, PltFile) -> + Prog = <<"-module(missing_plt_file). + t() -> foo.">>, + {ok, BeamFile} = compile(Config, Prog, missing_plt_file, []), + Files = [BeamFile], + _ = file:delete(PltFile), + _ = dialyzer:run([{files,Files}, + {init_plt, PltFile}, + {output_plt, PltFile}, + {analysis_type, incremental}]), + BeamFile. + +missing_plt_file_2(Config, PltFile, BeamFile2) -> + BeamFile = create2(Config, PltFile), + ok = file:delete(BeamFile), + try incr(PltFile, BeamFile2), true + catch throw:{dialyzer_error, _} -> false + end. + +create2(Config, PltFile) -> + Prog = <<"-module(missing_plt_file). + t() -> foo.">>, + {ok, BeamFile} = compile(Config, Prog, missing_plt_file, []), + Files = [BeamFile], + _ = file:delete(PltFile), + _ = dialyzer:run([{files,Files}, + {output_plt, PltFile}, + {analysis_type, incremental}]), + BeamFile. + +incr(PltFile, BeamFile2) -> + Files = [BeamFile2], + dialyzer:run([{files, Files}, + {plts,[PltFile]}, + {analysis_type, incremental}]). + +mod_dep_from_record_definition_field_value_default_unused(Config) -> + DependerSrc = <<" + -module(depender). + + -record(my_record, + { num_field = type_deps:get_num() :: number(), + str_field, + bool_field + }). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, []}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_record_definition_field_value_default_used(Config) -> + DependerSrc = <<" + -module(depender). + -export([f/0]). + + -record(my_record, + { num_field = type_deps:get_num() :: number(), + str_field, + bool_field + }). + + f() -> #my_record{str_field = \"foo\", bool_field = true}. % type_deps:get_num() used implicitly here + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_behaviour(Config) -> + DependerSrc = <<" + -module(depender). + -behaviour(type_deps). + -export([quux/1]). + + quux(N) -> N + 1. % Depends on behaviour module to check the callback implementation + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_record_definition_field_type(Config) -> + DependerSrc = <<" + -module(depender). + + -record(my_record, + { num_field = 1 :: type_deps:number_like(), + str_field, + bool_field + }). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_overloaded_callback(Config) -> + DependerSrc1 = <<" + -module(depender). + + -callback f(string()) -> string() + ; (type_deps:number_like()) -> type_deps:number_like(). + ">>, + DependerSrc2 = <<" + -module(depender). + + -callback f(type_deps:number_like()) -> type_deps:number_like() + ; (string()) -> string(). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc1, ExpectedTypeDepsInPlt), + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc2, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_overloaded_fun_spec(Config) -> + DependerSrc1 = <<" + -module(depender). + -export([f/1]). + + -spec f({a, atom()}) -> atom() + ; ({n, type_deps:number_like()}) -> type_deps:number_like(). + f({a, X}) when is_atom(X) -> X; + f({n, X}) when is_number(X) -> X. + ">>, + DependerSrc2 = <<" + -module(depender). + -export([f/1]). + + -spec f({n, type_deps:number_like()}) -> type_deps:number_like() + ; ({a, atom()}) -> atom(). + f({n, X}) when is_number(X) -> X; + f({a, X}) when is_atom(X) -> X. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc1, ExpectedTypeDepsInPlt), + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc2, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_overloaded_fun_spec(Config) -> + DependerSrc1 = <<" + -module(depender). + -compile({nowarn_unused_function, [f/1]}). + + -spec f({a, atom()}) -> atom() + ; ({n, type_deps:number_like()}) -> type_deps:number_like(). + f({a, X}) when is_atom(X) -> X; + f({n, X}) when is_number(X) -> X. + ">>, + DependerSrc2 = <<" + -module(depender). + -compile({nowarn_unused_function, [f/1]}). + + -spec f({n, type_deps:number_like()}) -> type_deps:number_like() + ; ({a, atom()}) -> atom(). + f({n, X}) when is_number(X) -> X; + f({a, X}) when is_atom(X) -> X. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc1, ExpectedTypeDepsInPlt), + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc2, ExpectedTypeDepsInPlt). + +mod_dep_from_callback_constraint(Config) -> + DependerSrc1 = <<" + -module(depender). + + -callback f(X) -> string() when X :: type_deps:number_like(). + ">>, + DependerSrc2 = <<" + -module(depender). + + -callback f(X :: type_deps:number_like()) -> string(). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc1, ExpectedTypeDepsInPlt), + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc2, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_fun_spec_constraint(Config) -> + DependerSrc = <<" + -module(depender). + -compile({nowarn_unused_function, [f/1]}). + + -spec f(N) -> number() when N :: type_deps:number_like(). + f(N) -> N. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_fun_spec_constraint(Config) -> + DependerSrc = <<" + -module(depender). + -export([f/1]). + + -spec f(N) -> number() when N :: type_deps:number_like(). + f(N) -> N. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_callback_return(Config) -> + DependerSrc = <<" + -module(depender). + + -callback f(string()) -> type_deps:number_like(). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_callback_args(Config) -> + DependerSrc = <<" + -module(depender). + + -callback f(type_deps:number_like()) -> string(). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_opaque_type(Config) -> + DependerSrc = <<" + -module(depender). + + -opaque my_type() :: {string(), type_deps:number_like()}. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_opaque_type(Config) -> + DependerSrc = <<" + -module(depender). + -export_type([my_type/0]). + + -opaque my_type() :: {string(), type_deps:number_like()}. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_opaque_type_args(Config) -> + DependerSrc = <<" + -module(depender). + + -type my_type() :: type_deps:my_opaque(number()). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_opaque_type_args(Config) -> + DependerSrc = <<" + -module(depender). + -export_type([my_type/0]). + + -type my_type() :: type_deps:my_opaque(number()). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_type_args(Config) -> + DependerSrc = <<" + -module(depender). + + -type my_type() :: {string(), type_deps:number_like()}. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_type_args(Config) -> + DependerSrc = <<" + -module(depender). + -export_type([my_type/0]). + + -type my_type() :: {string(), type_deps:number_like()}. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_fun_spec_args(Config) -> + DependerSrc = <<" + -module(depender). + -compile({nowarn_unused_function, [f/1]}). + + -spec f(type_deps:number_like()) -> number(). + f(N) -> N. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_fun_spec_args(Config) -> + DependerSrc = <<" + -module(depender). + -export([f/1]). + + -spec f(N :: type_deps:number_like()) -> number(). + f(N) -> N. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_fun_spec_return(Config) -> + DependerSrc = <<" + -module(depender). + -compile({nowarn_unused_function, [f/0]}). + + -spec f() -> type_deps:number_like(). + f() -> 1. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_fun_spec_return(Config) -> + DependerSrc = <<" + -module(depender). + -export([f/0]). + + -spec f() -> type_deps:number_like(). + f() -> 1. + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_unexported_type(Config) -> + DependerSrc = <<" + -module(depender). + + -type my_type() :: type_deps:list_like(number()). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +mod_dep_from_exported_type(Config) -> + DependerSrc = <<" + -module(depender). + -export_type([my_type/0]). + + -type my_type() :: type_deps:list_like(number()). + ">>, + ExpectedTypeDepsInPlt = [{depender, []}, {type_deps, [depender]}], + ok = check_plt_deps(Config, ?FUNCTION_NAME, DependerSrc, ExpectedTypeDepsInPlt). + +check_plt_deps(Config, TestName, DependerSrc, ExpectedTypeDepsInPltUnsorted) -> + PrivDir = proplists:get_value(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(TestName) ++ ".iplt"), + {ok, DepsBeamFile} = compile(Config, type_deps, []), + {ok, DependerBeamFile} = compile(Config, DependerSrc, depender, []), + [] = run_dialyzer(incremental, [DependerBeamFile, DepsBeamFile], [{init_plt, PltFile}, {output_plt, PltFile}]), + {_ResPlt, #iplt_info{mod_deps = DepsByModule}} = dialyzer_iplt:plt_and_info_from_file(PltFile), + + ActualTypeDepsInPlt = + lists:sort(dict:to_list(dict:erase(erlang, DepsByModule))), + ExpectedTypeDepsInPlt = + lists:usort(ExpectedTypeDepsInPltUnsorted), + + ?assertEqual( + ExpectedTypeDepsInPlt, + ActualTypeDepsInPlt, + {missing, ExpectedTypeDepsInPlt -- ActualTypeDepsInPlt, + extra, ActualTypeDepsInPlt -- ExpectedTypeDepsInPlt}). + +%% Builds the named module using the source in the iplt_SUITE_data dir +compile(Config, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = proplists:get_value(priv_dir,Config), + DataDir = proplists:get_value(data_dir,Config), + SrcFilename = filename:join([DataDir, Source]), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(SrcFilename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +%% Builds the named module using the literal source given +compile(Config, Prog, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = proplists:get_value(priv_dir,Config), + Filename = filename:join([PrivDir, Source]), + ok = file:write_file(Filename, Prog), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(Filename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +run_dialyzer(Analysis, Files, Opts) -> + dialyzer:run([{analysis_type, Analysis}, + {files, Files}, + {from, byte_code} | + Opts]). + + +m_src_without_warning() -> <<" + -module(m). + -export([updt/3]). + + updt(X, K, V) -> X#{ K => V }. + ">>. + +m_src_with_warning() -> <<" + -module(m). + -export([updt/3]). + + -spec updt(list(), term(), term()) -> list(). % Warning: Spec is wrong! Function takes a map, not a list + updt(X, K, V) -> X#{ K => V }. + ">>. + +adding_warning_apps_after_a_run_without_them_causes_any_new_warnings_to_be_reported(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + Opts = [{init_plt, [PltFile]}], + OptsNoContractWarn = [{init_plt, [PltFile]}, {warnings, [no_contracts]}], + {ok, Beam} = compile(Config, m_src_with_warning(), m, []), + ?assertEqual( + [], + run_dialyzer(incremental, [Beam], OptsNoContractWarn)), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [Beam], Opts)). + +removing_warning_apps_after_a_run_with_them_causes_any_warnings_for_the_removed_apps_not_to_be_reported(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + Opts = [{init_plt, [PltFile]}], + OptsNoContractWarn = [{init_plt, [PltFile]}, {warnings, [no_contracts]}], + {ok, Beam} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [Beam], Opts)), + ?assertEqual( + [], + run_dialyzer(incremental, [Beam], OptsNoContractWarn)). + +removing_legal_warnings_with_existing_stored_warnings_in_plt_does_not_result_in_old_warnings_being_printed(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + Opts = [{init_plt, [PltFile]}], + {ok, BeamFileBefore} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [BeamFileBefore], Opts)), + {ok, BeamFileAfter} = compile(Config, m_src_without_warning(), m, []), + ?assertEqual( + [], + run_dialyzer(incremental, [BeamFileAfter], Opts)). + +adding_legal_warnings_with_existing_stored_warnings_in_plt_results_in_new_warnings_being_printed(Config) -> + PrivDir = ?config(priv_dir, Config), + PltFile = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + Opts = [{init_plt, [PltFile]}], + {ok, BeamFileAfter} = compile(Config, m_src_without_warning(), m, []), + ?assertEqual( + [], + run_dialyzer(incremental, [BeamFileAfter], Opts)), + {ok, BeamFileBefore} = compile(Config, m_src_with_warning(), m, []), + ?assertMatch( + [{warn_contract_types, + {_, {5,4}}, + {invalid_contract,[m,updt,3,{[1],true},"([any()],term(),term()) -> [any()]","(map(),_,_) -> map()"]}} + ], + run_dialyzer(incremental, [BeamFileBefore], Opts)). + +reading_from_one_plt_and_writing_to_another_does_not_mutate_the_input_plt(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + -export([bar/0]). + + -spec bar() -> ok. + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt1 = filename:join(PrivDir, "prog1.iplt"), + ?assertMatch([], dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt1}, + {from, byte_code}])), + + % Now PLT v1 should exist + Plt1ContentsBefore = dialyzer_plt:get_all_contracts(dialyzer_iplt:from_file(Plt1)), + ?assertMatch({ok, {contract,_,_,_}}, + maps:find({foo,bar,0}, Plt1ContentsBefore)), + ?assertMatch(error, + maps:find({foo,baz,0}, Plt1ContentsBefore)), + + Prog2 = + <<"-module(foo). + -export([bar/0, baz/0]). + + -spec bar() -> ok. + bar() -> ok. + + -spec baz() -> ok. + baz() -> ok.">>, + {ok, Beam2} = compile(Config, Prog2, foo, []), + + Plt2 = filename:join(PrivDir, "prog2.iplt"), + ?assertMatch([], dialyzer:run([{analysis_type, incremental}, + {files, [Beam2]}, + {init_plt, Plt1}, + {output_plt, Plt2}, + {from, byte_code}])), + + % Now PLT v1 should be the same, but PLT v2 should have the changes in it + Plt1ContentsAfter = dialyzer_plt:get_all_contracts(dialyzer_iplt:from_file(Plt1)), + ?assertMatch({ok, {contract,_,_,_}}, + maps:find({foo,bar,0}, Plt1ContentsAfter)), + ?assertMatch(error, + maps:find({foo,baz,0}, Plt1ContentsAfter)), + + Plt2Contents = dialyzer_plt:get_all_contracts(dialyzer_iplt:from_file(Plt2)), + ?assertMatch({ok, {contract,_,_,_}}, + maps:find({foo,bar,0}, Plt2Contents)), + ?assertMatch({ok, {contract,_,_,_}}, + maps:find({foo,baz,0}, Plt2Contents)). + +reading_from_and_writing_to_one_plt_mutates_it(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Prog1 = + <<"-module(foo). + -export([bar/0]). + + -spec bar() -> ok. + bar() -> ok.">>, + {ok, Beam1} = compile(Config, Prog1, foo, []), + + Plt = filename:join(PrivDir, "mutate.iplt"), + ?assertMatch([], dialyzer:run([{analysis_type, incremental}, + {files, [Beam1]}, + {init_plt, Plt}, + {from, byte_code}])), + + % Now PLT should exist after running incremental mode + PltContentsBefore = dialyzer_plt:get_all_contracts(dialyzer_iplt:from_file(Plt)), + ?assertMatch({ok, {contract,_,_,_}}, + maps:find({foo,bar,0}, PltContentsBefore)), + ?assertMatch(error, + maps:find({foo,baz,0}, PltContentsBefore)), + + Prog2 = + <<"-module(foo). + -export([bar/0, baz/0]). + + -spec bar() -> ok. + bar() -> ok. + + -spec baz() -> ok. + baz() -> ok.">>, + {ok, Beam2} = compile(Config, Prog2, foo, []), + + ?assertMatch([], dialyzer:run([{analysis_type, incremental}, + {files, [Beam2]}, + {init_plt, Plt}, + {output_plt, Plt}, + {from, byte_code}])), + + % Now PLT should have been mutated to contain the new version of module 'foo' + PltContentsAfter = dialyzer_plt:get_all_contracts(dialyzer_iplt:from_file(Plt)), + ?assertMatch({ok, {contract,_,_,_}}, maps:find({foo,bar,0}, PltContentsAfter)), + ?assertMatch({ok, {contract,_,_,_}}, maps:find({foo,baz,0}, PltContentsAfter)). + +beams_with_no_debug_info_are_rejected(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Plt = filename:join(PrivDir, atom_to_list(?FUNCTION_NAME) ++ ".iplt"), + Src = <<" + -module(my_list). + + -export([my_size/1]). + + -spec my_size(list()) -> non_neg_integer(). + my_size([]) -> + 0; + my_size({N,_}) -> + N; + my_size([_|T]) -> + 1+my_size(T). + ">>, + SrcFilename = filename:join([PrivDir, "my_list.erl"]), + ok = file:write_file(SrcFilename, Src), + Opts = [{outdir, PrivDir}], % No debug info enabled + {ok, Module} = compile:file(SrcFilename, Opts), + BeamFile = filename:join([PrivDir, lists:concat([Module, ".beam"])]), + + ?assertThrow( + {dialyzer_error, "Could not compute MD5 for .beam (debug_info error) - did you forget to set the debug_info compilation option? " ++ _}, + run_dialyzer(incremental, [BeamFile], [{output_plt, Plt}])). diff --git a/lib/dialyzer/test/iplt_SUITE_data/type_deps.erl b/lib/dialyzer/test/iplt_SUITE_data/type_deps.erl new file mode 100644 index 0000000000..f6fcfc3d23 --- /dev/null +++ b/lib/dialyzer/test/iplt_SUITE_data/type_deps.erl @@ -0,0 +1,18 @@ +-module(type_deps). + +-export([func/1, get_num/0]). + +-export_type([number_like/0, my_opaque/1, list_like/1]). + +-type number_like() :: number(). +-type list_like(X) :: [X]. +-opaque my_opaque(X) :: {X,X}. + +-callback quux(number()) -> number(). + +-spec func(T) -> T. +func(X) -> + X + X. + +get_num() -> + 3. diff --git a/lib/dialyzer/test/map_SUITE_data/results/contract_violation b/lib/dialyzer/test/map_SUITE_data/results/contract_violation index d0dd42a900..782e154100 100644 --- a/lib/dialyzer/test/map_SUITE_data/results/contract_violation +++ b/lib/dialyzer/test/map_SUITE_data/results/contract_violation @@ -1,3 +1,6 @@ contract_violation.erl:12:2: The pattern #{I:=Loc} can never match the type #{} -contract_violation.erl:16:2: Invalid type specification for function contract_violation:beam_disasm_lines/2. The success typing is ('none' | <<_:32,_:_*8>>,_) -> #{pos_integer()=>{'location',_,_}} +contract_violation.erl:16:2: Invalid type specification for function contract_violation:beam_disasm_lines/2. + The success typing is contract_violation:beam_disasm_lines('none' | <<_:32,_:_*8>>,_) -> #{pos_integer()=>{'location',_,_}} + But the spec is contract_violation:beam_disasm_lines(binary() | 'none',module()) -> lines() + The return types do not overlap diff --git a/lib/dialyzer/test/map_SUITE_data/results/opaque_key b/lib/dialyzer/test/map_SUITE_data/results/opaque_key index b70157f1af..c3df7a5560 100644 --- a/lib/dialyzer/test/map_SUITE_data/results/opaque_key +++ b/lib/dialyzer/test/map_SUITE_data/results/opaque_key @@ -1,9 +1,24 @@ -opaque_key_adt.erl:35:2: Invalid type specification for function opaque_key_adt:s2/0. The success typing is () -> #{3:='a'} -opaque_key_adt.erl:41:2: Invalid type specification for function opaque_key_adt:s4/0. The success typing is () -> #{1:='a'} -opaque_key_adt.erl:44:2: Invalid type specification for function opaque_key_adt:s5/0. The success typing is () -> #{2:=3} -opaque_key_adt.erl:56:2: Invalid type specification for function opaque_key_adt:smt1/0. The success typing is () -> #{3:='a'} -opaque_key_adt.erl:59:2: Invalid type specification for function opaque_key_adt:smt2/0. The success typing is () -> #{1:='a'} +opaque_key_adt.erl:35:2: Invalid type specification for function opaque_key_adt:s2/0. + The success typing is opaque_key_adt:s2() -> #{3:='a'} + But the spec is opaque_key_adt:s2() -> s(atom() | 3) + The return types do not overlap +opaque_key_adt.erl:41:2: Invalid type specification for function opaque_key_adt:s4/0. + The success typing is opaque_key_adt:s4() -> #{1:='a'} + But the spec is opaque_key_adt:s4() -> s(integer()) + The return types do not overlap +opaque_key_adt.erl:44:2: Invalid type specification for function opaque_key_adt:s5/0. + The success typing is opaque_key_adt:s5() -> #{2:=3} + But the spec is opaque_key_adt:s5() -> s(1) + The return types do not overlap +opaque_key_adt.erl:56:2: Invalid type specification for function opaque_key_adt:smt1/0. + The success typing is opaque_key_adt:smt1() -> #{3:='a'} + But the spec is opaque_key_adt:smt1() -> smt(1) + The return types do not overlap +opaque_key_adt.erl:59:2: Invalid type specification for function opaque_key_adt:smt2/0. + The success typing is opaque_key_adt:smt2() -> #{1:='a'} + But the spec is opaque_key_adt:smt2() -> smt(1) + The return types do not overlap opaque_key_use.erl:13:5: The test opaque_key_use:t() =:= opaque_key_use:t(_) can never evaluate to 'true' opaque_key_use.erl:24:5: Attempt to test for equality between a term of type opaque_key_adt:t(_) and a term of opaque type opaque_key_adt:t() opaque_key_use.erl:37:1: Function adt_mm1/0 has no local return diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/int b/lib/dialyzer/test/opaque_SUITE_data/results/int index 42fd95e321..504013883f 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/int +++ b/lib/dialyzer/test/opaque_SUITE_data/results/int @@ -1,3 +1,9 @@ -int_adt.erl:28:2: Invalid type specification for function int_adt:add_f/2. The success typing is (number() | int_adt:int(),float()) -> number() | int_adt:int() -int_adt.erl:32:2: Invalid type specification for function int_adt:div_f/2. The success typing is (number() | int_adt:int(),number() | int_adt:int()) -> float() +int_adt.erl:28:2: Invalid type specification for function int_adt:add_f/2. + The success typing is int_adt:add_f(number() | int_adt:int(),float()) -> number() | int_adt:int() + But the spec is int_adt:add_f(int(),int()) -> int() + They do not overlap in the 2nd argument +int_adt.erl:32:2: Invalid type specification for function int_adt:div_f/2. + The success typing is int_adt:div_f(number() | int_adt:int(),number() | int_adt:int()) -> float() + But the spec is int_adt:div_f(int(),int()) -> int() + The return types do not overlap diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/multiple_wrong_opaques b/lib/dialyzer/test/opaque_SUITE_data/results/multiple_wrong_opaques index fd702bf1d6..0130be07b7 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/multiple_wrong_opaques +++ b/lib/dialyzer/test/opaque_SUITE_data/results/multiple_wrong_opaques @@ -1,2 +1,5 @@ -multiple_wrong_opaques.erl:5:2: Invalid type specification for function multiple_wrong_opaques:weird/1. The success typing is ('gazonk') -> 42 +multiple_wrong_opaques.erl:5:2: Invalid type specification for function multiple_wrong_opaques:weird/1. + The success typing is multiple_wrong_opaques:weird('gazonk') -> 42 + But the spec is multiple_wrong_opaques:weird(dict:dict() | gb_trees:tree()) -> 42 + They do not overlap in the 1st argument diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/para b/lib/dialyzer/test/opaque_SUITE_data/results/para index 0ba2a24996..77106c6afa 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/para +++ b/lib/dialyzer/test/opaque_SUITE_data/results/para @@ -12,13 +12,22 @@ para2.erl:31:5: The test 'a' =:= 'b' can never evaluate to 'true' para2.erl:61:5: Attempt to test for equality between a term of type para2_adt:c2() and a term of opaque type para2_adt:c1() para2.erl:66:5: The test 'a' =:= 'b' can never evaluate to 'true' para2.erl:88:5: The test para2:circ(_) =:= para2:circ(_,_) can never evaluate to 'true' -para3.erl:28:2: Invalid type specification for function para3:ot2/0. The success typing is () -> 'foo' +para3.erl:28:2: Invalid type specification for function para3:ot2/0. + The success typing is para3:ot2() -> 'foo' + But the spec is para3:ot2() -> ot1() + The return types do not overlap para3.erl:36:5: The pattern {{{17}}} can never match the type {{{{{{_,_,_,_,_}}}}}} -para3.erl:55:2: Invalid type specification for function para3:t2/0. The success typing is () -> 'foo' +para3.erl:55:2: Invalid type specification for function para3:t2/0. + The success typing is para3:t2() -> 'foo' + But the spec is para3:t2() -> t1() + The return types do not overlap para3.erl:65:5: The attempt to match a term of type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} against the pattern {{{{{17}}}}} breaks the opacity of para3_adt:ot1(_,_,_,_,_) para3.erl:68:5: The pattern {{{{17}}}} can never match the type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} para3.erl:74:2: The specification for para3:exp_adt/0 has an opaque subtype para3_adt:exp1(_) which is violated by the success typing () -> 3 -para4.erl:31:2: Invalid type specification for function para4:t/1. The success typing is (para4:d_all() | para4:d_tuple()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:31:2: Invalid type specification for function para4:t/1. + The success typing is para4:t(para4:d_all() | para4:d_tuple()) -> [{atom() | integer(),atom() | integer()}] + But the spec is para4:t(d_tuple()) -> [{tuple(),tuple()}] + The return types do not overlap para4.erl:79:5: The test para4_adt:int(_) =:= para4_adt:int(_) can never evaluate to 'true' para5.erl:13:5: Attempt to test for inequality between a term of type para5_adt:dd(_) and a term of opaque type para5_adt:d() para5.erl:8:5: The test para5_adt:d() =:= para5_adt:d() can never evaluate to 'true' diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/simple b/lib/dialyzer/test/opaque_SUITE_data/results/simple index 4959d14f15..4c211a4425 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/simple +++ b/lib/dialyzer/test/opaque_SUITE_data/results/simple @@ -21,7 +21,10 @@ rec_api.erl:123:5: The attempt to match a term of type #r3{f1::10} against the p rec_api.erl:24:18: Record construction #r1{f1::10} violates the declared type of field f1::rec_api:a() rec_api.erl:29:5: Matching of pattern {'r1', 10} tagged with a record name violates the declared type of #r1{f1::10} rec_api.erl:33:5: The attempt to match a term of type rec_adt:r1() against the pattern {'r1', 'a'} breaks the opacity of the term -rec_api.erl:35:2: Invalid type specification for function rec_api:adt_t1/1. The success typing is (#r1{f1::'a'}) -> #r1{f1::'a'} +rec_api.erl:35:2: Invalid type specification for function rec_api:adt_t1/1. + The success typing is rec_api:adt_t1(#r1{f1::'a'}) -> #r1{f1::'a'} + But the spec is rec_api:adt_t1(rec_adt:r1()) -> rec_adt:r1() + They do not overlap in the 1st argument, and the return types do not overlap rec_api.erl:40:2: The specification for rec_api:adt_r1/0 has an opaque subtype rec_adt:r1() which is violated by the success typing () -> #r1{f1::'a'} rec_api.erl:85:13: The attempt to match a term of type rec_adt:f() against the record field 'f' declared to be of type rec_api:f() breaks the opacity of the term rec_api.erl:99:18: Record construction #r2{f1::10} violates the declared type of field f1::rec_api:a() @@ -55,11 +58,20 @@ simple1_api.erl:319:16: Guard test not(and('true','true')) can never succeed simple1_api.erl:337:8: Clause guard cannot succeed. simple1_api.erl:342:8: Guard test B::simple1_adt:b2() =:= 'true' contains an opaque term as 1st argument simple1_api.erl:347:8: Guard test A::simple1_adt:b1() =:= 'true' contains an opaque term as 1st argument -simple1_api.erl:355:2: Invalid type specification for function simple1_api:bool_adt_t6/1. The success typing is ('true') -> 1 +simple1_api.erl:355:2: Invalid type specification for function simple1_api:bool_adt_t6/1. + The success typing is simple1_api:bool_adt_t6('true') -> 1 + But the spec is simple1_api:bool_adt_t6(simple1_adt:b1()) -> integer() + They do not overlap in the 1st argument simple1_api.erl:365:8: Clause guard cannot succeed. -simple1_api.erl:368:2: Invalid type specification for function simple1_api:bool_adt_t8/2. The success typing is (boolean(),boolean()) -> 1 +simple1_api.erl:368:2: Invalid type specification for function simple1_api:bool_adt_t8/2. + The success typing is simple1_api:bool_adt_t8(boolean(),boolean()) -> 1 + But the spec is simple1_api:bool_adt_t8(simple1_adt:b1(),simple1_adt:b2()) -> integer() + They do not overlap in the 1st and 2nd arguments simple1_api.erl:378:8: Clause guard cannot succeed. -simple1_api.erl:381:2: Invalid type specification for function simple1_api:bool_adt_t9/2. The success typing is ('false','false') -> 1 +simple1_api.erl:381:2: Invalid type specification for function simple1_api:bool_adt_t9/2. + The success typing is simple1_api:bool_adt_t9('false','false') -> 1 + But the spec is simple1_api:bool_adt_t9(simple1_adt:b1(),simple1_adt:b2()) -> integer() + They do not overlap in the 1st and 2nd arguments simple1_api.erl:407:12: The size simple1_adt:i1() breaks the opacity of A simple1_api.erl:418:9: The attempt to match a term of type non_neg_integer() against the variable A breaks the opacity of simple1_adt:i1() simple1_api.erl:425:9: The attempt to match a term of type non_neg_integer() against the variable B breaks the opacity of simple1_adt:i1() diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/recrec/dialyzer.hrl b/lib/dialyzer/test/opaque_SUITE_data/src/recrec/dialyzer.hrl index c0f287893e..33b61b1519 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/recrec/dialyzer.hrl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/recrec/dialyzer.hrl @@ -99,7 +99,7 @@ %%-------------------------------------------------------------------- -type anal_type() :: 'succ_typings' | 'plt_build'. --type anal_type1() :: anal_type() | 'plt_add' | 'plt_check' | 'plt_remove'. +-type anal_type1() :: anal_type() | 'plt_add' | 'plt_check' | 'plt_remove' | 'incremental'. -type contr_constr() :: {'subtype', erl_types:erl_type(), erl_types:erl_type()}. -type contract_pair() :: {erl_types:erl_type(), [contr_constr()]}. -type dial_define() :: {atom(), term()}. @@ -134,6 +134,7 @@ timing = false :: boolean() | 'debug', timing_server = none :: dialyzer_timing:timing_server(), callgraph_file = "" :: file:filename(), + mod_deps_file = "" :: file:filename(), solvers :: [solver()]}). -record(options, {files = [] :: [file:filename()], @@ -154,6 +155,7 @@ output_format = formatted :: format(), filename_opt = basename :: fopt(), callgraph_file = "" :: file:filename(), + mod_deps_file = "" :: file:filename(), check_plt = true :: boolean(), solvers = [] :: [solver()]}). diff --git a/lib/dialyzer/test/options2_SUITE_data/results/unknown_function b/lib/dialyzer/test/options2_SUITE_data/results/unknown_function new file mode 100644 index 0000000000..82bb96a7aa --- /dev/null +++ b/lib/dialyzer/test/options2_SUITE_data/results/unknown_function @@ -0,0 +1,3 @@ + +unknown_function.erl:10:5: Call to missing or unexported function unknown_function:function/0 +unknown_function.erl:14:5: Unknown function unknown:function/0 diff --git a/lib/dialyzer/test/options2_SUITE_data/src/unknown_function.erl b/lib/dialyzer/test/options2_SUITE_data/src/unknown_function.erl new file mode 100644 index 0000000000..a935e4cd73 --- /dev/null +++ b/lib/dialyzer/test/options2_SUITE_data/src/unknown_function.erl @@ -0,0 +1,14 @@ +-module(unknown_function). + +-export([ + unknown_function_on_unknown_module/0, + unknown_function_on_known_module/0 + ]). + +-spec unknown_function_on_known_module() -> ok. +unknown_function_on_known_module() -> + unknown_function:function(). + +-spec unknown_function_on_unknown_module() -> ok. +unknown_function_on_unknown_module() -> + unknown:function(). diff --git a/lib/dialyzer/test/small_SUITE_data/results/behaviour_info b/lib/dialyzer/test/small_SUITE_data/results/behaviour_info index 6497ddae80..ea17933586 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/behaviour_info +++ b/lib/dialyzer/test/small_SUITE_data/results/behaviour_info @@ -1,2 +1,2 @@ -with_bad_format_status.erl:12:1: The inferred type for the 1st argument of format_status/2 ('bad_arg') is not a supertype of 'normal' | 'terminate', which is expected type for this argument in the callback of the gen_server behaviour +with_bad_format_status.erl:12:1: The inferred type for the 1st argument of format_status/2 ('bad_arg') has nothing in common with 'normal' | 'terminate', which is expected type for this argument in the callback of the gen_server behaviour diff --git a/lib/dialyzer/test/small_SUITE_data/results/binary_nonempty b/lib/dialyzer/test/small_SUITE_data/results/binary_nonempty index 5275482a59..dbfaf63d6e 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/binary_nonempty +++ b/lib/dialyzer/test/small_SUITE_data/results/binary_nonempty @@ -1,16 +1,37 @@ binary_nonempty.erl:12:1: Function t2/0 has no local return binary_nonempty.erl:13:8: The call binary_nonempty:t2(<<>>) breaks the contract (nonempty_binary()) -> 'foo' -binary_nonempty.erl:15:2: Invalid type specification for function binary_nonempty:t2/1. The success typing is (<<>>) -> 'foo' +binary_nonempty.erl:15:2: Invalid type specification for function binary_nonempty:t2/1. + The success typing is binary_nonempty:t2(<<>>) -> 'foo' + But the spec is binary_nonempty:t2(nonempty_binary()) -> 'foo' + They do not overlap in the 1st argument binary_nonempty.erl:19:1: Function t3/0 has no local return binary_nonempty.erl:20:8: The call binary_nonempty:t3(<<>>) breaks the contract (<<_:1,_:_*1>>) -> 'foo' -binary_nonempty.erl:22:2: Invalid type specification for function binary_nonempty:t3/1. The success typing is (<<>>) -> 'foo' +binary_nonempty.erl:22:2: Invalid type specification for function binary_nonempty:t3/1. + The success typing is binary_nonempty:t3(<<>>) -> 'foo' + But the spec is binary_nonempty:t3(<<_:1,_:_*1>>) -> 'foo' + They do not overlap in the 1st argument binary_nonempty.erl:26:1: Function t4/0 has no local return binary_nonempty.erl:27:8: The call binary_nonempty:t4(<<>>) breaks the contract (<<_:8,_:_*8>>) -> 'foo' -binary_nonempty.erl:29:2: Invalid type specification for function binary_nonempty:t4/1. The success typing is (<<>>) -> 'foo' -binary_nonempty.erl:33:2: Invalid type specification for function binary_nonempty:t5/1. The success typing is (<<>>) -> 'foo' -binary_nonempty.erl:38:2: Invalid type specification for function binary_nonempty:t6/1. The success typing is (<<_:8>>) -> 'foo' -binary_nonempty.erl:43:2: Invalid type specification for function binary_nonempty:t7/1. The success typing is (<<_:1>>) -> 'foo' +binary_nonempty.erl:29:2: Invalid type specification for function binary_nonempty:t4/1. + The success typing is binary_nonempty:t4(<<>>) -> 'foo' + But the spec is binary_nonempty:t4(<<_:8,_:_*8>>) -> 'foo' + They do not overlap in the 1st argument +binary_nonempty.erl:33:2: Invalid type specification for function binary_nonempty:t5/1. + The success typing is binary_nonempty:t5(<<>>) -> 'foo' + But the spec is binary_nonempty:t5(nonempty_binary()) -> 'foo' + They do not overlap in the 1st argument +binary_nonempty.erl:38:2: Invalid type specification for function binary_nonempty:t6/1. + The success typing is binary_nonempty:t6(<<_:8>>) -> 'foo' + But the spec is binary_nonempty:t6(<<>>) -> 'foo' + They do not overlap in the 1st argument +binary_nonempty.erl:43:2: Invalid type specification for function binary_nonempty:t7/1. + The success typing is binary_nonempty:t7(<<_:1>>) -> 'foo' + But the spec is binary_nonempty:t7(<<>>) -> 'foo' + They do not overlap in the 1st argument binary_nonempty.erl:5:1: Function t1/0 has no local return binary_nonempty.erl:6:8: The call binary_nonempty:t1(<<>>) breaks the contract (nonempty_bitstring()) -> 'foo' -binary_nonempty.erl:8:2: Invalid type specification for function binary_nonempty:t1/1. The success typing is (<<>>) -> 'foo' +binary_nonempty.erl:8:2: Invalid type specification for function binary_nonempty:t1/1. + The success typing is binary_nonempty:t1(<<>>) -> 'foo' + But the spec is binary_nonempty:t1(nonempty_bitstring()) -> 'foo' + They do not overlap in the 1st argument diff --git a/lib/dialyzer/test/small_SUITE_data/results/binary_redef2 b/lib/dialyzer/test/small_SUITE_data/results/binary_redef2 index 71968b801b..19559b6dfb 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/binary_redef2 +++ b/lib/dialyzer/test/small_SUITE_data/results/binary_redef2 @@ -1,3 +1,9 @@ -binary_redef2.erl:15:2: Invalid type specification for function binary_redef2:t1/1. The success typing is (3) -> 6 -binary_redef2.erl:20:2: Invalid type specification for function binary_redef2:new/0. The success typing is () -> 3 +binary_redef2.erl:15:2: Invalid type specification for function binary_redef2:t1/1. + The success typing is binary_redef2:t1(3) -> 6 + But the spec is binary_redef2:t1(nonempty_bitstring()) -> nonempty_bitstring() + They do not overlap in the 1st argument, and the return types do not overlap +binary_redef2.erl:20:2: Invalid type specification for function binary_redef2:new/0. + The success typing is binary_redef2:new() -> 3 + But the spec is binary_redef2:new() -> nonempty_binary() + The return types do not overlap diff --git a/lib/dialyzer/test/small_SUITE_data/results/chars b/lib/dialyzer/test/small_SUITE_data/results/chars index ec7b468e43..a91e21d181 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/chars +++ b/lib/dialyzer/test/small_SUITE_data/results/chars @@ -1,4 +1,7 @@ -chars.erl:37:2: Invalid type specification for function chars:f/1. The success typing is (#{'b':=50}) -> 'ok' +chars.erl:37:2: Invalid type specification for function chars:f/1. + The success typing is chars:f(#{'b':=50}) -> 'ok' + But the spec is chars:f(#{'a':=49,'b'=>50,'c'=>51}) -> 'ok' + They do not overlap in the 1st argument chars.erl:40:11: The call chars:f(#{'b'=>50}) breaks the contract (#{'a':=49,'b'=>50,'c'=>51}) -> 'ok' chars.erl:40:1: Function t1/0 has no local return diff --git a/lib/dialyzer/test/small_SUITE_data/results/contract5 b/lib/dialyzer/test/small_SUITE_data/results/contract5 index 10ea8ca362..9ffccbc19b 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contract5 +++ b/lib/dialyzer/test/small_SUITE_data/results/contract5 @@ -1,2 +1,5 @@ -contract5.erl:13:2: Invalid type specification for function contract5:t/0. The success typing is () -> #bar{baz::'not_a_boolean'} +contract5.erl:13:2: Invalid type specification for function contract5:t/0. + The success typing is contract5:t() -> #bar{baz::'not_a_boolean'} + But the spec is contract5:t() -> #bar{baz::boolean()} + The return types do not overlap diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes index 44fd6056bd..8645aa9078 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes @@ -20,7 +20,10 @@ contracts_with_subtypes.erl:218:2: The pattern 42 can never match the type {'ok' contracts_with_subtypes.erl:235:3: The pattern 1 can never match the type string() contracts_with_subtypes.erl:238:2: The pattern {'ok', _} can never match the type {'ok',_,string()} contracts_with_subtypes.erl:239:2: The pattern 'alpha' can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:23:2: Invalid type specification for function contracts_with_subtypes:extract2/0. The success typing is () -> 'something' +contracts_with_subtypes.erl:23:2: Invalid type specification for function contracts_with_subtypes:extract2/0. + The success typing is contracts_with_subtypes:extract2() -> 'something' + But the spec is contracts_with_subtypes:extract2() -> 'ok' + The return types do not overlap contracts_with_subtypes.erl:240:2: The pattern {'ok', 42} can never match the type {'ok',_,string()} contracts_with_subtypes.erl:241:2: The pattern 42 can never match the type {'ok',_,string()} contracts_with_subtypes.erl:267:1: Function flat_ets_new_t/0 has no local return @@ -30,7 +33,10 @@ contracts_with_subtypes.erl:295:22: The call contracts_with_subtypes:factored_et contracts_with_subtypes.erl:77:16: The call contracts_with_subtypes:foo1(5) breaks the contract (Arg1) -> Res when Arg1 :: atom(), Res :: atom() contracts_with_subtypes.erl:78:16: The call contracts_with_subtypes:foo2(5) breaks the contract (Arg1) -> Res when Arg1 :: Arg2, Arg2 :: atom(), Res :: atom() contracts_with_subtypes.erl:79:16: The call contracts_with_subtypes:foo3(5) breaks the contract (Arg1) -> Res when Arg2 :: atom(), Arg1 :: Arg2, Res :: atom() -contracts_with_subtypes.erl:7:2: Invalid type specification for function contracts_with_subtypes:extract/0. The success typing is () -> 'something' +contracts_with_subtypes.erl:7:2: Invalid type specification for function contracts_with_subtypes:extract/0. + The success typing is contracts_with_subtypes:extract() -> 'something' + But the spec is contracts_with_subtypes:extract() -> 'ok' + The return types do not overlap contracts_with_subtypes.erl:80:16: The call contracts_with_subtypes:foo4(5) breaks the contract (Type) -> Type when Type :: atom() contracts_with_subtypes.erl:81:16: The call contracts_with_subtypes:foo5(5) breaks the contract (Type::atom()) -> Type::atom() contracts_with_subtypes.erl:82:16: The call contracts_with_subtypes:foo6(5) breaks the contract (Type) -> Type when Type :: atom() diff --git a/lib/dialyzer/test/small_SUITE_data/results/empty_list_infimum b/lib/dialyzer/test/small_SUITE_data/results/empty_list_infimum index cf44c15458..b53b251a39 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/empty_list_infimum +++ b/lib/dialyzer/test/small_SUITE_data/results/empty_list_infimum @@ -1,2 +1,5 @@ -empty_list_infimum.erl:38:2: Invalid type specification for function empty_list_infimum:list_vhost_permissions/1. The success typing is (_) -> [[{_,_}]] +empty_list_infimum.erl:38:2: Invalid type specification for function empty_list_infimum:list_vhost_permissions/1. + The success typing is empty_list_infimum:list_vhost_permissions(_) -> [[{_,_}]] + But the spec is empty_list_infimum:list_vhost_permissions(vhost()) -> infos() + The return types do not overlap diff --git a/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 index bfada119a2..a8026b787f 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 +++ b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 @@ -1,2 +1,5 @@ -scala_user.erl:5:2: Invalid type specification for function scala_user:is_list/2. The success typing is (maybe_improper_list() | tuple(),_) -> boolean() +scala_user.erl:5:2: Invalid type specification for function scala_user:is_list/2. + The success typing is scala_user:is_list(maybe_improper_list() | tuple(),_) -> boolean() + But the spec is scala_user:is_list(atom(),scala_data:data()) -> boolean() + They do not overlap in the 1st argument diff --git a/lib/dialyzer/test/small_SUITE_data/results/invalid_specs b/lib/dialyzer/test/small_SUITE_data/results/invalid_specs index 0de8f0fcb4..306be3f76a 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/invalid_specs +++ b/lib/dialyzer/test/small_SUITE_data/results/invalid_specs @@ -1,3 +1,6 @@ -invalid_spec1.erl:5:2: Invalid type specification for function invalid_spec1:get_plan_dirty/1. The success typing is ([string()]) -> {maybe_improper_list(),[atom()]} +invalid_spec1.erl:5:2: Invalid type specification for function invalid_spec1:get_plan_dirty/1. + The success typing is invalid_spec1:get_plan_dirty([string()]) -> {maybe_improper_list(),[atom()]} + But the spec is invalid_spec1:get_plan_dirty([string()]) -> {{atom(),any()},[atom()]} + The return types do not overlap invalid_spec2.erl:5:1: Function foo/0 has no local return diff --git a/lib/dialyzer/test/small_SUITE_data/results/maps_sum b/lib/dialyzer/test/small_SUITE_data/results/maps_sum index 83e7c73ef2..df2a90387b 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/maps_sum +++ b/lib/dialyzer/test/small_SUITE_data/results/maps_sum @@ -1,4 +1,7 @@ -maps_sum.erl:15:2: Invalid type specification for function maps_sum:wrong1/1. The success typing is (maps:iterator(_,_) | map()) -> any() +maps_sum.erl:15:2: Invalid type specification for function maps_sum:wrong1/1. + The success typing is maps_sum:wrong1(maps:iterator(_,_) | map()) -> any() + But the spec is maps_sum:wrong1([{atom(),term()}]) -> integer() + They do not overlap in the 1st argument maps_sum.erl:26:1: Function wrong2/1 has no local return maps_sum.erl:27:17: The call lists:foldl(fun((_,_,_) -> any()),0,Data::any()) will never return since it differs in the 1st argument from the success typing arguments: (fun((_,_) -> any()),any(),[any()]) diff --git a/lib/dialyzer/test/small_SUITE_data/results/predef b/lib/dialyzer/test/small_SUITE_data/results/predef index f57f78d59e..e89dd8db87 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/predef +++ b/lib/dialyzer/test/small_SUITE_data/results/predef @@ -1,8 +1,29 @@ -predef.erl:19:2: Invalid type specification for function predef:array/1. The success typing is (array:array(_)) -> array:array(_) -predef.erl:24:2: Invalid type specification for function predef:dict/1. The success typing is (dict:dict(_,_)) -> dict:dict(_,_) -predef.erl:29:2: Invalid type specification for function predef:digraph/1. The success typing is (digraph:graph()) -> [any()] -predef.erl:39:2: Invalid type specification for function predef:gb_set/1. The success typing is (gb_sets:set(_)) -> gb_sets:set(_) -predef.erl:44:2: Invalid type specification for function predef:gb_tree/1. The success typing is (gb_trees:tree(_,_)) -> gb_trees:tree(_,_) -predef.erl:49:2: Invalid type specification for function predef:queue/1. The success typing is (queue:queue(_)) -> queue:queue(_) -predef.erl:54:2: Invalid type specification for function predef:set/1. The success typing is (sets:set(_)) -> sets:set(_) +predef.erl:19:2: Invalid type specification for function predef:array/1. + The success typing is predef:array(array:array(_)) -> array:array(_) + But the spec is predef:array(array()) -> array:array() + They do not overlap in the 1st argument +predef.erl:24:2: Invalid type specification for function predef:dict/1. + The success typing is predef:dict(dict:dict(_,_)) -> dict:dict(_,_) + But the spec is predef:dict(dict()) -> dict:dict() + They do not overlap in the 1st argument +predef.erl:29:2: Invalid type specification for function predef:digraph/1. + The success typing is predef:digraph(digraph:graph()) -> [any()] + But the spec is predef:digraph(digraph()) -> [digraph:edge()] + They do not overlap in the 1st argument +predef.erl:39:2: Invalid type specification for function predef:gb_set/1. + The success typing is predef:gb_set(gb_sets:set(_)) -> gb_sets:set(_) + But the spec is predef:gb_set(gb_set()) -> gb_sets:set() + They do not overlap in the 1st argument +predef.erl:44:2: Invalid type specification for function predef:gb_tree/1. + The success typing is predef:gb_tree(gb_trees:tree(_,_)) -> gb_trees:tree(_,_) + But the spec is predef:gb_tree(gb_tree()) -> gb_trees:tree() + They do not overlap in the 1st argument +predef.erl:49:2: Invalid type specification for function predef:queue/1. + The success typing is predef:queue(queue:queue(_)) -> queue:queue(_) + But the spec is predef:queue(queue()) -> queue:queue() + They do not overlap in the 1st argument +predef.erl:54:2: Invalid type specification for function predef:set/1. + The success typing is predef:set(sets:set(_)) -> sets:set(_) + But the spec is predef:set(set()) -> sets:set() + They do not overlap in the 1st argument diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_update b/lib/dialyzer/test/small_SUITE_data/results/record_update index b61d2e66b3..d2747d0440 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/record_update +++ b/lib/dialyzer/test/small_SUITE_data/results/record_update @@ -1,2 +1,5 @@ -record_update.erl:7:2: Invalid type specification for function record_update:quux/2. The success typing is (#foo{bar::atom()},atom()) -> #foo{bar::atom()} +record_update.erl:7:2: Invalid type specification for function record_update:quux/2. + The success typing is record_update:quux(#foo{bar::atom()},atom()) -> #foo{bar::atom()} + But the spec is record_update:quux(#foo{},string()) -> #foo{} + They do not overlap in the 2nd argument diff --git a/lib/dialyzer/test/small_SUITE_data/results/redefine_builtin_type b/lib/dialyzer/test/small_SUITE_data/results/redefine_builtin_type new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/redefine_builtin_type diff --git a/lib/dialyzer/test/small_SUITE_data/results/redefine_builtins b/lib/dialyzer/test/small_SUITE_data/results/redefine_builtins new file mode 100644 index 0000000000..c7da0cf72a --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/redefine_builtins @@ -0,0 +1,5 @@ + +a.erl:4:2: Invalid type specification for function a:vi/1. + The success typing is a:vi(integer()) -> 'ok' + But the spec is a:vi(b:integer()) -> 'ok' + They do not overlap in the 1st argument diff --git a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash index 4d72467a06..9e415b469b 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash +++ b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash @@ -1,12 +1,30 @@ -tuple_set_crash.erl:103:2: Invalid type specification for function tuple_set_crash:parse_device_properties/1. The success typing is (<<_:48>>) -> [{'controller_description',binary()} | {'controller_name',binary()} | {'controller_status',byte()} | {'fw_version',<<_:24>>}] -tuple_set_crash.erl:123:2: Invalid type specification for function tuple_set_crash:parse_video_target_info/1. The success typing is (<<_:48>>) -> [{'status',byte()} | {'target_id',non_neg_integer()},...] -tuple_set_crash.erl:127:2: Invalid type specification for function tuple_set_crash:parse_audio_target_info/1. The success typing is (<<_:48>>) -> [{'master_volume',char()} | {'status',byte()} | {'target_id',non_neg_integer()},...] -tuple_set_crash.erl:138:2: Invalid type specification for function tuple_set_crash:parse_av_device_info/1. The success typing is (<<_:48>>) -> [{'address',byte()} | {'device_id',non_neg_integer()} | {'model',binary()} | {'status',byte()},...] +tuple_set_crash.erl:103:2: Invalid type specification for function tuple_set_crash:parse_device_properties/1. + The success typing is tuple_set_crash:parse_device_properties(<<_:48>>) -> [{'controller_description',binary()} | {'controller_name',binary()} | {'controller_status',byte()} | {'fw_version',<<_:24>>}] + But the spec is tuple_set_crash:parse_device_properties(binary()) -> config_change() + The return types do not overlap +tuple_set_crash.erl:123:2: Invalid type specification for function tuple_set_crash:parse_video_target_info/1. + The success typing is tuple_set_crash:parse_video_target_info(<<_:48>>) -> [{'status',byte()} | {'target_id',non_neg_integer()},...] + But the spec is tuple_set_crash:parse_video_target_info(binary()) -> config_change() + The return types do not overlap +tuple_set_crash.erl:127:2: Invalid type specification for function tuple_set_crash:parse_audio_target_info/1. + The success typing is tuple_set_crash:parse_audio_target_info(<<_:48>>) -> [{'master_volume',char()} | {'status',byte()} | {'target_id',non_neg_integer()},...] + But the spec is tuple_set_crash:parse_audio_target_info(binary()) -> [config_change()] + The return types do not overlap +tuple_set_crash.erl:138:2: Invalid type specification for function tuple_set_crash:parse_av_device_info/1. + The success typing is tuple_set_crash:parse_av_device_info(<<_:48>>) -> [{'address',byte()} | {'device_id',non_neg_integer()} | {'model',binary()} | {'status',byte()},...] + But the spec is tuple_set_crash:parse_av_device_info(binary()) -> [config_change()] + The return types do not overlap tuple_set_crash.erl:141:25: The pattern <<TargetId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> -tuple_set_crash.erl:155:2: Invalid type specification for function tuple_set_crash:parse_video_output_info/1. The success typing is (<<_:48>>) -> [{'audio_volume',char()} | {'display_type',binary()} | {'output_id',non_neg_integer()},...] +tuple_set_crash.erl:155:2: Invalid type specification for function tuple_set_crash:parse_video_output_info/1. + The success typing is tuple_set_crash:parse_video_output_info(<<_:48>>) -> [{'audio_volume',char()} | {'display_type',binary()} | {'output_id',non_neg_integer()},...] + But the spec is tuple_set_crash:parse_video_output_info(binary()) -> [config_change()] + The return types do not overlap tuple_set_crash.erl:158:25: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> -tuple_set_crash.erl:171:2: Invalid type specification for function tuple_set_crash:parse_audio_output_info/1. The success typing is (<<_:48>>) -> [{'output_id',non_neg_integer()},...] +tuple_set_crash.erl:171:2: Invalid type specification for function tuple_set_crash:parse_audio_output_info/1. + The success typing is tuple_set_crash:parse_audio_output_info(<<_:48>>) -> [{'output_id',non_neg_integer()},...] + But the spec is tuple_set_crash:parse_audio_output_info(binary()) -> [config_change()] + The return types do not overlap tuple_set_crash.erl:174:25: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> tuple_set_crash.erl:177:25: The pattern <<AudioVolume:16/integer-little-unit:1,Rest2/binary>> can never match the type <<_:8>> tuple_set_crash.erl:180:25: The pattern <<Delay:16/integer-little-unit:1,_Padding/binary>> can never match the type <<_:8>> diff --git a/lib/dialyzer/test/small_SUITE_data/results/types_arity b/lib/dialyzer/test/small_SUITE_data/results/types_arity index fae7455996..9842bad61b 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/types_arity +++ b/lib/dialyzer/test/small_SUITE_data/results/types_arity @@ -1,2 +1,5 @@ -types_arity.erl:16:2: Invalid type specification for function types_arity:test2/0. The success typing is () -> {'node','a','nil','nil'} +types_arity.erl:16:2: Invalid type specification for function types_arity:test2/0. + The success typing is types_arity:test2() -> {'node','a','nil','nil'} + But the spec is types_arity:test2() -> tree() + The return types do not overlap diff --git a/lib/dialyzer/test/small_SUITE_data/src/redefine_builtin_type.erl b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtin_type.erl new file mode 100644 index 0000000000..9cf80cafb6 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtin_type.erl @@ -0,0 +1,24 @@ +-module(redefine_builtin_type). +-export([lookup/2, verify_mfa/1, verify_pid/1]). + +-type map() :: {atom(), erlang:map()}. + +-spec lookup(atom(), map()) -> {'ok', term()} | 'error'. + +lookup(Key, {Key, Map}) when is_atom(Key), is_map(Map) -> + {ok, Map}; +lookup(Key1, {Key2, Map}) when is_atom(Key1), is_atom(Key2), is_map(Map) -> + error. + +%% Type `mfa()` depends on `erlang::module()`. Make sure that `mfa()` +%% does not attempt to use our local definition of `module()`. + +-type module() :: pid(). + +-spec verify_mfa(mfa()) -> 'ok'. +verify_mfa({M, F, A}) when is_atom(M), is_atom(F), is_integer(A) -> + ok. + +-spec verify_pid(module()) -> 'ok'. +verify_pid(Pid) when is_pid(Pid) -> + ok. diff --git a/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/a.erl b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/a.erl new file mode 100644 index 0000000000..274906d554 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/a.erl @@ -0,0 +1,14 @@ +-module(a). +-export([vi/1, sum/2, vc/1]). + +-spec vi(b:integer()) -> 'ok'. +vi(I) when is_integer(I) -> + ok. + +-spec sum(b:integer(), integer()) -> integer(). +sum([A], B) -> + A + B. + +-spec vc(b:collection()) -> 'ok'. +vc({Int, List}) when length(List) =:= Int -> + ok. diff --git a/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/b.erl b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/b.erl new file mode 100644 index 0000000000..c11f591036 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/redefine_builtins/b.erl @@ -0,0 +1,6 @@ +-module(b). +-export_type([integer/0, collection/0]). + +-type integer() :: [integer()]. + +-type collection() :: {erlang:integer(), integer()}. diff --git a/lib/dialyzer/test/typer_SUITE.erl b/lib/dialyzer/test/typer_SUITE.erl index da5b961643..661357d383 100644 --- a/lib/dialyzer/test/typer_SUITE.erl +++ b/lib/dialyzer/test/typer_SUITE.erl @@ -19,37 +19,78 @@ %% -module(typer_SUITE). --export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, - init_per_group/2,end_per_group/2, - smoke/1]). +-export([all/0,suite/0, + smoke/1, + smoke_incremental_plt/1, + gh_6296_no_spec_flag_does_not_break_records/1, + contract_violation/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [smoke]. + [smoke, + smoke_incremental_plt, + gh_6296_no_spec_flag_does_not_break_records, + contract_violation]. -groups() -> - []. - -init_per_suite(Config) -> +smoke(Config) -> OutDir = proplists:get_value(priv_dir, Config), case dialyzer_common:check_plt(OutDir) of fail -> {skip, "Plt creation/check failed."}; - ok -> [{dialyzer_options, []}|Config] + ok -> + Code = <<"-module(typer_test_module). + -compile([export_all,nowarn_export_all]). + a(L) -> + L ++ [1,2,3].">>, + PrivDir = proplists:get_value(priv_dir, Config), + Src = filename:join(PrivDir, "typer_test_module.erl"), + ok = file:write_file(Src, Code), + Args = "--plt " ++ PrivDir ++ "dialyzer_plt", + Res = ["^$", + "^%% File:", + "^%% ----", + "^-spec a", + "^_OK_"], + run(Config, Args, Src, Res), + ok end. -end_per_suite(_Config) -> - ok. +gh_6296_no_spec_flag_does_not_break_records(Config) -> + Code = <<"-module(gh_6296). + -export([record_pattern/1]). -init_per_group(_GroupName, Config) -> - Config. + -record(my_rec, {is_foo :: boolean(), + bar :: non_neg_integer()}). -end_per_group(_GroupName, Config) -> - Config. + -spec record_pattern(#my_rec{}) -> atom(). + record_pattern(#my_rec{is_foo = IsFoo}) -> + IsFoo. -smoke(Config) -> + some_other_function_to_trigger_the_issue(undefined) -> ok.">>, + PrivDir = proplists:get_value(priv_dir, Config), + Src = filename:join(PrivDir, "gh_6296.erl"), + ok = file:write_file(Src, Code), + {ok, Beam} = compile(Config, Code, gh_6296, []), + Plt = PrivDir ++ "dialyzer_iplt", + _ = dialyzer:run([{analysis_type, incremental}, + {files, [Beam]}, + {apps, [stdlib, kernel, erts]}, + {from, byte_code}, + {init_plt, Plt}, + {output_plt, Plt}]), + Args = io_lib:format("--no_spec --show --plt ~ts", [Plt]), + Res = ["^$", + "^%% File:", + "^%% ----", + "^-spec record_pattern", + "^-spec some_other_function_to_trigger_the_issue", + "^_OK_"], + run(Config, Args, Src, Res), + ok. + +smoke_incremental_plt(Config) -> Code = <<"-module(typer_test_module). -compile([export_all,nowarn_export_all]). a(L) -> @@ -57,7 +98,15 @@ smoke(Config) -> PrivDir = proplists:get_value(priv_dir, Config), Src = filename:join(PrivDir, "typer_test_module.erl"), ok = file:write_file(Src, Code), - Args = "--plt " ++ PrivDir ++ "dialyzer_plt", + {ok, Beam} = compile(Config, Code, typer_test_module, []), + Plt = PrivDir ++ "dialyzer_iplt", + _ = dialyzer:run([{analysis_type, incremental}, + {files, [Beam]}, + {apps, [stdlib, kernel, erts]}, + {from, byte_code}, + {init_plt, Plt}, + {output_plt, Plt}]), + Args = "--plt " ++ Plt, Res = ["^$", "^%% File:", "^%% ----", @@ -66,6 +115,38 @@ smoke(Config) -> run(Config, Args, Src, Res), ok. +contract_violation(Config) -> + OutDir = proplists:get_value(priv_dir, Config), + case dialyzer_common:check_plt(OutDir) of + fail -> + {skip, "Plt creation/check failed."}; + ok -> + Code = <<"-module(typer_test_module). + -export([foo/1]). + -spec foo(boolean()) -> string(). + foo(N) -> + integer_to_list(N).">>, + PrivDir = proplists:get_value(priv_dir, Config), + Src = filename:join(PrivDir, "typer_test_module.erl"), + ok = file:write_file(Src, Code), + Args = "--plt " ++ PrivDir ++ "dialyzer_plt", + Res = ["^typer: Error in contract of function typer_test_module:foo/1", + "^\t The contract is: \\(boolean\\(\\)\\) -> string\\(\\)", + "^\t but the inferred signature is: \\(integer\\(\\)\\) -> string\\(\\)", + "_ERROR_"], + run(Config, Args, Src, Res), + ok + end. + +compile(Config, Prog, Module, CompileOpts) -> + Source = lists:concat([Module, ".erl"]), + PrivDir = proplists:get_value(priv_dir,Config), + Filename = filename:join([PrivDir, Source]), + ok = file:write_file(Filename, Prog), + Opts = [{outdir, PrivDir}, debug_info | CompileOpts], + {ok, Module} = compile:file(Filename, Opts), + {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + typer() -> case os:find_executable("typer") of false -> diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 4280b8f62e..7c6c7ed732 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -2241,21 +2241,9 @@ static DistFlags preferred_flags(void) { DistFlags flags = DFLAG_MANDATORY_25_DIGEST - | DFLAG_EXTENDED_REFERENCES + | DFLAG_DIST_MANDATORY | DFLAG_DIST_MONITOR - | DFLAG_EXTENDED_PIDS_PORTS - | DFLAG_FUN_TAGS - | DFLAG_NEW_FUN_TAGS - | DFLAG_NEW_FLOATS - | DFLAG_SMALL_ATOM_TAGS - | DFLAG_UTF8_ATOMS - | DFLAG_MAP_TAG - | DFLAG_BIG_CREATION - | DFLAG_EXPORT_PTR_TAG - | DFLAG_BIT_BINARIES - | DFLAG_HANDSHAKE_23 - | DFLAG_V4_NC - | DFLAG_UNLINK_ID; + | DFLAG_SMALL_ATOM_TAGS; return flags; } diff --git a/lib/erl_interface/src/connect/ei_connect_int.h b/lib/erl_interface/src/connect/ei_connect_int.h index d9be32d42c..ba71728d42 100644 --- a/lib/erl_interface/src/connect/ei_connect_int.h +++ b/lib/erl_interface/src/connect/ei_connect_int.h @@ -103,7 +103,12 @@ typedef EI_ULONGLONG DistFlags; | DFLAG_NEW_FLOATS \ | DFLAG_MAP_TAG \ | DFLAG_EXPORT_PTR_TAG \ - | DFLAG_BIT_BINARIES) + | DFLAG_BIT_BINARIES \ + | DFLAG_HANDSHAKE_23) + +/* New mandatory flags for distribution in OTP 26. */ +#define DFLAG_DIST_MANDATORY_26 (DFLAG_V4_NC \ + | DFLAG_UNLINK_ID) /* Mandatory flags for distribution. */ @@ -111,8 +116,8 @@ typedef EI_ULONGLONG DistFlags; * Mandatory flags for distribution. Keep them in sync with * erts/emulator/beam/dist.h. */ -#define DFLAG_DIST_MANDATORY DFLAG_DIST_MANDATORY_25 - +#define DFLAG_DIST_MANDATORY (DFLAG_DIST_MANDATORY_25 \ + | DFLAG_DIST_MANDATORY_26) ei_cnode *ei_fd_to_cnode(int fd); int ei_distversion(int fd); diff --git a/lib/erl_interface/test/ei_tmo_SUITE.erl b/lib/erl_interface/test/ei_tmo_SUITE.erl index d0e5f4514a..8d84714303 100644 --- a/lib/erl_interface/test/ei_tmo_SUITE.erl +++ b/lib/erl_interface/test/ei_tmo_SUITE.erl @@ -77,7 +77,9 @@ end_per_testcase(_Case, _Config) -> -define(DFLAG_MAP_TAG, 16#20000). -define(DFLAG_BIG_CREATION, 16#40000). -define(DFLAG_HANDSHAKE_23, 16#1000000). +-define(DFLAG_UNLINK_ID, 16#2000000). -define(DFLAG_MANDATORY_25_DIGEST, 16#4000000). +-define(DFLAG_V4_NC, 16#400000000). %% From OTP R9 extended references are compulsory. %% From OTP R10 extended pids and ports are compulsory. @@ -85,7 +87,8 @@ end_per_testcase(_Case, _Config) -> %% From OTP 21 NEW_FUN_TAGS is compulsory (no more tuple fallback {fun, ...}). %% From OTP 23 BIG_CREATION is compulsory. %% From OTP 25 NEW_FLOATS, MAP_TAG, EXPORT_PTR_TAG, and BIT_BINARIES are compulsory. --define(COMPULSORY_DFLAGS, + +-define(DFLAGS_MANDATORY_25, (?DFLAG_EXTENDED_REFERENCES bor ?DFLAG_FUN_TAGS bor ?DFLAG_EXTENDED_PIDS_PORTS bor @@ -98,6 +101,16 @@ end_per_testcase(_Case, _Config) -> ?DFLAG_BIT_BINARIES bor ?DFLAG_HANDSHAKE_23)). +%% From OTP 26 V4_NC, and UNLINK_ID are compulsory. + +-define(DFLAGS_MANDATORY_26, + (?DFLAG_V4_NC bor + ?DFLAG_UNLINK_ID)). + +-define(COMPULSORY_DFLAGS, + (?DFLAGS_MANDATORY_25 bor + ?DFLAGS_MANDATORY_26)). + %% Check the framework. framework_check(Config) when is_list(Config) -> %%dbg:tracer(), @@ -406,8 +419,8 @@ ei_dflags(Config) -> normal_accept(Config, ?COMPULSORY_DFLAGS), %% Test compatibility with future versions. - normal_connect(Config, ?DFLAG_MANDATORY_25_DIGEST), - normal_accept(Config, ?DFLAG_MANDATORY_25_DIGEST), + normal_connect(Config, ?DFLAG_MANDATORY_25_DIGEST bor ?DFLAGS_MANDATORY_26), + normal_accept(Config, ?DFLAG_MANDATORY_25_DIGEST bor ?DFLAGS_MANDATORY_26), ok. diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 2d02ddd2be..a74a6fbbba 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -351,7 +351,11 @@ <tag><c><![CDATA[ssl]]></c></tag> <item> <p>This is the <c>SSL/TLS</c> connecting configuration option.</p> - <p>Defaults to <c>[]</c>. See <seeerl marker="ssl:ssl">ssl:connect/[2,3,4]</seeerl> for available options.</p> + <p>Default value is obtained by calling + <seemfa marker="#ssl_verify_host_options/1"><c>httpc:ssl_verify_host_options(true)</c>. + </seemfa>. + See <seeerl marker="ssl:ssl">ssl:connect/[2,3,4]</seeerl> for available options. + </p> </item> <tag><c><![CDATA[autoredirect]]></c></tag> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 97d0d2b47f..3674bf3bb7 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -116,11 +116,11 @@ request(Url, Profile) -> %% {ssl, ssl_options()} | %% {essl, ssl_options()} %% ssl_options() = [ssl_option()] -%% ssl_option() = {verify, code()} | -%% {depth, depth()} | +%% ssl_option() = {verify, code()} | +%% {depth, depth()} | %% {certfile, path()} | -%% {keyfile, path()} | {password, string()} | {cacertfile, path()} | -%% {ciphers, string()} +%% {keyfile, path()} | {password, string()} | +%% {cacertfile, path()} | {ciphers, string()} %% Options - [Option] %% Option - {sync, Boolean} | %% {body_format, BodyFormat} | @@ -765,12 +765,14 @@ http_options_default() -> error end, + Ssl = ssl_verify_host_options(true), + UrlDecodePost = boolfun(), [ {version, {value, "HTTP/1.1"}, #http_options.version, VersionPost}, {timeout, {value, ?HTTP_REQUEST_TIMEOUT}, #http_options.timeout, TimeoutPost}, {autoredirect, {value, true}, #http_options.autoredirect, AutoRedirectPost}, - {ssl, {value, {?HTTP_DEFAULT_SSL_KIND, []}}, #http_options.ssl, SslPost}, + {ssl, {value, {?HTTP_DEFAULT_SSL_KIND, Ssl}}, #http_options.ssl, SslPost}, {proxy_auth, {value, undefined}, #http_options.proxy_auth, ProxyAuthPost}, {relaxed, {value, false}, #http_options.relaxed, RelaxedPost}, {url_encode, {value, false}, #http_options.url_encode, UrlDecodePost}, diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 32dbf6b97b..68f1c71850 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -32,12 +32,14 @@ -include("http_internal.hrl"). -include("httpc_internal.hrl"). %% Note: This directive should only be used in test suites. --compile(export_all). +-compile([export_all, nowarn_export_all]). -define(URL_START, "http://"). -define(TLS_URL_START, "https://"). -define(NOT_IN_USE_PORT, 8997). +-define(SSL_NO_VERIFY, {ssl, [{verify, verify_none}]}). + %% Using hardcoded file path to keep it below 107 characters %% (maximum length supported by erlang) -define(UNIX_SOCKET, "/tmp/inets_httpc_SUITE.sock"). @@ -394,17 +396,17 @@ head() -> head(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, - {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, Request, [], []). + {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, Request, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- get() -> [{doc, "Test http get request against local server"}]. get(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, - {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), + {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [?SSL_NO_VERIFY], []), inets_test_lib:check_body(Body), - {ok, {{_,200,_}, [_ | _], BinBody}} = httpc:request(get, Request, [], [{body_format, binary}]), + {ok, {{_,200,_}, [_ | _], BinBody}} = httpc:request(get, Request, [?SSL_NO_VERIFY], [{body_format, binary}]), true = is_binary(BinBody). @@ -412,7 +414,7 @@ get_query_string() -> [{doc, "Test http get request with query string against local server"}]. get_query_string(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html?foo=bar", Config), []}, - {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), + {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [?SSL_NO_VERIFY], []), inets_test_lib:check_body(Body). @@ -421,7 +423,7 @@ get_space() -> [{"Test http get request with '%20' in the path of the URL."}]. get_space(Config) when is_list(Config) -> Request = {url(group_name(Config), "/space%20.html", Config), []}, - {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), + {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [?SSL_NO_VERIFY], []), inets_test_lib:check_body(Body). @@ -445,11 +447,11 @@ post(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], [_ | _]}} = httpc:request(post, {URL, [{"expect","100-continue"}], - "text/plain", Body}, [], []), + "text/plain", Body}, [?SSL_NO_VERIFY], []), {ok, {{_,504,_}, [_ | _], []}} = httpc:request(post, {URL, [{"expect","100-continue"}], - "text/plain", "foobar"}, [], []). + "text/plain", "foobar"}, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- delete() -> [{"Test http delete request against local server. We do in this case " @@ -468,11 +470,11 @@ delete(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], [_ | _]}} = httpc:request(delete, {URL, [{"expect","100-continue"}], - "text/plain", Body}, [], []), + "text/plain", Body}, [?SSL_NO_VERIFY], []), {ok, {{_,504,_}, [_ | _], []}} = httpc:request(delete, {URL, [{"expect","100-continue"}], - "text/plain", "foobar"}, [], []). + "text/plain", "foobar"}, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- patch() -> @@ -494,7 +496,7 @@ patch(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], [_ | _]}} = httpc:request(patch, {URL, [{"expect","100-continue"}], - "text/plain", Body}, [], []). + "text/plain", Body}, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- post_stream() -> @@ -522,20 +524,20 @@ post_stream(Config) when is_list(Config) -> httpc:request(post, {URL, [{"expect", "100-continue"}, {"content-length", "100"}], - "text/plain", {BodyFun, 100}}, [], []), + "text/plain", {BodyFun, 100}}, [?SSL_NO_VERIFY], []), {ok, {{_,504,_}, [_ | _], []}} = httpc:request(post, {URL, [{"expect", "100-continue"}, {"content-length", "10"}], - "text/plain", {BodyFun, 10}}, [], []). + "text/plain", {BodyFun, 10}}, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- trace() -> [{doc, "Perform a TRACE request."}]. trace(Config) when is_list(Config) -> Request = {url(group_name(Config), "/trace.html", Config), []}, - case httpc:request(trace, Request, [], []) of + case httpc:request(trace, Request, [?SSL_NO_VERIFY], []) of {ok, {{_,200,_}, [_ | _], "TRACE /trace.html" ++ _}} -> ok; Other -> @@ -546,7 +548,7 @@ trace(Config) when is_list(Config) -> pipeline(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, - {ok, _} = httpc:request(get, Request, [], [], pipeline), + {ok, _} = httpc:request(get, Request, [?SSL_NO_VERIFY], [], pipeline), %% Make sure pipeline session is registered ct:sleep(4000), @@ -556,7 +558,7 @@ pipeline(Config) when is_list(Config) -> persistent_connection(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, - {ok, _} = httpc:request(get, Request, [], [], persistent), + {ok, _} = httpc:request(get, Request, [?SSL_NO_VERIFY], [], persistent), %% Make sure pipeline session is registered ct:sleep(4000), @@ -569,7 +571,7 @@ async(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, RequestId} = - httpc:request(get, Request, [], [{sync, false}]), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}]), Body = receive {http, {RequestId, {{_, 200, _}, _, BinBody}}} -> @@ -580,7 +582,7 @@ async(Config) when is_list(Config) -> inets_test_lib:check_body(binary_to_list(Body)), {ok, NewRequestId} = - httpc:request(get, Request, [], [{sync, false}]), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}]), ok = httpc:cancel_request(NewRequestId). %%------------------------------------------------------------------------- @@ -592,7 +594,7 @@ save_to_file(Config) when is_list(Config) -> URL = url(group_name(Config), "/dummy.html", Config), Request = {URL, []}, {ok, saved_to_file} - = httpc:request(get, Request, [], [{stream, FilePath}]), + = httpc:request(get, Request, [?SSL_NO_VERIFY], [{stream, FilePath}]), {ok, Bin} = file:read_file(FilePath), {ok, {{_,200,_}, [_ | _], Body}} = httpc:request(URL), Bin == Body. @@ -605,7 +607,7 @@ save_to_file_async(Config) when is_list(Config) -> FilePath = filename:join(PrivDir, "dummy.html"), URL = url(group_name(Config), "/dummy.html", Config), Request = {URL, []}, - {ok, RequestId} = httpc:request(get, Request, [], + {ok, RequestId} = httpc:request(get, Request, [?SSL_NO_VERIFY], [{stream, FilePath}, {sync, false}]), receive @@ -676,10 +678,10 @@ redirect_multiple_choises(Config) when is_list(Config) -> URL300 = url(group_name(Config), "/300.html", Config), catch {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL300, []}, [], []), + = httpc:request(get, {URL300, []}, [?SSL_NO_VERIFY], []), {ok, {{_,300,_}, [_ | _], _}} = - httpc:request(get, {URL300, []}, [{autoredirect, false}], []). + httpc:request(get, {URL300, []}, [{autoredirect, false},?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- redirect_moved_permanently() -> [{doc, "The server SHOULD generate a Location header field in the response " @@ -692,14 +694,14 @@ redirect_moved_permanently(Config) when is_list(Config) -> URL301 = url(group_name(Config), "/301.html", Config), {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL301, []}, [], []), + = httpc:request(get, {URL301, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL301, []}, [], []), + = httpc:request(head, {URL301, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL301, [],"text/plain", "foobar"}, - [], []). + [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- redirect_found() -> [{doc, "The server SHOULD generate a Location header field in the response " @@ -712,14 +714,14 @@ redirect_found(Config) when is_list(Config) -> URL302 = url(group_name(Config), "/302.html", Config), {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL302, []}, [], []), + = httpc:request(get, {URL302, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL302, []}, [], []), + = httpc:request(head, {URL302, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL302, [],"text/plain", "foobar"}, - [], []). + [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- redirect_see_other() -> [{doc, "The different URI SHOULD be given by the Location field in the response. " @@ -730,14 +732,14 @@ redirect_see_other(Config) when is_list(Config) -> URL303 = url(group_name(Config), "/303.html", Config), {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL303, []}, [], []), + = httpc:request(get, {URL303, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL303, []}, [], []), + = httpc:request(head, {URL303, []}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL303, [],"text/plain", "foobar"}, - [], []). + [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- redirect_temporary_redirect() -> [{doc, "The server SHOULD generate a Location header field in the response " @@ -806,7 +808,7 @@ redirect_loop(Config) when is_list(Config) -> URL = url(group_name(Config), "/redirectloop.html", Config), {ok, {{_,300,_}, [_ | _], _}} - = httpc:request(get, {URL, []}, [], []). + = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- redirect_http_to_https() -> @@ -818,14 +820,15 @@ redirect_http_to_https(Config) when is_list(Config) -> Headers = [{"x-test-301-url", TargetUrl}], {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL301, Headers}, [], []), + = httpc:request(get, {URL301, Headers}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL301, Headers}, [], []), + = httpc:request(head, {URL301, Headers}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL301, Headers, "text/plain", "foobar"}, - [], []). + [?SSL_NO_VERIFY], []). + %%------------------------------------------------------------------------- redirect_relative_different_port() -> [{doc, "Test that a 30X redirect with a relative target, but different " @@ -859,7 +862,7 @@ cookie(Config) when is_list(Config) -> Request0 = {url(group_name(Config), "/cookie.html", Config), []}, {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, Request0, [], []), + = httpc:request(get, Request0, [?SSL_NO_VERIFY], []), %% Populate table to be used by the "dummy" server ets:new(cookie, [named_table, public, set]), @@ -868,7 +871,7 @@ cookie(Config) when is_list(Config) -> Request1 = {url(group_name(Config), "/", Config), []}, {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, Request1, [], []), + = httpc:request(get, Request1, [?SSL_NO_VERIFY], []), [{session_cookies, [_|_]}] = httpc:which_cookies(httpc:default_profile()), @@ -887,7 +890,7 @@ cookie_profile(Config) when is_list(Config) -> Request0 = {url(group_name(Config), "/cookie.html", Config), []}, {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, Request0, [], [], cookie_test), + = httpc:request(get, Request0, [?SSL_NO_VERIFY], [], cookie_test), %% Populate table to be used by the "dummy" server ets:new(cookie, [named_table, public, set]), @@ -896,7 +899,7 @@ cookie_profile(Config) when is_list(Config) -> Request1 = {url(group_name(Config), "/", Config), []}, {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, Request1, [], [], cookie_test), + = httpc:request(get, Request1, [?SSL_NO_VERIFY], [], cookie_test), ets:delete(cookie), inets:stop(httpc, cookie_test). @@ -910,7 +913,7 @@ empty_set_cookie(Config) when is_list(Config) -> Request0 = {url(group_name(Config), "/empty_set_cookie.html", Config), []}, {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, Request0, [], []), + = httpc:request(get, Request0, [?SSL_NO_VERIFY], []), ok = httpc:set_options([{cookies, disabled}]). @@ -922,7 +925,7 @@ invalid_set_cookie(Config) when is_list(Config) -> URL = url(group_name(Config), "/invalid_set_cookie.html", Config), {ok, {{_,200,_}, [_|_], [_|_]}} = - httpc:request(get, {URL, []}, [], []), + httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []), ok = httpc:set_options([{cookies, disabled}]). @@ -933,10 +936,10 @@ headers_as_is(Config) when is_list(Config) -> URL = url(group_name(Config), "/dummy.html", Config), {ok, {{_,200,_}, [_|_], [_|_]}} = httpc:request(get, {URL, [{"Host", "localhost"},{"Te", ""}]}, - [], [{headers_as_is, true}]), + [?SSL_NO_VERIFY], [{headers_as_is, true}]), {ok, {{_,400,_}, [_|_], [_|_]}} = - httpc:request(get, {URL, [{"Te", ""}]},[], [{headers_as_is, true}]). + httpc:request(get, {URL, [{"Te", ""}]}, [?SSL_NO_VERIFY], [{headers_as_is, true}]). %%------------------------------------------------------------------------- userinfo(doc) -> @@ -948,12 +951,12 @@ userinfo(Config) when is_list(Config) -> URLAuth = url(group_name(Config), "alladin:sesame@" ++ Host ++ ":","/userinfo.html", Config), {ok, {{_,200,_}, [_ | _], _}} - = httpc:request(get, {URLAuth, []}, [], []), + = httpc:request(get, {URLAuth, []}, [?SSL_NO_VERIFY], []), URLUnAuth = url(group_name(Config), "alladin:foobar@" ++ Host ++ ":","/userinfo.html", Config), {ok, {{_,401, _}, [_ | _], _}} = - httpc:request(get, {URLUnAuth, []}, [], []). + httpc:request(get, {URLUnAuth, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -962,7 +965,7 @@ page_does_not_exist(doc) -> page_does_not_exist(Config) when is_list(Config) -> URL = url(group_name(Config), "/doesnotexist.html", Config), {ok, {{_,404,_}, [_ | _], [_ | _]}} - = httpc:request(get, {URL, []}, [], []). + = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- streaming_error(doc) -> @@ -972,9 +975,9 @@ streaming_error(Config) when is_list(Config) -> Method = get, Request = {url(group_name(Config), "/dummy.html", Config), []}, {error, streaming_error} = httpc:request(Method, Request, - [], [{sync, true}, {stream, {self, once}}]), + [?SSL_NO_VERIFY], [{sync, true}, {stream, {self, once}}]), {error, streaming_error} = httpc:request(Method, Request, - [], [{sync, true}, {stream, self}]). + [?SSL_NO_VERIFY], [{sync, true}, {stream, self}]). %%------------------------------------------------------------------------- server_does_not_exist(doc) -> @@ -984,14 +987,14 @@ server_does_not_exist(Config) when is_list(Config) -> {error, _} = httpc:request(get, {"http://localhost:" ++ integer_to_list(?NOT_IN_USE_PORT) - ++ "/", []},[], []). + ++ "/", []},[?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- no_content_204(doc) -> ["Test the case that the HTTP 204 no content header - Solves OTP 6982"]; no_content_204(Config) when is_list(Config) -> URL = url(group_name(Config), "/no_content.html", Config), - {ok, {{_,204,_}, [], []}} = httpc:request(URL). + {ok, {{_,204,_}, [], []}} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1000,7 +1003,7 @@ tolerate_missing_CR() -> "as delimiter. Solves OTP-7304"}]. tolerate_missing_CR(Config) when is_list(Config) -> URL = url(group_name(Config), "/missing_CR.html", Config), - {ok, {{_,200,_}, _, [_ | _]}} = httpc:request(URL). + {ok, {{_,200,_}, _, [_ | _]}} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- empty_body() -> @@ -1009,7 +1012,7 @@ empty_body() -> empty_body(Config) when is_list(Config) -> URL = url(group_name(Config), "/empty.html", Config), {ok, {{_,200,_}, [_ | _], []}} = - httpc:request(get, {URL, []}, [], []). + httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1017,13 +1020,13 @@ transfer_encoding() -> [{doc, "Transfer encoding is case insensitive. Solves OTP-6807"}]. transfer_encoding(Config) when is_list(Config) -> URL = url(group_name(Config), "/capital_transfer_encoding.html", Config), - {ok, {{_,200,_}, [_|_], [_ | _]}} = httpc:request(URL). + {ok, {{_,200,_}, [_|_], [_ | _]}} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- transfer_encoding_identity(Config) when is_list(Config) -> URL = url(group_name(Config), "/identity_transfer_encoding.html", Config), - {ok, {{_,200,_}, [_|_], "IDENTITY"}} = httpc:request(URL). + {ok, {{_,200,_}, [_|_], "IDENTITY"}} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1031,7 +1034,7 @@ empty_response_header() -> [{doc, "Test the case that the HTTP server does not send any headers. Solves OTP-6830"}]. empty_response_header(Config) when is_list(Config) -> URL = url(group_name(Config), "/no_headers.html", Config), - {ok, {{_,200,_}, [], [_ | _]}} = httpc:request(URL). + {ok, {{_,200,_}, [], [_ | _]}} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1043,8 +1046,8 @@ bad_response(Config) when is_list(Config) -> URL0 = url(group_name(Config), "/missing_crlf.html", Config), URL1 = url(group_name(Config), "/wrong_statusline.html", Config), - {error, timeout} = httpc:request(get, {URL0, []}, [{timeout, 400}], []), - {error, Reason} = httpc:request(URL1), + {error, timeout} = httpc:request(get, {URL0, []}, [{timeout, 400},?SSL_NO_VERIFY], []), + {error, Reason} = httpc:request(get, {URL1, []}, [?SSL_NO_VERIFY], []), ct:print("Wrong Statusline: ~p~n", [Reason]). %%------------------------------------------------------------------------- @@ -1053,7 +1056,7 @@ timeout_redirect() -> [{doc, "Test that timeout works for redirects, check ERL-420."}]. timeout_redirect(Config) when is_list(Config) -> URL = url(group_name(Config), "/redirect_to_missing_crlf.html", Config), - {error, timeout} = httpc:request(get, {URL, []}, [{timeout, 400}], []). + {error, timeout} = httpc:request(get, {URL, []}, [{timeout, 400},?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1064,7 +1067,7 @@ internal_server_error(Config) when is_list(Config) -> URL500 = url(group_name(Config), "/500.html", Config), {ok, {{_,500,_}, [_ | _], _}} - = httpc:request(get, {URL500, []}, [], []), + = httpc:request(get, {URL500, []}, [?SSL_NO_VERIFY], []), URL503 = url(group_name(Config), "/503.html", Config), @@ -1073,12 +1076,12 @@ internal_server_error(Config) when is_list(Config) -> ets:insert(unavailable, {503, unavailable}), {ok, {{_,200, _}, [_ | _], [_|_]}} = - httpc:request(get, {URL503, []}, [], []), + httpc:request(get, {URL503, []}, [?SSL_NO_VERIFY], []), ets:insert(unavailable, {503, long_unavailable}), {ok, {{_,503, _}, [_ | _], [_|_]}} = - httpc:request(get, {URL503, []}, [], []), + httpc:request(get, {URL503, []}, [?SSL_NO_VERIFY], []), ets:delete(unavailable). @@ -1093,7 +1096,7 @@ invalid_http(Config) when is_list(Config) -> URL = url(group_name(Config), "/invalid_http.html", Config), {error, {could_not_parse_as_http, _} = Reason} = - httpc:request(get, {URL, []}, [], []), + httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []), ct:print("Parse error: ~p ~n", [Reason]). @@ -1108,7 +1111,7 @@ invalid_chunk_size(Config) when is_list(Config) -> URL = url(group_name(Config), "/invalid_chunk_size.html", Config), {error, {chunk_size, _} = Reason} = - httpc:request(get, {URL, []}, [], []), + httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []), ct:print("Parse error: ~p ~n", [Reason]). @@ -1121,10 +1124,10 @@ emulate_lower_versions(Config) when is_list(Config) -> URL = url(group_name(Config), "/dummy.html", Config), {ok, {{"HTTP/1.0", 200, _}, [_ | _], Body1 = [_ | _]}} = - httpc:request(get, {URL, []}, [{version, "HTTP/1.0"}], []), + httpc:request(get, {URL, []}, [{version, "HTTP/1.0"}, ?SSL_NO_VERIFY], []), inets_test_lib:check_body(Body1), {ok, {{"HTTP/1.1", 200, _}, [_ | _], Body2 = [_ | _]}} = - httpc:request(get, {URL, []}, [{version, "HTTP/1.1"}], []), + httpc:request(get, {URL, []}, [{version, "HTTP/1.1"}, ?SSL_NO_VERIFY], []), inets_test_lib:check_body(Body2). %%------------------------------------------------------------------------- @@ -1136,12 +1139,12 @@ relaxed(Config) when is_list(Config) -> URL = url(group_name(Config), "/missing_reason_phrase.html", Config), {error, Reason} = - httpc:request(get, {URL, []}, [{relaxed, false}], []), + httpc:request(get, {URL, []}, [{relaxed, false}, ?SSL_NO_VERIFY], []), ct:print("Not relaxed: ~p~n", [Reason]), {ok, {{_, 200, _}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, []}, [{relaxed, true}], []). + httpc:request(get, {URL, []}, [{relaxed, true}, ?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1169,7 +1172,7 @@ headers(Config) when is_list(Config) -> Mod}, {"From","webmaster@erlang.se"}, {"Date", Date} - ]}, [], []), + ]}, [?SSL_NO_VERIFY], []), Mod1 = httpd_util:rfc1123_date( calendar:gregorian_seconds_to_datetime( @@ -1178,7 +1181,7 @@ headers(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], [_ | _]}} = httpc:request(get, {URL, [{"If-UnModified-Since", Mod1} - ]}, [], []), + ]}, [?SSL_NO_VERIFY], []), Tag = httpd_util:create_etag(FileInfo), @@ -1186,13 +1189,13 @@ headers(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], [_ | _]}} = httpc:request(get, {URL, [{"If-Match", Tag} - ]}, [], []), + ]}, [?SSL_NO_VERIFY], []), {ok, {{_,200,_}, [_ | _], _}} = httpc:request(get, {URL, [{"If-None-Match", "NotEtag,NeihterEtag"}, {"Connection", "Close"} - ]}, [], []). + ]}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- headers_dummy() -> ["Test the code for handling headers we do not want/can send " @@ -1251,14 +1254,14 @@ headers_dummy(Config) when is_list(Config) -> {"Last-Modified", "Sat, 29 Oct 1994 19:43:31 GMT"}, {"Trailer","1#User-Agent"} ], "text/plain", FooBar}, - [], []). + [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- headers_with_obs_fold(Config) when is_list(Config) -> Request = {url(group_name(Config), "/obs_folded_headers.html", Config), []}, - {ok, {{_,200,_}, Headers, [_|_]}} = httpc:request(get, Request, [], []), + {ok, {{_,200,_}, Headers, [_|_]}} = httpc:request(get, Request, [?SSL_NO_VERIFY], []), "a b" = proplists:get_value("folded", Headers). %%------------------------------------------------------------------------- @@ -1269,8 +1272,8 @@ headers_conflict_chunked_with_length(doc) -> "and must receive successful response in relaxed mode"]; headers_conflict_chunked_with_length(Config) when is_list(Config) -> Request = {url(group_name(Config), "/headers_conflict_chunked_with_length.html", Config), []}, - {error, {could_not_parse_as_http, _}} = httpc:request(get, Request, [{relaxed, false}], []), - {ok,{{_,200,_},_,_}} = httpc:request(get, Request, [{relaxed, true}], []), + {error, {could_not_parse_as_http, _}} = httpc:request(get, Request, [{relaxed, false}, ?SSL_NO_VERIFY], []), + {ok,{{_,200,_},_,_}} = httpc:request(get, Request, [{relaxed, true}, ?SSL_NO_VERIFY], []), ok. %%------------------------------------------------------------------------- @@ -1280,13 +1283,13 @@ invalid_headers_key(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), [{cookie, "valid cookie"}]}, {error, {headers_error, invalid_field}} = - httpc:request(get, Request, [], []). + httpc:request(get, Request, [?SSL_NO_VERIFY], []). invalid_headers_value(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), [{"cookie", atom_value}]}, {error, {headers_error, invalid_value}} = - httpc:request(get, Request, [], []). + httpc:request(get, Request, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1326,7 +1329,7 @@ test_header_type(Config, Method, Value) -> {Method, Value, httpc:request(Method, make_request(Config, Method, Value), - [], + [?SSL_NO_VERIFY], [])}. make_request(Config, Method, Value) -> @@ -1379,7 +1382,10 @@ invalid_method(Config) -> binary_url(Config) -> URL = uri_string:normalize(url(group_name(Config), "/dummy.html", Config)), - {ok, _Response} = httpc:request(unicode:characters_to_binary(URL)). + case group_name(Config) of + https -> ok; + _ -> {ok, _Response} = httpc:request(unicode:characters_to_binary(URL)) + end. %%------------------------------------------------------------------------- @@ -1458,7 +1464,7 @@ invalid_uri(Config) -> %%------------------------------------------------------------------------- remote_socket_close(Config) when is_list(Config) -> URL = url(group_name(Config), "/just_close.html", Config), - {error, socket_closed_remotely} = httpc:request(URL). + {error, socket_closed_remotely} = httpc:request(get, {URL, []}, [?SSL_NO_VERIFY], []). %%------------------------------------------------------------------------- @@ -1468,7 +1474,7 @@ remote_socket_close_async(Config) when is_list(Config) -> Options = [{sync, false}], Profile = httpc:default_profile(), {ok, RequestId} = - httpc:request(get, Request, [], Options, Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], Options, Profile), receive {http, {RequestId, {error, socket_closed_remotely}}} -> ok @@ -1536,7 +1542,7 @@ inet_opts(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, Timeout = timer:seconds(1), ConnTimeout = Timeout + timer:seconds(1), - HttpOptions = [{timeout, Timeout}, {connect_timeout, ConnTimeout}], + HttpOptions = [{timeout, Timeout}, {connect_timeout, ConnTimeout}, ?SSL_NO_VERIFY], Options0 = [{socket_opts, [{tos, 87}, {recbuf, 16#FFFF}, {sndbuf, 16#FFFF}]}], @@ -1547,7 +1553,7 @@ inet_opts(Config) when is_list(Config) -> Options1 = [{socket_opts, [{tos, 84}, {recbuf, 32#1FFFF}, {sndbuf, 32#1FFFF}]}], - {ok, {{_,200,_}, [_ | _], ReplyBody1 = [_ | _]}} = httpc:request(get, Request, [], Options1), + {ok, {{_,200,_}, [_ | _], ReplyBody1 = [_ | _]}} = httpc:request(get, Request, [?SSL_NO_VERIFY], Options1), inets_test_lib:check_body(ReplyBody1). %%------------------------------------------------------------------------- @@ -1876,7 +1882,7 @@ post_with_content_type(Config) when is_list(Config) -> URL = url(group_name(Config), "/delete_no_body.html", Config), %% Simulated server replies 500 if 'Content-Type' header is present {ok, {{_,500,_}, _, _}} = - httpc:request(post, {URL, [], "application/x-www-form-urlencoded", ""}, [], []). + httpc:request(post, {URL, [], "application/x-www-form-urlencoded", ""}, [?SSL_NO_VERIFY], []). %%-------------------------------------------------------------------- request_options() -> @@ -1906,7 +1912,7 @@ stream(ReceiverPid, Receiver, Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, Options = [{sync, false}, {receiver, Receiver}], {ok, RequestId} = - httpc:request(get, Request, [], Options), + httpc:request(get, Request, [?SSL_NO_VERIFY], Options), Body = receive {reply, ReceiverPid, {RequestId, {{_, 200, _}, _, B}}} -> @@ -1953,9 +1959,9 @@ stream_deliver(ReplyInfo, Type, ReceiverPid) -> stream_test(Request, To) -> {ok, {{_,200,_}, [_ | _], Body}} = - httpc:request(get, Request, [], []), + httpc:request(get, Request, [?SSL_NO_VERIFY], []), {ok, RequestId} = - httpc:request(get, Request, [], [{sync, false}, To]), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}, To]), StreamedBody = receive @@ -1971,9 +1977,9 @@ stream_test(Request, To) -> not_streamed_test(Request, To) -> {ok, {{_,Code,_}, [_ | _], Body}} = - httpc:request(get, Request, [], [{body_format, binary}]), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{body_format, binary}]), {ok, RequestId} = - httpc:request(get, Request, [], [{body_format, binary}, {sync, false}, To]), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{body_format, binary}, {sync, false}, To]), receive {http, {RequestId, {{_, Code, _}, _Headers, Body}}} -> @@ -2156,20 +2162,20 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> keep_alive_requests(Request, Profile) -> {ok, RequestIdA0} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), {ok, RequestIdA1} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), {ok, RequestIdA2} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), receive_replys([RequestIdA0, RequestIdA1, RequestIdA2]), {ok, RequestIdB0} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), {ok, RequestIdB1} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), {ok, RequestIdB2} = - httpc:request(get, Request, [], [{sync, false}], Profile), + httpc:request(get, Request, [?SSL_NO_VERIFY], [{sync, false}], Profile), ok = httpc:cancel_request(RequestIdB1, Profile), ct:print("Cancel ~p~n", [RequestIdB1]), diff --git a/lib/inets/test/httpc_proxy_SUITE.erl b/lib/inets/test/httpc_proxy_SUITE.erl index 02751dfdf4..249b2369ed 100644 --- a/lib/inets/test/httpc_proxy_SUITE.erl +++ b/lib/inets/test/httpc_proxy_SUITE.erl @@ -42,6 +42,8 @@ [self(),?MODULE] ++ begin A end) end). +-define(SSL_NO_VERIFY, {ssl, [{verify, verify_none}]}). + %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- @@ -159,7 +161,7 @@ http_head(Config) when is_list(Config) -> Method = head, URL = url("/index.html", Config), Request = {URL,[]}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},[_|_],[]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -176,13 +178,13 @@ http_get(Config) when is_list(Config) -> Timeout = timer:seconds(1), ConnTimeout = Timeout + timer:seconds(1), - HttpOpts1 = [{timeout,Timeout},{connect_timeout,ConnTimeout}], + HttpOpts1 = [{timeout,Timeout},{connect_timeout,ConnTimeout},?SSL_NO_VERIFY], Opts1 = [], {ok,{{_,200,_},[_|_],[_|_]=B1}} = httpc:request(Method, Request, HttpOpts1, Opts1), inets_test_lib:check_body(B1), - HttpOpts2 = [], + HttpOpts2 = [?SSL_NO_VERIFY], Opts2 = [{body_format,binary}], {ok,{{_,200,_},[_|_],B2}} = httpc:request(Method, Request, HttpOpts2, Opts2), @@ -196,7 +198,7 @@ http_options(Config) when is_list(Config) -> Method = options, URL = url("/index.html", Config), Request = {URL,[]}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},Headers,_}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -211,7 +213,7 @@ http_trace(Config) when is_list(Config) -> Method = trace, URL = url("/index.html", Config), Request = {URL,[]}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},[_|_],"TRACE "++_}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -227,7 +229,7 @@ http_post(Config) when is_list(Config) -> Method = post, URL = url("/index.html", Config), Request = {URL,[],"text/plain","foobar"}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -244,7 +246,7 @@ http_put(Config) when is_list(Config) -> Content = "<html><body> <h1>foo</h1> <p>bar</p> </body></html>", Request = {URL,[],"html",Content}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,405,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -260,7 +262,7 @@ http_delete(Config) when is_list(Config) -> Method = delete, URL = url("/delete.html", Config), Request = {URL,[]}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,405,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -276,7 +278,7 @@ http_delete_body(Config) when is_list(Config) -> URL = url("/delete.html", Config), Content = "foo=bar", Request = {URL,[],"application/x-www-form-urlencoded",Content}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,405,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -302,7 +304,7 @@ http_headers(Config) when is_list(Config) -> {"Referer", "http://otp.ericsson.se:8000/product/internal"}], Request = {URL,Headers}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -319,7 +321,7 @@ http_proxy_auth(Config) when is_list(Config) -> Method = get, URL = url("/index.html", Config), Request = {URL,[]}, - HttpOpts = [{proxy_auth,{"foo","bar"}}], + HttpOpts = [{proxy_auth,{"foo","bar"}}, ?SSL_NO_VERIFY], Opts = [], {ok,{{_,200,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -333,7 +335,7 @@ http_doesnotexist(Config) when is_list(Config) -> Method = get, URL = url("/doesnotexist.html", Config), Request = {URL,[]}, - HttpOpts = [{proxy_auth,{"foo","bar"}}], + HttpOpts = [{proxy_auth,{"foo","bar"}}, ?SSL_NO_VERIFY], Opts = [], {ok,{{_,404,_},[_|_],[_|_]}} = httpc:request(Method, Request, HttpOpts, Opts), @@ -347,7 +349,7 @@ http_stream(Config) when is_list(Config) -> Method = get, URL = url("/index.html", Config), Request = {URL,[]}, - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Opts1 = [{body_format,binary}], {ok,{{_,200,_},[_|_],Body}} = @@ -385,12 +387,12 @@ http_emulate_lower_versions(Config) when is_list(Config) -> Request = {URL,[]}, Opts = [], - HttpOpts1 = [{version,"HTTP/1.0"}], + HttpOpts1 = [{version,"HTTP/1.0"},?SSL_NO_VERIFY], {ok,{{_,200,_},[_|_],[_|_]=B2}} = httpc:request(Method, Request, HttpOpts1, Opts), inets_test_lib:check_body(B2), - HttpOpts2 = [{version,"HTTP/1.1"}], + HttpOpts2 = [{version,"HTTP/1.1"},?SSL_NO_VERIFY], {ok,{{_,200,_},[_|_],[_|_]=B3}} = httpc:request(Method, Request, HttpOpts2, Opts), inets_test_lib:check_body(B3), @@ -406,7 +408,7 @@ http_not_modified_otp_6821(Config) when is_list(Config) -> Opts = [], Request1 = {URL,[]}, - HttpOpts1 = [], + HttpOpts1 = [?SSL_NO_VERIFY], {ok,{{_,200,_},ReplyHeaders,[_|_]}} = httpc:request(Method, Request1, HttpOpts1, Opts), ETag = header_value("etag", ReplyHeaders), @@ -416,7 +418,7 @@ http_not_modified_otp_6821(Config) when is_list(Config) -> {URL, [{"If-None-Match",ETag}, {"If-Modified-Since",LastModified}]}, - HttpOpts2 = [{timeout,15000}], % Limit wait for bug result + HttpOpts2 = [{timeout,15000}, ?SSL_NO_VERIFY], % Limit wait for bug result {ok,{{_,304,_},_,[]}} = % Page Unchanged httpc:request(Method, Request2, HttpOpts2, Opts), @@ -440,7 +442,7 @@ https_connect_error(Config) when is_list(Config) -> URL = "https://" ++ HttpServer ++ ":" ++ integer_to_list(HttpPort) ++ "/index.html", Opts = [], - HttpOpts = [], + HttpOpts = [?SSL_NO_VERIFY], Request = {URL,[]}, {error,{failed_connect,[_,{tls,_,_}]}} = httpc:request(Method, Request, HttpOpts, Opts). @@ -453,7 +455,7 @@ http_timeout(Config) when is_list(Config) -> URL = url("/index.html", Config), Request = {URL,[]}, Timeout = timer:seconds(1), - HttpOpts1 = [{timeout, Timeout}, {connect_timeout, 0}], + HttpOpts1 = [{timeout, Timeout}, {connect_timeout, 0}, ?SSL_NO_VERIFY], {error, {failed_connect, [{to_address,{"localhost",8000}}, diff --git a/lib/inets/test/inets_socketwrap_SUITE.erl b/lib/inets/test/inets_socketwrap_SUITE.erl index b88cff4e90..562f4ee7f1 100644 --- a/lib/inets/test/inets_socketwrap_SUITE.erl +++ b/lib/inets/test/inets_socketwrap_SUITE.erl @@ -73,12 +73,11 @@ start_httpd_fd(Config) when is_list(Config) -> InetPort = inets_test_lib:inet_port(node()), ct:pal("Node: ~p Port ~p~n", [Node, InetPort]), Wrapper = filename:join(DataDir, "setuid_socket_wrap"), - Cmd = Wrapper ++ - " -s -httpd_80,0:" ++ integer_to_list(InetPort) - ++ " -p " ++ os:find_executable("erl") ++ - " -- " ++ NodeArg, - ct:pal("cmd: ~p~n", [Cmd]), - case open_port({spawn, Cmd}, [stderr_to_stdout]) of + Args = ["-s","-httpd_80,0:" ++ integer_to_list(InetPort), + "-p",os:find_executable("erl"),"--" | NodeArg], + ct:pal("cmd: ~p ~p~n", [Wrapper, Args]), + case open_port({spawn_executable, Wrapper}, + [stderr_to_stdout,{args,Args}]) of Port when is_port(Port) -> wait_node_up(Node, 10), ct:pal("~p", [rpc:call(Node, init, get_argument, [httpd_80])]), @@ -97,19 +96,18 @@ start_httpd_fd(Config) when is_list(Config) -> setup_node_info(nonode@nohost) -> {skip, needs_distributed_node}; setup_node_info(Node) -> - Static = "-detached -noinput", Name = "inets_fd_test", NameSw = case net_kernel:longnames() of - false -> "-sname "; - _ -> "-name " - end, - StrNode = - Static ++ " " - ++ NameSw ++ " " ++ Name ++ " " - ++ "-setcookie " ++ atom_to_list(erlang:get_cookie()), - [_, Location] = string:tokens(atom_to_list(Node), "$@"), - TestNode = Name ++ "@" ++ Location, - {list_to_atom(TestNode), StrNode}. + false -> "-sname"; + _ -> "-name" + end, + NodeArgs = ["-detached","-noinput", + NameSw, Name, "-setcookie", atom_to_list(erlang:get_cookie())], + + [_, Location] = string:tokens(atom_to_list(Node), "$@"), + TestNode = Name ++ "@" ++ Location, + + {list_to_atom(TestNode), NodeArgs}. wait_node_up(Node, 0) -> ct:fail({failed_to_start_node, Node}); diff --git a/lib/inets/test/rules.mk b/lib/inets/test/rules.mk index c4a62a87ed..cc3150934d 100644 --- a/lib/inets/test/rules.mk +++ b/lib/inets/test/rules.mk @@ -18,11 +18,7 @@ DEFAULT_TARGETS = opt debug instr release release_docs clean docs # ---------------------------------------------------- EMULATOR = beam -ifdef BOOTSTRAP -ERL_COMPILE_FLAGS += +slim -else ERL_COMPILE_FLAGS += +debug_info -endif ERLC_WFLAGS = -W ERLC = erlc $(ERLC_WFLAGS) $(ERLC_FLAGS) ERL.beam = erl.beam -boot start_clean @@ -47,8 +43,3 @@ $(EBIN)/%.beam: $(ESRC)/%.erl .erl.beam: $(ERLC) -bbeam $(ERL_COMPILE_FLAGS) -o$(dir $@) $< - - - - - diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractConnection.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractConnection.java index 5949bce166..6f4f9993f0 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractConnection.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractConnection.java @@ -425,29 +425,12 @@ public abstract class AbstractConnection extends Thread { header.write1(passThrough); header.write1(version); - if ((peer.flags & AbstractNode.dFlagUnlinkId) != 0) { - // header - header.write_tuple_head(4); - header.write_long(unlinkIdTag); - header.write_long(unlink_id); - header.write_any(from); - header.write_any(dest); - } - else { - /* - * A node that isn't capable of talking the new link protocol. - * - * Send an old unlink op, and send ourselves an unlink-ack. We may - * end up in an inconsistent state as we could before the new link - * protocol was introduced... - */ - // header - header.write_tuple_head(3); - header.write_long(unlinkTag); - header.write_any(from); - header.write_any(dest); - deliver(new OtpMsg(unlinkIdAckTag, dest, from, unlink_id)); - } + // header + header.write_tuple_head(4); + header.write_long(unlinkIdTag); + header.write_long(unlink_id); + header.write_any(from); + header.write_any(dest); // fix up length in preamble header.poke4BE(0, header.size() - 4); @@ -471,26 +454,25 @@ public abstract class AbstractConnection extends Thread { if (!connected) { throw new IOException("Not connected"); } - if ((peer.flags & AbstractNode.dFlagUnlinkId) != 0) { - @SuppressWarnings("resource") + + @SuppressWarnings("resource") final OtpOutputStream header = new OtpOutputStream(headerLen); - // preamble: 4 byte length + "passthrough" tag - header.write4BE(0); // reserve space for length - header.write1(passThrough); - header.write1(version); + // preamble: 4 byte length + "passthrough" tag + header.write4BE(0); // reserve space for length + header.write1(passThrough); + header.write1(version); - // header - header.write_tuple_head(4); - header.write_long(unlinkIdAckTag); - header.write_long(unlink_id); - header.write_any(from); - header.write_any(dest); - // fix up length in preamble - header.poke4BE(0, header.size() - 4); + // header + header.write_tuple_head(4); + header.write_long(unlinkIdAckTag); + header.write_long(unlink_id); + header.write_any(from); + header.write_any(dest); + // fix up length in preamble + header.poke4BE(0, header.size() - 4); - do_send(header); - } + do_send(header); } diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java index 179511c0a5..419ec7e3d7 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java @@ -111,9 +111,14 @@ public class AbstractNode implements OtpTransportFactory { | dFlagBitBinaries | dFlagHandshake23; + /* New mandatory flags in OTP 26 */ + static final long mandatoryFlags26 = dFlagV4PidsRefs + | dFlagUnlinkId; + /* Mandatory flags for distribution. Keep them in sync with DFLAG_DIST_MANDATORY in erts/emulator/beam/dist.h. */ - static final long mandatoryFlags = mandatoryFlags25; + static final long mandatoryFlags = mandatoryFlags25 + | mandatoryFlags26; int ntype = NTYPE_R6; int proto = 0; // tcp/ip @@ -121,8 +126,6 @@ public class AbstractNode implements OtpTransportFactory { int distLow = 6; // Cannot talk to nodes before OTP 23 private int creation = 0x710000; long flags = mandatoryFlags - | dFlagUnlinkId - | dFlagV4PidsRefs | dFlagMandatory25Digest; /* initialize hostname and default cookie */ diff --git a/lib/jinterface/prebuild.keep b/lib/jinterface/prebuild.keep new file mode 100644 index 0000000000..8e695ec83a --- /dev/null +++ b/lib/jinterface/prebuild.keep @@ -0,0 +1 @@ +doc diff --git a/lib/kernel/doc/src/Makefile b/lib/kernel/doc/src/Makefile index fa2e635324..a011358bff 100644 --- a/lib/kernel/doc/src/Makefile +++ b/lib/kernel/doc/src/Makefile @@ -68,7 +68,6 @@ XML_REF3_FILES = application.xml \ seq_trace.xml \ socket.xml \ wrap_log_reader.xml \ - user.xml \ zlib_stub.xml \ $(XML_REF3_ESOCK_EFILES) diff --git a/lib/kernel/doc/src/global.xml b/lib/kernel/doc/src/global.xml index 407ca52a8b..9d2442c84a 100644 --- a/lib/kernel/doc/src/global.xml +++ b/lib/kernel/doc/src/global.xml @@ -452,4 +452,3 @@ <seeerl marker="net_kernel"><c>net_kernel(3)</c></seeerl></p> </section> </erlref> - diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 326b53a873..17edb81002 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -669,7 +669,6 @@ erl -kernel logger '[{handler,default,logger_std_h,#{formatter=>{logger_formatte <seeerl marker="pg"><c>pg(3)</c></seeerl>, <seeerl marker="rpc"><c>rpc(3)</c></seeerl>, <seeerl marker="seq_trace"><c>seq_trace(3)</c></seeerl>, - <seeerl marker="user"><c>user(3)</c></seeerl>, <seeerl marker="stdlib:timer"><c>timer(3)</c></seeerl></p> </section> </appref> diff --git a/lib/kernel/doc/src/ref_man.xml b/lib/kernel/doc/src/ref_man.xml index 7864764685..7382e40610 100644 --- a/lib/kernel/doc/src/ref_man.xml +++ b/lib/kernel/doc/src/ref_man.xml @@ -69,7 +69,6 @@ <xi:include href="rpc.xml"/> <xi:include href="seq_trace.xml"/> <xi:include href="socket.xml"/> - <xi:include href="user.xml"/> <xi:include href="wrap_log_reader.xml"/> <xi:include href="zlib_stub.xml"/> </application> diff --git a/lib/kernel/doc/src/specs.xml b/lib/kernel/doc/src/specs.xml index e856351ce4..fd3877d9a8 100644 --- a/lib/kernel/doc/src/specs.xml +++ b/lib/kernel/doc/src/specs.xml @@ -35,7 +35,6 @@ <xi:include href="../specs/specs_rpc.xml"/> <xi:include href="../specs/specs_seq_trace.xml"/> <xi:include href="../specs/specs_socket.xml"/> - <xi:include href="../specs/specs_user.xml"/> <xi:include href="../specs/specs_wrap_log_reader.xml"/> <xi:include href="../specs/specs_zlib_stub.xml"/> </specs> diff --git a/lib/kernel/doc/src/user.xml b/lib/kernel/doc/src/user.xml deleted file mode 100644 index 38eac1c221..0000000000 --- a/lib/kernel/doc/src/user.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>1996</year> - <year>2016</year> - <holder>Ericsson AB, All Rights Reserved</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - The Initial Developer of the Original Code is Ericsson AB. - </legalnotice> - - <title>user</title> - <prepared>Robert Virding</prepared> - <docno>1</docno> - <date>1996-10-10</date> - <rev>A</rev> - </header> - <module>user</module> - <modulesummary>Standard I/O server.</modulesummary> - <description> - <p><c>user</c> is a server that responds to all messages - defined in the I/O interface. The code in <c>user.erl</c> can be - used as a model for building alternative I/O servers.</p> - </description> -</erlref> - diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl index 16320b64e9..4eef2bb3fc 100644 --- a/lib/kernel/include/dist.hrl +++ b/lib/kernel/include/dist.hrl @@ -72,6 +72,14 @@ ?DFLAG_BIG_CREATION bor ?DFLAG_HANDSHAKE_23)). +%% New mandatory flags in OTP 26 +-define(MANDATORY_DFLAGS_26, (?DFLAG_V4_NC bor + ?DFLAG_UNLINK_ID)). + +%% All mandatory flags +-define(DFLAGS_MANDATORY, (?MANDATORY_DFLAGS_25 bor + ?MANDATORY_DFLAGS_26)). + %% Also update dflag2str() in ../src/dist_util.erl %% when adding flags... diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 2149e89776..954211bac3 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -138,9 +138,9 @@ MODULES = \ seq_trace \ socket \ standard_error \ - user \ user_drv \ user_sup \ + prim_tty \ raw_file_io \ raw_file_io_compressed \ raw_file_io_inflate \ diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl index 3eb87d73c7..1730e1e822 100644 --- a/lib/kernel/src/application.erl +++ b/lib/kernel/src/application.erl @@ -397,13 +397,8 @@ get_env(Application, Key) -> Def :: term(), Val :: term(). -get_env(Application, Key, Def) -> - case get_env(Application, Key) of - {ok, Val} -> - Val; - undefined -> - Def - end. +get_env(Application, Key, Default) -> + application_controller:get_env(Application, Key, Default). -spec get_all_env() -> Env when Env :: [{Par :: atom(), Val :: term()}]. diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 1f428c560b..c5b4445bb8 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -27,7 +27,7 @@ change_application_data/2, prep_config_change/0, config_change/1, which_applications/0, which_applications/1, loaded_applications/0, info/0, set_env/2, - get_pid_env/2, get_env/2, get_pid_all_env/1, get_all_env/1, + get_pid_env/2, get_env/2, get_env/3, get_pid_all_env/1, get_all_env/1, get_pid_key/2, get_key/2, get_pid_all_key/1, get_all_key/1, get_master/1, get_application/1, get_application_module/1, start_type/1, permit_application/2, do_config_diff/2, @@ -343,11 +343,15 @@ get_pid_env(Master, Key) -> end. get_env(AppName, Key) -> - case ets:lookup(ac_tab, {env, AppName, Key}) of - [{_, Val}] -> {ok, Val}; - _ -> undefined + NotFound = make_ref(), + case ets:lookup_element(ac_tab, {env, AppName, Key}, 2, NotFound) of + NotFound -> undefined; + Val -> {ok, Val} end. +get_env(AppName, Key, Default) -> + ets:lookup_element(ac_tab, {env, AppName, Key}, 2, Default). + get_pid_all_env(Master) -> case ets:match(ac_tab, {{application_master, '$1'}, Master}) of [[AppName]] -> get_all_env(AppName); @@ -442,10 +446,7 @@ start_type(Master) -> get_master(AppName) -> - case ets:lookup(ac_tab, {application_master, AppName}) of - [{_, Pid}] -> Pid; - _ -> undefined - end. + ets:lookup_element(ac_tab, {application_master, AppName}, 2, undefined). get_application(Master) -> case ets:match(ac_tab, {{application_master, '$1'}, Master}) of @@ -1541,6 +1542,11 @@ make_appl_i({application, Name, Opts}) when is_atom(Name), is_list(Opts) -> end, Phases = get_opt(start_phases, Opts, undefined), Env = get_opt(env, Opts, []), + case check_para(Env, Name) of + ok -> ok; + {error, Reason} -> + throw({error, {invalid_options, Reason}}) + end, MaxP = get_opt(maxP, Opts, infinity), MaxT = get_opt(maxT, Opts, infinity), IncApps = get_opt(included_applications, Opts, []), diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 001f3ac981..acf145d59f 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -71,6 +71,7 @@ start_link/0, which/1, get_doc/1, + get_doc/2, where_is_file/1, where_is_file/2, set_primary_archive/4, @@ -860,34 +861,52 @@ where_is_file(Tail, File, Path, Files) -> Res :: #docs_v1{}, Reason :: non_existing | missing | file:posix(). get_doc(Mod) when is_atom(Mod) -> + get_doc(Mod, #{sources => [eep48, debug_info]}). + +get_doc(Mod, #{sources:=[Source|Sources]}=Options) -> + GetDoc = fun(Fn) -> R = case Source of + debug_info -> get_doc_chunk_from_ast(Fn); + eep48 -> get_doc_chunk(Fn, Mod) + end, + case R of + {error, missing} -> get_doc(Mod, Options#{sources=>Sources}); + _ -> R + end + end, case which(Mod) of preloaded -> - Fn = filename:join([code:lib_dir(erts),"ebin",atom_to_list(Mod) ++ ".beam"]), - get_doc_chunk(Fn, Mod); + ErtsDir = code:lib_dir(erts), + ErtsEbinDir = + case filelib:is_dir(filename:join([ErtsDir,"ebin"])) of + true -> filename:join([ErtsDir,"ebin"]); + false -> filename:join([ErtsDir,"preloaded","ebin"]) + end, + Fn = filename:join([ErtsEbinDir, atom_to_list(Mod) ++ ".beam"]), + GetDoc(Fn); Error when is_atom(Error) -> {error, Error}; Fn -> - get_doc_chunk(Fn, Mod) - end. + GetDoc(Fn) + end; +get_doc(_, #{sources:=[]}) -> + {error, missing}. get_doc_chunk(Filename, Mod) when is_atom(Mod) -> case beam_lib:chunks(Filename, ["Docs"]) of {error,beam_lib,{missing_chunk,_,_}} -> - case get_doc_chunk(Filename, atom_to_list(Mod)) of - {error,missing} -> - get_doc_chunk_from_ast(Filename); - Error -> - Error - end; + get_doc_chunk(Filename, atom_to_list(Mod)); {error,beam_lib,{file_error,_Filename,_Err}} -> get_doc_chunk(Filename, atom_to_list(Mod)); {ok, {Mod, [{"Docs",Bin}]}} -> {ok,binary_to_term(Bin)} end; get_doc_chunk(Filename, Mod) -> + RootDir = code:root_dir(), case filename:dirname(Filename) of Filename -> {error,missing}; + RootDir -> + {error,missing}; Dir -> ChunkFile = filename:join([Dir,"doc","chunks",Mod ++ ".chunk"]), case file:read_file(ChunkFile) of @@ -904,24 +923,36 @@ get_doc_chunk_from_ast(Filename) -> case beam_lib:chunks(Filename, [abstract_code]) of {error,beam_lib,{missing_chunk,_,_}} -> {error,missing}; + {error,beam_lib,{file_error,_,_}} -> + {error, missing}; {ok, {_Mod, [{abstract_code, {raw_abstract_v1, AST}}]}} -> Docs = get_function_docs_from_ast(AST), + Types = get_type_docs_from_ast(AST), {ok, #docs_v1{ anno = 0, beam_language = erlang, module_doc = none, - metadata = #{ generated => true, otp_doc_vsn => ?CURR_DOC_VERSION }, - docs = Docs }}; + metadata = #{ generated => true, otp_doc_vsn => ?CURR_DOC_VERSION}, + docs = Docs++Types }}; {ok, {_Mod, [{abstract_code,no_abstract_code}]}} -> {error,missing}; Error -> Error end. +get_type_docs_from_ast(AST) -> + lists:flatmap(fun(E) -> get_type_docs_from_ast(E, AST) end, AST). +get_type_docs_from_ast({attribute, Anno, type, {TypeName, _, Ps}}=Meta, _) -> + Arity = length(Ps), + Signature = io_lib:format("~p/~p",[TypeName,Arity]), + [{{type, TypeName, Arity},Anno,[unicode:characters_to_binary(Signature)],none,#{signature => [Meta]}}]; +get_type_docs_from_ast(_, _) -> + []. + get_function_docs_from_ast(AST) -> lists:flatmap(fun(E) -> get_function_docs_from_ast(E, AST) end, AST). get_function_docs_from_ast({function,Anno,Name,Arity,_Code}, AST) -> Signature = io_lib:format("~p/~p",[Name,Arity]), - Specs = lists:filter( + Specs = lists:filter( fun({attribute,_Ln,spec,{FA,_}}) -> case FA of {F,A} -> diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index cfda284ad7..f7a9ba4898 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -728,8 +728,8 @@ recv_name(#hs_data{socket = Socket, f_recv = Recv} = HSData) -> case Recv(Socket, 0, infinity) of {ok, [$N | _] = Data} -> recv_name_new(HSData, Data); - _ -> - ?shutdown(no_node) + Other -> + ?shutdown({no_node, Other}) end. recv_name_new(HSData, diff --git a/lib/kernel/src/erl_compile_server.erl b/lib/kernel/src/erl_compile_server.erl index f4b719068e..b679b277b8 100644 --- a/lib/kernel/src/erl_compile_server.erl +++ b/lib/kernel/src/erl_compile_server.erl @@ -68,7 +68,7 @@ handle_call({compile, Parameters}, From, #st{jobs=Jobs}=St0) -> case verify_context(PathArgs, Parameters, St0) of {ok, St1} -> #{cwd := Cwd, encoding := Enc} = Parameters, - PidRef = spawn_monitor(fun() -> exit(do_compile(ErlcArgs, Cwd, Enc)) end), + PidRef = spawn_monitor(fun() -> exit(do_compile(ErlcArgs, unicode:characters_to_list(Cwd, Enc), Enc)) end), St = St1#st{jobs=Jobs#{PidRef => From}}, {noreply, St#st{timeout=?IDLE_TIMEOUT}}; wrong_config -> @@ -145,8 +145,9 @@ do_compile(ErlcArgs, Cwd, Enc) -> {error, StdOutput, StdErrorOutput} end. -parse_command_line(#{command_line := CmdLine, cwd := Cwd}) -> - parse_command_line_1(CmdLine, Cwd, [], []). +parse_command_line(#{command_line := CmdLine0, cwd := Cwd, encoding := Enc}) -> + CmdLine = lists:map(fun(A) -> unicode:characters_to_list(A, Enc) end, CmdLine0), + parse_command_line_1(CmdLine, unicode:characters_to_list(Cwd, Enc), [], []). parse_command_line_1(["-pa", Pa|T], Cwd, PaAcc, PzAcc) -> parse_command_line_1(T, Cwd, [Pa|PaAcc], PzAcc); diff --git a/lib/kernel/src/erl_erts_errors.erl b/lib/kernel/src/erl_erts_errors.erl index 4d23112b83..fe7758dae0 100644 --- a/lib/kernel/src/erl_erts_errors.erl +++ b/lib/kernel/src/erl_erts_errors.erl @@ -352,8 +352,19 @@ format_erlang_error(demonitor, [_], _) -> format_erlang_error(demonitor, [Ref,Options], _) -> Arg1 = must_be_ref(Ref), [Arg1,maybe_option_list_error(Options, Arg1)]; -format_erlang_error(display_string, [_], _) -> +format_erlang_error(display_string, [_], none) -> [not_string]; +format_erlang_error(display_string, [_], Cause) -> + maybe_posix_message(Cause, false); +format_erlang_error(display_string, [Device, _], none) -> + case lists:member(Device,[stdin,stdout,stderr]) of + true -> + [[],not_string]; + false -> + [not_device,[]] + end; +format_erlang_error(display_string, [_, _], Cause) -> + maybe_posix_message(Cause, true); format_erlang_error(element, [Index, Tuple], _) -> [if not is_integer(Index) -> @@ -1430,8 +1441,23 @@ is_flat_char_list([H|T]) -> is_flat_char_list([]) -> true; is_flat_char_list(_) -> false. +maybe_posix_message(Cause, HasDevice) -> + case erl_posix_msg:message(Cause) of + "unknown POSIX error" -> + unknown; + PosixStr when HasDevice -> + [unicode:characters_to_binary( + io_lib:format("~ts (~tp)",[PosixStr, Cause]))]; + PosixStr when not HasDevice -> + [{general, + unicode:characters_to_binary( + io_lib:format("~ts (~tp)",[PosixStr, Cause]))}] + end. + format_error_map([""|Es], ArgNum, Map) -> format_error_map(Es, ArgNum + 1, Map); +format_error_map([{general, E}|Es], ArgNum, Map) -> + format_error_map(Es, ArgNum, Map#{ general => expand_error(E)}); format_error_map([E|Es], ArgNum, Map) -> format_error_map(Es, ArgNum + 1, Map#{ArgNum => expand_error(E)}); format_error_map([], _, Map) -> @@ -1519,6 +1545,8 @@ expand_error(not_ref) -> <<"not a reference">>; expand_error(not_string) -> <<"not a list of characters">>; +expand_error(not_device) -> + <<"not a valid device type">>; expand_error(not_tuple) -> <<"not a tuple">>; expand_error(range) -> diff --git a/lib/kernel/src/group.erl b/lib/kernel/src/group.erl index 8410c1a4b5..62d427f3a4 100644 --- a/lib/kernel/src/group.erl +++ b/lib/kernel/src/group.erl @@ -21,51 +21,49 @@ %% A group leader process for user io. --export([start/2, start/3, server/3]). --export([interfaces/1]). +-export([start/2, start/3, whereis_shell/0, server/4]). start(Drv, Shell) -> start(Drv, Shell, []). start(Drv, Shell, Options) -> - spawn_link(group, server, [Drv, Shell, Options]). + Ancestors = [self() | case get('$ancestors') of + undefined -> []; + Anc -> Anc + end], + spawn_link(group, server, [Ancestors, Drv, Shell, Options]). -server(Drv, Shell, Options) -> +server(Ancestors, Drv, Shell, Options) -> process_flag(trap_exit, true), + _ = [put('$ancestors', Ancestors) || Shell =/= {}], edlin:init(), put(line_buffer, proplists:get_value(line_buffer, Options, group_history:load())), put(read_mode, list), put(user_drv, Drv), - put(expand_fun, - proplists:get_value(expand_fun, Options, - fun(B) -> edlin_expand:expand(B) end)), + ExpandFun = case proplists:get_value(expand_fun, Options, + fun edlin_expand:expand/2) of + Fun when is_function(Fun, 1) -> fun(X,_) -> Fun(X) end; + Fun -> Fun + end, + put(expand_fun,ExpandFun), put(echo, proplists:get_value(echo, Options, true)), - - start_shell(Shell), - server_loop(Drv, get(shell), []). - -%% Return the pid of user_drv and the shell process. -%% Note: We can't ask the group process for this info since it -%% may be busy waiting for data from the driver. -interfaces(Group) -> - case process_info(Group, dictionary) of - {dictionary,Dict} -> - get_pids(Dict, [], false); - _ -> - [] + put(expand_below, proplists:get_value(expand_below, Options, true)), + + server_loop(Drv, start_shell(Shell), []). + +whereis_shell() -> + case node(group_leader()) of + Node when Node =:= node() -> + case user_drv:whereis_group() of + undefined -> undefined; + GroupPid -> + {dictionary, Dict} = erlang:process_info(GroupPid, dictionary), + proplists:get_value(shell, Dict) + end; + OtherNode -> + erpc:call(OtherNode, group, whereis_shell, []) end. -get_pids([Drv = {user_drv,_} | Rest], Found, _) -> - get_pids(Rest, [Drv | Found], true); -get_pids([Sh = {shell,_} | Rest], Found, Active) -> - get_pids(Rest, [Sh | Found], Active); -get_pids([_ | Rest], Found, Active) -> - get_pids(Rest, Found, Active); -get_pids([], Found, true) -> - Found; -get_pids([], _Found, false) -> - []. - %% start_shell(Shell) %% Spawn a shell with its group_leader from the beginning set to ourselves. %% If Shell a pid the set its group_leader. @@ -81,7 +79,8 @@ start_shell(Shell) when is_function(Shell) -> start_shell(Shell) when is_pid(Shell) -> group_leader(self(), Shell), % we are the shells group leader link(Shell), % we're linked to it. - put(shell, Shell); + put(shell, Shell), + Shell; start_shell(_Shell) -> ok. @@ -92,9 +91,11 @@ start_shell1(M, F, Args) -> Shell when is_pid(Shell) -> group_leader(G, self()), link(Shell), % we're linked to it. - put(shell, Shell); + put(shell, Shell), + Shell; Error -> % start failure exit(Error) % let the group process crash + end. start_shell1(Fun) -> @@ -104,19 +105,20 @@ start_shell1(Fun) -> Shell when is_pid(Shell) -> group_leader(G, self()), link(Shell), % we're linked to it. - put(shell, Shell); + put(shell, Shell), + Shell; Error -> % start failure exit(Error) % let the group process crash end. server_loop(Drv, Shell, Buf0) -> receive - {io_request,From,ReplyAs,Req} when is_pid(From) -> + {io_request,From,ReplyAs,Req} when is_pid(From) -> %% This io_request may cause a transition to a couple of %% selective receive loops elsewhere in this module. Buf = io_request(Req, From, ReplyAs, Drv, Shell, Buf0), server_loop(Drv, Shell, Buf); - {reply,{{From,ReplyAs},Reply}} -> + {reply,{From,ReplyAs},Reply} -> io_reply(From, ReplyAs, Reply), server_loop(Drv, Shell, Buf0); {driver_id,ReplyTo} -> @@ -152,30 +154,40 @@ exit_shell(Reason) -> get_tty_geometry(Drv) -> Drv ! {self(),tty_geometry}, receive - {Drv,tty_geometry,Geometry} -> - Geometry + {Drv,tty_geometry,Geometry} -> + Geometry after 2000 -> - timeout + timeout end. get_unicode_state(Drv) -> Drv ! {self(),get_unicode_state}, receive - {Drv,get_unicode_state,UniState} -> - UniState; - {Drv,get_unicode_state,error} -> - {error, internal} + {Drv,get_unicode_state,UniState} -> + UniState; + {Drv,get_unicode_state,error} -> + {error, internal} after 2000 -> - {error,timeout} + {error,timeout} end. set_unicode_state(Drv,Bool) -> Drv ! {self(),set_unicode_state,Bool}, receive - {Drv,set_unicode_state,_OldUniState} -> - ok + {Drv,set_unicode_state,_OldUniState} -> + ok after 2000 -> - timeout + timeout + end. + +get_terminal_state(Drv) -> + Drv ! {self(),get_terminal_state}, + receive + {Drv,get_terminal_state,UniState} -> + UniState; + {Drv,get_terminal_state,error} -> + {error, internal} + after 2000 -> + {error,timeout} end. - io_request(Req, From, ReplyAs, Drv, Shell, Buf0) -> case io_request(Req, Drv, Shell, {From,ReplyAs}, Buf0) of @@ -197,12 +209,12 @@ io_request(Req, From, ReplyAs, Drv, Shell, Buf0) -> end. -%% Put_chars, unicode is the normal message, characters are always in +%% Put_chars, unicode is the normal message, characters are always in %%standard unicode %% format. -%% You might be tempted to send binaries unchecked, but the driver +%% You might be tempted to send binaries unchecked, but the driver %% expects unicode, so that is what we should send... -%% io_request({put_chars,unicode,Binary}, Drv, Buf) when is_binary(Binary) -> +%% io_request({put_chars,unicode,Binary}, Drv, Buf) when is_binary(Binary) -> %% send_drv(Drv, {put_chars,Binary}), %% {ok,ok,Buf}; %% @@ -211,7 +223,7 @@ io_request(Req, From, ReplyAs, Drv, Shell, Buf0) -> io_request({put_chars,unicode,Chars}, Drv, _Shell, From, Buf) -> case catch unicode:characters_to_binary(Chars,utf8) of Binary when is_binary(Binary) -> - send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), + send_drv(Drv, {put_chars_sync, unicode, Binary, From}), {noreply,Buf}; _ -> {error,{error,{put_chars, unicode,Chars}},Buf} @@ -219,12 +231,12 @@ io_request({put_chars,unicode,Chars}, Drv, _Shell, From, Buf) -> io_request({put_chars,unicode,M,F,As}, Drv, _Shell, From, Buf) -> case catch apply(M, F, As) of Binary when is_binary(Binary) -> - send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), + send_drv(Drv, {put_chars_sync, unicode, Binary, From}), {noreply,Buf}; Chars -> case catch unicode:characters_to_binary(Chars,utf8) of B when is_binary(B) -> - send_drv(Drv, {put_chars_sync, unicode, B, {From,ok}}), + send_drv(Drv, {put_chars_sync, unicode, B, From}), {noreply,Buf}; _ -> {error,{error,F},Buf} @@ -233,12 +245,12 @@ io_request({put_chars,unicode,M,F,As}, Drv, _Shell, From, Buf) -> io_request({put_chars,latin1,Binary}, Drv, _Shell, From, Buf) when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, unicode:characters_to_binary(Binary,latin1), - {From,ok}}), + From}), {noreply,Buf}; io_request({put_chars,latin1,Chars}, Drv, _Shell, From, Buf) -> case catch unicode:characters_to_binary(Chars,latin1) of Binary when is_binary(Binary) -> - send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), + send_drv(Drv, {put_chars_sync, unicode, Binary, From}), {noreply,Buf}; _ -> {error,{error,{put_chars,latin1,Chars}},Buf} @@ -248,12 +260,12 @@ io_request({put_chars,latin1,M,F,As}, Drv, _Shell, From, Buf) -> Binary when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, unicode:characters_to_binary(Binary,latin1), - {From,ok}}), + From}), {noreply,Buf}; Chars -> case catch unicode:characters_to_binary(Chars,latin1) of B when is_binary(B) -> - send_drv(Drv, {put_chars_sync, unicode, B, {From,ok}}), + send_drv(Drv, {put_chars_sync, unicode, B, From}), {noreply,Buf}; _ -> {error,{error,F},Buf} @@ -431,8 +443,8 @@ getopts(Drv,Buf) -> true -> unicode; _ -> latin1 end}, - {ok,[Exp,Echo,Bin,Uni],Buf}. - + Tty = {terminal, get_terminal_state(Drv)}, + {ok,[Exp,Echo,Bin,Uni,Tty],Buf}. %% get_chars_*(Prompt, Module, Function, XtraArgument, Drv, Buffer) %% Gets characters from the input Drv until as the applied function @@ -485,11 +497,20 @@ get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf0, State, Encoding) -> get_chars_apply(Pbs, M, F, Xa, Drv, Shell, Buf, State0, Line, Encoding) -> case catch M:F(State0, cast(Line,get(read_mode), Encoding), Encoding, Xa) of + {stop,Result,eof} -> + {ok,Result,eof}; {stop,Result,Rest} -> - {ok,Result,append(Rest, Buf, Encoding)}; + case {M,F} of + {io_lib, get_until} -> + save_line_buffer(Line, get_lines(new_stack(get(line_buffer)))), + {ok,Result,append(Rest, Buf, Encoding)}; + _ -> + {ok,Result,append(Rest, Buf, Encoding)} + end; {'EXIT',_} -> {error,{error,err_func(M, F, Xa)},[]}; State1 -> + save_line_buffer(Line, get_lines(new_stack(get(line_buffer)))), get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf, State1, Encoding) end. @@ -529,10 +550,21 @@ get_line(Chars, Pbs, Drv, Shell, Encoding) -> get_line1(edlin:edit_line(Chars, Cont), Drv, Shell, new_stack(get(line_buffer)), Encoding). -get_line1({done,Line,Rest,Rs}, Drv, _Shell, Ls, _Encoding) -> +get_line1({done,Line,Rest,Rs}, Drv, _Shell, _Ls, _Encoding) -> send_drv_reqs(Drv, Rs), - save_line_buffer(Line, get_lines(Ls)), {done,Line,Rest}; +get_line1({undefined,{_A, Mode, Char}, _Cs, Cont, Rs}, Drv, Shell, Ls0, Encoding) + when ((Mode =:= none) and (Char =:= $\^O)) -> + send_drv_reqs(Drv, Rs), + Buffer = edlin:current_chars(Cont), + send_drv(Drv, {open_editor, Buffer}), + receive + {Drv, {editor_data, Cs}} -> + send_drv_reqs(Drv, edlin:erase_line(Cont)), + {more_chars,NewCont,NewRs} = edlin:start(edlin:prompt(Cont)), + send_drv_reqs(Drv, NewRs), + get_line1(edlin:edit_line(Cs, NewCont), Drv, Shell, Ls0, Encoding) + end; get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Shell, Ls0, Encoding) when ((Mode =:= none) and (Char =:= $\^P)) or ((Mode =:= meta_left_sq_bracket) and (Char =:= $A)) -> @@ -590,21 +622,60 @@ get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Shell, Ls, Encoding) {more_chars,Ncont,Nrs} = edlin:start(Pbs, search), send_drv_reqs(Drv, Nrs), get_line1(edlin:edit_line1(Cs, Ncont), Drv, Shell, Ls, Encoding); -get_line1({expand, Before, Cs0, Cont,Rs}, Drv, Shell, Ls0, Encoding) -> +get_line1({Expand, Before, Cs0, Cont,Rs}, Drv, Shell, Ls0, Encoding) + when Expand =:= expand; Expand =:= expand_full -> send_drv_reqs(Drv, Rs), ExpandFun = get(expand_fun), - {Found, Add, Matches} = ExpandFun(Before), + {Found, CompleteChars, Matches} = ExpandFun(Before, []), case Found of - no -> send_drv(Drv, beep); - yes -> ok + no -> send_drv(Drv, beep); + _ -> ok end, - Cs1 = append(Add, Cs0, Encoding), %%XXX:PaN should this always be unicode? - Cs = case Matches of - [] -> Cs1; - _ -> MatchStr = edlin_expand:format_matches(Matches), - send_drv(Drv, {put_chars, unicode, unicode:characters_to_binary(MatchStr,unicode)}), - [$\^L | Cs1] - end, + {Width, _Height} = get_tty_geometry(Drv), + Cs1 = append(CompleteChars, Cs0, Encoding), + + MatchStr = case Matches of + [] -> []; + _ -> edlin_expand:format_matches(Matches, Width) + end, + Cs = case {Cs1, MatchStr} of + {_, []} -> Cs1; + {Cs1, _} when Cs1 =/= [] -> Cs1; + _ -> + NlMatchStr = unicode:characters_to_binary("\n"++MatchStr), + case get(expand_below) of + true -> + Lines = string:split(string:trim(MatchStr), "\n", all), + NoLines = length(Lines), + if NoLines > 5, Expand =:= expand -> + %% Only show 5 lines to start with + [L1,L2,L3,L4,L5|_] = Lines, + String = lists:join( + $\n, + [L1,L2,L3,L4,L5, + io_lib:format("Press tab to see all ~p expansions", + [edlin_expand:number_matches(Matches)])]), + send_drv(Drv, {put_expand, unicode, + unicode:characters_to_binary(String)}), + Cs1; + true -> + case get_tty_geometry(Drv) of + {_, Rows} when Rows > NoLines -> + %% If all lines fit on screen, we expand below + send_drv(Drv, {put_expand, unicode, NlMatchStr}), + Cs1; + _ -> + %% If there are more results than fit on + %% screen we expand above + send_drv(Drv, {put_chars, unicode, NlMatchStr}), + [$\e, $l | Cs1] + end + end; + false -> + send_drv(Drv, {put_chars, unicode, NlMatchStr}), + [$\e, $l | Cs1] + end + end, get_line1(edlin:edit_line(Cs, Cont), Drv, Shell, Ls0, Encoding); get_line1({undefined,_Char,Cs,Cont,Rs}, Drv, Shell, Ls, Encoding) -> send_drv_reqs(Drv, Rs), @@ -674,7 +745,7 @@ more_data(What, Cont0, Drv, Shell, Ls, Encoding) -> io_request(Req, From, ReplyAs, Drv, Shell, []), %WRONG!!! send_drv_reqs(Drv, edlin:redraw_line(Cont)), get_line1({more_chars,Cont,[]}, Drv, Shell, Ls, Encoding); - {reply,{{From,ReplyAs},Reply}} -> + {reply,{From,ReplyAs},Reply} -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), more_data(What, Cont0, Drv, Shell, Ls, Encoding); @@ -702,7 +773,7 @@ get_line_echo_off1({Chars,[]}, Drv, Shell) -> {io_request,From,ReplyAs,Req} when is_pid(From) -> io_request(Req, From, ReplyAs, Drv, Shell, []), get_line_echo_off1({Chars,[]}, Drv, Shell); - {reply,{{From,ReplyAs},Reply}} when From =/= undefined -> + {reply,{From,ReplyAs},Reply} when From =/= undefined -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), get_line_echo_off1({Chars,[]},Drv, Shell); @@ -713,6 +784,8 @@ get_line_echo_off1({Chars,[]}, Drv, Shell) -> {'EXIT',Shell,R} -> exit(R) end; +get_line_echo_off1(eof, _Drv, _Shell) -> + {done,eof,eof}; get_line_echo_off1({Chars,Rest}, _Drv, _Shell) -> {done,lists:reverse(Chars),case Rest of done -> []; _ -> Rest end}. @@ -729,7 +802,7 @@ get_chars_echo_off1(Drv, Shell) -> {io_request,From,ReplyAs,Req} when is_pid(From) -> io_request(Req, From, ReplyAs, Drv, Shell, []), get_chars_echo_off1(Drv, Shell); - {reply,{{From,ReplyAs},Reply}} when From =/= undefined -> + {reply,{From,ReplyAs},Reply} when From =/= undefined -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), get_chars_echo_off1(Drv, Shell); @@ -750,8 +823,10 @@ get_chars_echo_off1(Drv, Shell) -> %% - ^d in posix/icanon mode: eof, delete-forward in edlin %% - ^r in posix/icanon mode: reprint (silly in echo-off mode :-)) %% - ^w in posix/icanon mode: word-erase (produces a beep in edlin) +edit_line(eof, []) -> + eof; edit_line(eof, Chars) -> - {Chars,done}; + {Chars,eof}; edit_line([],Chars) -> {Chars,[]}; edit_line([$\r,$\n|Cs],Chars) -> @@ -877,7 +952,7 @@ get_password1({Chars,[]}, Drv, Shell) -> %% set to []. But do we expect anything but plain output? get_password1({Chars, []}, Drv, Shell); - {reply,{{From,ReplyAs},Reply}} -> + {reply,{From,ReplyAs},Reply} -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), get_password1({Chars, []},Drv, Shell); diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl index 408f563909..4937bc0751 100644 --- a/lib/kernel/src/inet_db.erl +++ b/lib/kernel/src/inet_db.erl @@ -579,9 +579,7 @@ res_update(Option, TagTm) -> end. db_get(Name) -> - try ets:lookup_element(inet_db, Name, 2) - catch error:badarg -> undefined - end. + ets:lookup_element(inet_db, Name, 2, undefined). add_rr(RR) -> %% Questionable if we need to support this; diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index b53de6281b..15280b7f05 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -30,6 +30,8 @@ %% Generalized dist API -export([gen_listen/3, gen_accept/2, gen_accept_connection/6, gen_setup/6, gen_select/2, gen_address/1]). +%% OTP internal (e.g ssl) +-export([gen_hs_data/2, nodelay/0]). %% internal exports @@ -42,6 +44,8 @@ -include("dist.hrl"). -include("dist_util.hrl"). +-define(PROTOCOL, tcp). + %% ------------------------------------------------------------ %% Select this protocol based on node name %% select(Node) => Bool @@ -72,6 +76,34 @@ gen_address(Driver) -> get_tcp_address(Driver). %% ------------------------------------------------------------ +%% Set up the general fields in #hs_data{} +%% ------------------------------------------------------------ +gen_hs_data(Driver, Socket) -> + Nodelay = nodelay(), + #hs_data{ + socket = Socket, + f_send = fun Driver:send/2, + f_recv = fun Driver:recv/3, + f_setopts_pre_nodeup = + fun (S) -> + inet:setopts( + S, + [{active, false}, {packet, 4}, Nodelay]) + end, + f_setopts_post_nodeup = + fun (S) -> + inet:setopts( + S, + [{active, true}, {packet,4}, + {deliver, port}, binary, Nodelay]) + end, + f_getll = fun inet:getll/1, + mf_tick = fun (S) -> ?MODULE:tick(Driver, S) end, + mf_getstat = fun ?MODULE:getstat/1, + mf_setopts = fun ?MODULE:setopts/2, + mf_getopts = fun ?MODULE:getopts/2}. + +%% ------------------------------------------------------------ %% Create the listen socket, i.e. the port that this erlang %% node is accessible through. %% ------------------------------------------------------------ @@ -194,7 +226,7 @@ gen_accept(Driver, Listen) -> accept_loop(Driver, Kernel, Listen) -> case Driver:accept(Listen) of {ok, Socket} -> - Kernel ! {accept,self(),Socket,Driver:family(),tcp}, + Kernel ! {accept,self(),Socket,Driver:family(),?PROTOCOL}, _ = controller(Driver, Kernel, Socket), accept_loop(Driver, Kernel, Listen); Error -> @@ -243,40 +275,19 @@ do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) -> Timer = dist_util:start_timer(SetupTime), case check_ip(Driver, Socket) of true -> - HSData = #hs_data{ - kernel_pid = Kernel, - this_node = MyNode, - socket = Socket, - timer = Timer, - this_flags = 0, - allowed = Allowed, - f_send = fun Driver:send/2, - f_recv = fun Driver:recv/3, - f_setopts_pre_nodeup = - fun(S) -> - inet:setopts(S, - [{active, false}, - {packet, 4}, - nodelay()]) - end, - f_setopts_post_nodeup = - fun(S) -> - inet:setopts(S, - [{active, true}, - {deliver, port}, - {packet, 4}, - binary, - nodelay()]) - end, - f_getll = fun(S) -> - inet:getll(S) - end, - f_address = fun(S, Node) -> get_remote_id(Driver, S, Node) end, - mf_tick = fun(S) -> ?MODULE:tick(Driver, S) end, - mf_getstat = fun ?MODULE:getstat/1, - mf_setopts = fun ?MODULE:setopts/2, - mf_getopts = fun ?MODULE:getopts/2 - }, + Family = Driver:family(), + HSData = + (gen_hs_data(Driver, Socket)) + #hs_data{ + kernel_pid = Kernel, + this_node = MyNode, + timer = Timer, + this_flags = 0, + allowed = Allowed, + f_address = + fun (S, Node) -> + get_remote_id(Family, S, Node) + end}, dist_util:handshake_other_started(HSData); {false,IP} -> error_msg("** Connection attempt from " @@ -304,13 +315,13 @@ nodelay() -> %% ------------------------------------------------------------ %% Get remote information about a Socket. %% ------------------------------------------------------------ -get_remote_id(Driver, Socket, Node) -> +get_remote_id(Family, Socket, Node) -> case inet:peername(Socket) of {ok,Address} -> case split_node(atom_to_list(Node), $@, []) of [_,Host] -> #net_address{address=Address,host=Host, - protocol=tcp,family=Driver:family()}; + protocol=?PROTOCOL,family=Family}; _ -> %% No '@' or more than one '@' in node name. ?shutdown(no_node) @@ -373,49 +384,24 @@ do_setup_connect(Driver, Kernel, Node, Address, AddressFamily, connect_options([{active, false}, {packet, 2}])) of {ok, Socket} -> - HSData = #hs_data{ - kernel_pid = Kernel, - other_node = Node, - this_node = MyNode, - socket = Socket, - timer = Timer, - this_flags = 0, - other_version = Version, - f_send = fun Driver:send/2, - f_recv = fun Driver:recv/3, - f_setopts_pre_nodeup = - fun(S) -> - inet:setopts - (S, - [{active, false}, - {packet, 4}, - nodelay()]) - end, - f_setopts_post_nodeup = - fun(S) -> - inet:setopts - (S, - [{active, true}, - {deliver, port}, - {packet, 4}, - nodelay()]) - end, - - f_getll = fun inet:getll/1, - f_address = - fun(_,_) -> - #net_address{ - address = {Ip,TcpPort}, - host = Address, - protocol = tcp, - family = AddressFamily} - end, - mf_tick = fun(S) -> ?MODULE:tick(Driver, S) end, - mf_getstat = fun ?MODULE:getstat/1, - request_type = Type, - mf_setopts = fun ?MODULE:setopts/2, - mf_getopts = fun ?MODULE:getopts/2 - }, + HSData = + (gen_hs_data(Driver, Socket)) + #hs_data{ + kernel_pid = Kernel, + other_node = Node, + this_node = MyNode, + timer = Timer, + this_flags = 0, + other_version = Version, + f_address = + fun(_,_) -> + #net_address{ + address = {Ip,TcpPort}, + host = Address, + protocol = ?PROTOCOL, + family = AddressFamily} + end, + request_type = Type}, dist_util:handshake_we_started(HSData); _ -> %% Other Node may have closed since @@ -492,7 +478,7 @@ get_tcp_address(Driver) -> {ok, Host} = inet:gethostname(), #net_address { host = Host, - protocol = tcp, + protocol = ?PROTOCOL, family = Driver:family() }. diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 1d854c49dc..6f73acb032 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -83,9 +83,9 @@ os, ram_file, rpc, - user, user_drv, user_sup, + prim_tty, disk_log, disk_log_1, disk_log_server, @@ -158,6 +158,7 @@ {shell_docs_ansi,auto} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-13.1", "stdlib-4.1.1", "sasl-3.0", "crypto-5.0"]} + {runtime_dependencies, ["erts-@OTP-18248@", "stdlib-@OTP-17932@", + "sasl-3.0", "crypto-5.0"]} ] }. diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index 579b4f8f73..f64ec85b32 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -48,7 +48,8 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config = add_default_config(Config0), Meta1 = maybe_add_legacy_header(Level,Meta,Config), Template = maps:get(template,Config), - {BT,AT0} = lists:splitwith(fun(msg) -> false; (_) -> true end, Template), + LinearTemplate = linearize_template(Meta1,Template), + {BT,AT0} = lists:splitwith(fun(msg) -> false; (_) -> true end, LinearTemplate), {DoMsg,AT} = case AT0 of [msg|Rest] -> {true,Rest}; @@ -89,6 +90,18 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) end, truncate(B,MsgStr,A,maps:get(max_size,Config)). +linearize_template(Data,[{Key,IfExist,Else}|Format]) -> + BranchForUse = + case value(Key,Data) of + {ok,_Value} -> linearize_template(Data,IfExist); + error -> linearize_template(Data,Else) + end, + BranchForUse ++ linearize_template(Data,Format); +linearize_template(Data,[StrOrKey|Format]) -> + [StrOrKey|linearize_template(Data,Format)]; +linearize_template(_Data,[]) -> + []. + trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> trim(T,Rev); trim([H|T],false) when is_list(H) -> @@ -110,13 +123,6 @@ trim(String,_) -> do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; -do_format(Level,Data,[{Key,IfExist,Else}|Format],Config) -> - String = - case value(Key,Data) of - {ok,Value} -> do_format(Level,Data#{Key=>Value},IfExist,Config); - error -> do_format(Level,Data,Else,Config) - end, - [String|do_format(Level,Data,Format,Config)]; do_format(Level,Data,[Key|Format],Config) when is_atom(Key) orelse (is_list(Key) andalso is_atom(hd(Key))) -> diff --git a/lib/kernel/src/logger_simple_h.erl b/lib/kernel/src/logger_simple_h.erl index fe8db09e83..72b0236183 100644 --- a/lib/kernel/src/logger_simple_h.erl +++ b/lib/kernel/src/logger_simple_h.erl @@ -68,14 +68,14 @@ log(#{msg:=_,meta:=#{time:=_}=M}=Log,_Config) -> %% Log directly from client just to get it out case maps:get(internal_log_event, M, false) of false -> - do_log( + do_log(simple, #{level=>error, msg=>{report,{error,simple_handler_process_dead}}, meta=>#{time=>logger:timestamp()}}); true -> ok end, - do_log(Log); + do_log(simple,Log); _ -> ?MODULE ! {log,Log} end, @@ -90,9 +90,9 @@ log(_,_) -> init(Starter) -> register(?MODULE,self()), Starter ! {self(),started}, - loop(#{buffer_size=>10,dropped=>0,buffer=>[]}). + loop(rich, #{buffer_size=>10,dropped=>0,buffer=>[]}). -loop(Buffer) -> +loop(Mode, Buffer) -> receive stop -> %% We replay the logger messages if there is @@ -109,11 +109,11 @@ loop(Buffer) -> unlink(whereis(logger)), ok; {log,#{msg:=_,meta:=#{time:=_}}=Log} -> - do_log(Log), - loop(update_buffer(Buffer,Log)); + NewMode = do_log(Mode, Log), + loop(NewMode, update_buffer(Buffer,Log)); _ -> %% Unexpected message - flush it! - loop(Buffer) + loop(Mode, Buffer) end. update_buffer(#{buffer_size:=0,dropped:=D}=Buffer,_Log) -> @@ -139,16 +139,42 @@ drop_msg(N) -> %%%----------------------------------------------------------------- %%% Internal -do_log(Log) -> - try - Str = logger_formatter:format(Log, - #{ legacy_header => true, single_line => false - ,depth => unlimited, time_offset => "" - }), - erlang:display_string(lists:flatten(unicode:characters_to_list(Str))) - catch _E:_R:_ST -> - % erlang:display({_E,_R,_ST}), - display_log(Log) +%% If the init process is busy (for instance doing a shutdown) +%% we can get blocked while trying to load code. So we spawn a process +%% for each log message that can potentially block. If the logging cannot +%% be done within 300ms, we instead log the raw log message to stdout +%% and switch mode to always log using the raw format. +do_log(simple, Log) -> + display_log(Log), simple; +do_log(rich = Mode, Log) -> + + {Pid, Ref} = + spawn_monitor( + fun() -> + Str = logger_formatter:format( + Log, + #{ legacy_header => true, single_line => false, + depth => unlimited, time_offset => "" + }), + erlang:display_string(stdout, lists:flatten(unicode:characters_to_list(Str))) + end), + receive + {'DOWN', Ref, _, _, normal} -> + Mode; + {'DOWN', Ref, _, _, _Else} -> + display_log(Log), + Mode + after 300 -> + %% init:terminate/3 sleeps for 500 ms before exiting, + %% so we wait for 300 ms for the log to happen + exit(Pid, kill), + receive + {'DOWN', Ref, _, _, normal} -> + Mode; + {'DOWN', Ref, _, _, _Else} -> + display_log(Log), + simple + end end. display_log(#{msg:={report,Report}, @@ -165,6 +191,7 @@ display_date(Timestamp) when is_integer(Timestamp) -> {{Y,Mo,D},{H,Mi,S}} = erlang:universaltime_to_localtime( erlang:posixtime_to_universaltime(Sec)), erlang:display_string( + stdout, integer_to_list(Y) ++ "-" ++ pad(Mo,2) ++ "-" ++ pad(D,2) ++ " " ++ @@ -182,7 +209,8 @@ pad(Str,Size) -> display({string,Chardata}) -> try unicode:characters_to_list(Chardata) of - String -> erlang:display_string(String), erlang:display_string("\n") + String -> erlang:display_string(stdout, String), + erlang:display_string(stdout, "\n") catch _:_ -> erlang:display(Chardata) end; display({report,Report}) when is_map(Report) -> @@ -190,9 +218,9 @@ display({report,Report}) when is_map(Report) -> display({report,Report}) -> display_report(Report); display({F, A}) when is_list(F), is_list(A) -> - erlang:display_string(F ++ "\n"), + erlang:display_string(stdout, F ++ "\n"), [begin - erlang:display_string("\t"), + erlang:display_string(stdout, "\t"), erlang:display(Arg) end || Arg <- A], ok. @@ -203,7 +231,7 @@ display_report(Atom, A) when is_atom(Atom) -> AtomString = atom_to_list(Atom), AtomLength = length(AtomString), Padding = lists:duplicate(ColumnWidth - AtomLength, $\s), - erlang:display_string(AtomString ++ Padding), + erlang:display_string(stdout, AtomString ++ Padding), display_report(A); display_report(F, A) -> erlang:display({F, A}). @@ -216,13 +244,12 @@ display_report([A, []]) -> display_report(A = [_|_]) -> case lists:all(fun({Key,_Value}) -> is_atom(Key); (_) -> false end, A) of true -> - erlang:display_string("\n"), + erlang:display_string(stdout, "\n"), lists:foreach( fun({Key, Value}) -> erlang:display_string( - " " ++ - atom_to_list(Key) ++ - ": "), + stdout, + " " ++ atom_to_list(Key) ++ ": "), erlang:display(Value) end, A); false -> diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 1fe740d5ec..1b2fabad72 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -507,6 +507,15 @@ ensure_open(Filename, Modes) -> exit({could_not_create_dir_for_file,Error}) end. +write_to_dev(Bin,#{dev:=standard_io}=State) -> + try + io:put_chars(user, Bin) + catch _E:_R -> + io:put_chars( + standard_error, "Failed to write log message to stdout, trying stderr\n"), + io:put_chars(standard_error, Bin) + end, + State; write_to_dev(Bin,#{dev:=DevName}=State) -> io:put_chars(DevName, Bin), State; diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index 2123e3e1bf..e78706e9f3 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1022,7 +1022,7 @@ handle_info({dist_ctrlr, Ctrlr, Node, SetupPid} = Msg, %% %% A node has successfully been connected. %% -handle_info({SetupPid, {nodeup,Node,Address,Type,NamedMe}}, +handle_info({SetupPid, {nodeup,Node,Address,Type,NamedMe} = Nodeup}, #state{tick = Tick} = State) -> case ets:lookup(sys_dist, Node) of [Conn] when (Conn#connection.state =:= pending) @@ -1043,6 +1043,7 @@ handle_info({SetupPid, {nodeup,Node,Address,Type,NamedMe}}, true -> State#state{node = node()}; false -> State end, + verbose(Nodeup, 1, State1), {noreply, State1}; _ -> SetupPid ! {self(), bad_request}, @@ -1157,7 +1158,6 @@ handle_info({'DOWN', ReqId, process, _Pid, Reason}, %% Handle different types of process terminations. %% handle_info({'EXIT', From, Reason}, State) -> - verbose({'EXIT', From, Reason}, 1, State), handle_exit(From, Reason, State); %% @@ -1271,30 +1271,33 @@ handle_exit(Pid, Reason, State) -> catch do_handle_exit(Pid, Reason, State). do_handle_exit(Pid, Reason, State) -> - listen_exit(Pid, State), - accept_exit(Pid, State), + listen_exit(Pid, Reason, State), + accept_exit(Pid, Reason, State), conn_own_exit(Pid, Reason, State), dist_ctrlr_exit(Pid, Reason, State), - pending_own_exit(Pid, State), - ticker_exit(Pid, State), - restarter_exit(Pid, State), + pending_own_exit(Pid, Reason, State), + ticker_exit(Pid, Reason, State), + restarter_exit(Pid, Reason, State), + verbose({'EXIT', Pid, Reason}, 1, State), {noreply,State}. -listen_exit(Pid, State) -> +listen_exit(Pid, Reason, State) -> case lists:keymember(Pid, ?LISTEN_ID, State#state.listen) of true -> + verbose({listen_exit, Pid, Reason}, 1, State), error_msg("** Netkernel terminating ... **\n", []), throw({stop,no_network,State}); false -> false end. -accept_exit(Pid, State) -> +accept_exit(Pid, Reason, State) -> Listen = State#state.listen, case lists:keysearch(Pid, ?ACCEPT_ID, Listen) of {value, ListenR} -> ListenS = ListenR#listen.listen, Mod = ListenR#listen.module, + verbose({accept_exit, Pid, Reason, Mod}, 1, State), AcceptPid = Mod:accept(ListenS), L = lists:keyreplace(Pid, ?ACCEPT_ID, Listen, ListenR#listen{accept = AcceptPid}), @@ -1306,16 +1309,20 @@ accept_exit(Pid, State) -> conn_own_exit(Pid, Reason, #state{conn_owners = Owners} = State) -> case maps:get(Pid, Owners, undefined) of undefined -> false; - Node -> throw({noreply, nodedown(Pid, Node, Reason, State)}) + Node -> + verbose({conn_own_exit, Pid, Reason, Node}, 1, State), + throw({noreply, nodedown(Pid, Node, Reason, State)}) end. dist_ctrlr_exit(Pid, Reason, #state{dist_ctrlrs = DCs} = State) -> case maps:get(Pid, DCs, undefined) of undefined -> false; - Node -> throw({noreply, nodedown(Pid, Node, Reason, State)}) + Node -> + verbose({dist_ctrlr_exit, Pid, Reason, Node}, 1, State), + throw({noreply, nodedown(Pid, Node, Reason, State)}) end. -pending_own_exit(Pid, #state{pend_owners = Pend} = State) -> +pending_own_exit(Pid, Reason, #state{pend_owners = Pend} = State) -> case maps:get(Pid, Pend, undefined) of undefined -> false; @@ -1323,31 +1330,43 @@ pending_own_exit(Pid, #state{pend_owners = Pend} = State) -> State1 = State#state { pend_owners = maps:remove(Pid, Pend)}, case get_conn(Node) of {ok, Conn} when Conn#connection.state =:= up_pending -> + verbose( + {pending_own_exit, Pid, Reason, Node, up_pending}, + 1, State), reply_waiting(Node,Conn#connection.waiting, true), Conn1 = Conn#connection { state = up, waiting = [], pending_owner = undefined }, ets:insert(sys_dist, Conn1); _ -> + verbose({pending_own_exit, Pid, Reason, Node}, 1, State), ok end, throw({noreply, State1}) end. -ticker_exit(Pid, #state{tick = #tick{ticker = Pid, time = T} = Tck} = State) -> +ticker_exit( + Pid, Reason, + #state{tick = #tick{ticker = Pid, time = T} = Tck} = State) -> + verbose({ticker_exit, Pid, Reason, Tck}, 1, State), Tckr = restart_ticker(T), throw({noreply, State#state{tick = Tck#tick{ticker = Tckr}}}); -ticker_exit(Pid, #state{tick = #tick_change{ticker = Pid, - time = T} = TckCng} = State) -> +ticker_exit( + Pid, Reason, + #state{tick = #tick_change{ticker = Pid, time = T} = TckCng} = State) -> + verbose({ticker_exit, Pid, Reason, TckCng}, 1, State), Tckr = restart_ticker(T), - throw({noreply, State#state{tick = TckCng#tick_change{ticker = Tckr}}}); -ticker_exit(_, _) -> + throw({noreply, Reason, State#state{tick = TckCng#tick_change{ticker = Tckr}}}); +ticker_exit(_, _, _) -> false. -restarter_exit(Pid, State) -> +restarter_exit(Pid, Reason, State) -> case State#state.supervisor of {restart, Pid} -> - error_msg("** Distribution restart failed, net_kernel terminating... **\n", []), + verbose({restarter_exit, Pid, Reason}, 1, State), + error_msg( + "** Distribution restart failed, net_kernel terminating... **\n", + []), throw({stop, restarter_exit, State}); _ -> false @@ -2124,7 +2143,7 @@ register_error(false, Proto, Reason) -> proto_error(false, Proto, lists:flatten(S)); register_error(true, Proto, Reason) -> S = "Protocol '" ++ Proto ++ "': register/listen error: ", - erlang:display_string(S), + erlang:display_string(stdout, S), erlang:display(Reason). proto_error(CleanHalt, Proto, String) -> diff --git a/lib/kernel/src/pg.erl b/lib/kernel/src/pg.erl index 017b1942b4..7448dab307 100644 --- a/lib/kernel/src/pg.erl +++ b/lib/kernel/src/pg.erl @@ -203,12 +203,7 @@ get_members(Group) -> -spec get_members(Scope :: atom(), Group :: group()) -> [pid()]. get_members(Scope, Group) -> - try - ets:lookup_element(Scope, Group, 2) - catch - error:badarg -> - [] - end. + ets:lookup_element(Scope, Group, 2, []). %%-------------------------------------------------------------------- %% @doc @@ -219,12 +214,7 @@ get_local_members(Group) -> -spec get_local_members(Scope :: atom(), Group :: group()) -> [pid()]. get_local_members(Scope, Group) -> - try - ets:lookup_element(Scope, Group, 3) - catch - error:badarg -> - [] - end. + ets:lookup_element(Scope, Group, 3, []). %%-------------------------------------------------------------------- %% @doc @@ -492,7 +482,7 @@ sync_groups(Scope, ScopeMon, MG, RemoteGroups, [{Group, Pids} | Tail]) -> {Pids, NewRemoteGroups} -> sync_groups(Scope, ScopeMon, MG, NewRemoteGroups, Tail); {OldPids, NewRemoteGroups} -> - [{Group, AllOldPids, LocalPids}] = ets:lookup(Scope, Group), + [{_Group, AllOldPids, LocalPids}] = ets:lookup(Scope, Group), %% should be really rare... AllNewPids = Pids ++ AllOldPids -- OldPids, true = ets:insert(Scope, {Group, AllNewPids, LocalPids}), @@ -517,7 +507,7 @@ join_local([Pid | Tail], Group, Local) -> join_local_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, [Pid | All], [Pid | Local]}); [] -> ets:insert(Scope, {Group, [Pid], [Pid]}) @@ -525,7 +515,7 @@ join_local_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> notify_group(ScopeMon, MG, join, Group, [Pid]); join_local_update_ets(Scope, ScopeMon, MG, Group, Pids) -> case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, Pids ++ All, Pids ++ Local}); [] -> ets:insert(Scope, {Group, Pids, Pids}) @@ -534,7 +524,7 @@ join_local_update_ets(Scope, ScopeMon, MG, Group, Pids) -> join_remote_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, [Pid | All], Local}); [] -> ets:insert(Scope, {Group, [Pid], []}) @@ -542,7 +532,7 @@ join_remote_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> notify_group(ScopeMon, MG, join, Group, [Pid]); join_remote_update_ets(Scope, ScopeMon, MG, Group, Pids) -> case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, Pids ++ All, Local}); [] -> ets:insert(Scope, {Group, Pids, []}) @@ -576,10 +566,10 @@ leave_local([Pid | Tail], Group, Local) -> leave_local_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> case ets:lookup(Scope, Group) of - [{Group, [Pid], [Pid]}] -> + [{_Group, [Pid], [Pid]}] -> ets:delete(Scope, Group), notify_group(ScopeMon, MG, leave, Group, [Pid]); - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, lists:delete(Pid, All), lists:delete(Pid, Local)}), notify_group(ScopeMon, MG, leave, Group, [Pid]); [] -> @@ -588,7 +578,7 @@ leave_local_update_ets(Scope, ScopeMon, MG, Group, Pid) when is_pid(Pid) -> end; leave_local_update_ets(Scope, ScopeMon, MG, Group, Pids) -> case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> case All -- Pids of [] -> ets:delete(Scope, Group); @@ -603,10 +593,10 @@ leave_local_update_ets(Scope, ScopeMon, MG, Group, Pids) -> leave_remote_update_ets(Scope, ScopeMon, MG, Pid, Groups) when is_pid(Pid) -> _ = [ case ets:lookup(Scope, Group) of - [{Group, [Pid], []}] -> + [{_Group, [Pid], []}] -> ets:delete(Scope, Group), notify_group(ScopeMon, MG, leave, Group, [Pid]); - [{Group, All, Local}] -> + [{_Group, All, Local}] -> ets:insert(Scope, {Group, lists:delete(Pid, All), Local}), notify_group(ScopeMon, MG, leave, Group, [Pid]); [] -> @@ -616,7 +606,7 @@ leave_remote_update_ets(Scope, ScopeMon, MG, Pid, Groups) when is_pid(Pid) -> leave_remote_update_ets(Scope, ScopeMon, MG, Pids, Groups) -> _ = [ case ets:lookup(Scope, Group) of - [{Group, All, Local}] -> + [{_Group, All, Local}] -> case All -- Pids of [] when Local =:= [] -> ets:delete(Scope, Group); diff --git a/lib/kernel/src/prim_tty.erl b/lib/kernel/src/prim_tty.erl new file mode 100644 index 0000000000..e562eb5d64 --- /dev/null +++ b/lib/kernel/src/prim_tty.erl @@ -0,0 +1,983 @@ +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(prim_tty). + +%% Todo: +%% * Try to move buffer handling logic to Erlang +%% * This may not be possible for performance reasons, but should be tried +%% * It seems like unix decodes and then encodes utf8 when emitting it +%% * user_drv module should be able to handle both nif and driver without too many changes. +%% +%% Problems to solve: +%% * Do not use non blocking io +%% * Reset tty settings at _exit +%% * Allow remsh in oldshell (can we do this?) +%% * See if we can run a tty in windows shell +%% * Allow unicode detection for noshell/noinput +%% * ?Allow multi-line editing? +%% * The current implementation only allows the cursor to move and edit on current line +%% +%% Concepts to keep in mind: +%% Code point: A single unicode "thing", examples: "a", "😀" (unicode smilie) +%% Grapheme cluster: One or more code points, " +%% Logical character: Any character that the user typed or printed. +%% One unicode grapheme cluster is a logical character +%% Examples: "a", "\t", "😀" (unicode smilie), "\x{1F600}", "\e[0m" (ansi sequences), +%% "^C" +%% When we step or delete we count logical characters even if they are multiple chars. +%% (I'm unsure how ansi should be handled with regard to delete?) +%% +%% Actual characters: The actual unicode grapheme clusters printed +%% Column: The number of rendered columns for a logical character +%% +%% When navigating using move(left) and move(right) the terminal will move one +%% actual character, so if we want to move one logical character we may have to +%% emit more moves. The same is true when overwriting. +%% +%% When calculating the current column position we have to use the column size +%% of the characters as otherwise smilies will becomes incorrect. +%% +%% In the current ttysl_drv and also this implementation there are never any newlines +%% in the buffer. +%% +%% Printing of unicode characters: +%% Read this post: https://jeffquast.com/post/terminal_wcwidth_solution/ +%% Summary: the wcwidth implementation in libc is often lacking. So we should +%% create our own. We can get the size of all unicode graphemes by rendering +%% them on a terminal and see how much the cursor moves. We can query where the +%% cursor is by using "\033[6n"[1]. How many valid grapheme clusters are there? +%% +%% On consoles that does support fetching the surrent cursor position, we may get +%% away with only using that to check where we are and where to go. And on consoles +%% that do not we just have to make a best effort and use libc wcwidth. +%% +%% We need to know the width of characters when: +%% * Printing a \t +%% * Compensating for xn +%% [1]: https://www.linuxquestions.org/questions/programming-9/get-cursor-position-in-c-947833/ +%% Notes: +%% - [129306,127996] (hand with skintone) seems to move the cursor more than +%% it should on iTerm...The skintone code point seems to not +%% work at all on Linux and instead emits hand + brown square which is 4 characters +%% wide which is what the cursor on iTerm was positioned at. So maybe +%% there is a mismatch somewhere in iTerm wcwidth handling. +%% - edlin and user needs to agree on what one "character" is, right now edlin uses +%% code points and not grapheme clusters. +%% - We can deal with xn by always emitting it after a put_chars command. We must make +%% sure not to emit it before we emit any newline in the put_chars though as that +%% potentially will insert two newlines instead of one. +%% +%% Windows: +%% Since 2017:ish we can use Virtual Terminal Sequences[2] to control the terminal. +%% It seems like these are mostly ANSI escape comparitible, so it should be possible +%% to just use the same mechanism on windows and unix. +%% [2]: https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +%% Windows does not seem to have any wcwidth, so maybe we have to generate a similar table +%% as the PR in [3] and add string:width/1. +%% [3]: https://github.com/microsoft/terminal/pull/5795 +%% +%% +%% Things I've tried and discarded: +%% * Use get position to figure out xn fix for newlines +%% * There is too large a latency (about 10ms) to get the position, so things like +%% `c:i()` becomes a lot slower. +%% * Use tty insert mode +%% * This only works when the cursor is on the last line, when it is on a previous +%% line it only edit that line. +%% * Use tty delete mode +%% * Same problem as insert mode, it only deleted current line, and does not move +%% to previous line automatically. + +-export([init/1, reinit/2, isatty/1, handles/1, unicode/1, unicode/2, + handle_signal/2, window_size/1, handle_request/2, write/2, write/3, npwcwidth/1, + npwcwidthstring/1]). +-export([reader_stop/1, disable_reader/1, enable_reader/1]). + +-nifs([isatty/1, tty_create/0, tty_init/3, tty_set/1, setlocale/1, + tty_select/3, tty_window_size/1, write_nif/2, read_nif/2, isprint/1, + wcwidth/1, wcswidth/1, + sizeof_wchar/0, tgetent_nif/1, tgetnum_nif/1, tgetflag_nif/1, tgetstr_nif/1, + tgoto_nif/2, tgoto_nif/3, tty_read_signal/2]). + +%% Exported in order to remove "unused function" warning +-export([sizeof_wchar/0, wcswidth/1, tgoto/2, tgoto/3]). + +%% proc_lib exports +-export([reader/1, writer/1]). + +-on_load(on_load/0). + +%%-define(debug, true). +-ifdef(debug). +-define(dbg(Term), dbg(Term)). +-else. +-define(dbg(Term), ok). +-endif. +%% Copied from https://github.com/chalk/ansi-regex/blob/main/index.js +-define(ANSI_REGEXP, <<"^[\e",194,155,"][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?",7,")|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-nq-uy=><~]))">>). +-record(state, {tty, + reader, + writer, + options, + unicode, + buffer_before = [], %% Current line before cursor in reverse + buffer_after = [], %% Current line after cursor not in reverse + buffer_expand, %% Characters in expand buffer + cols = 80, + rows = 24, + xn = false, + clear = <<"\e[H\e[2J">>, + up = <<"\e[A">>, + down = <<"\n">>, + left = <<"\b">>, + right = <<"\e[C">>, + %% Tab to next 8 column windows is "\e[1I", for unix "ta" termcap + tab = <<"\e[1I">>, + delete_after_cursor = <<"\e[J">>, + insert = false, + delete = false, + position = <<"\e[6n">>, %% "u7" on my Linux + position_reply = <<"\e\\[([0-9]+);([0-9]+)R">>, + ansi_regexp = ?ANSI_REGEXP, + %% The SGR (Select Graphic Rendition) parameters https://en.wikipedia.org/wiki/ANSI_escape_code#SGR + ansi_sgr = <<"^[\e",194,155,"]\\[[0-9;:]*m">> + }). + +-type options() :: #{ tty => boolean(), + input => boolean(), + canon => boolean(), + echo => boolean(), + sig => boolean() + }. +-type request() :: + {putc, unicode:unicode_binary()} | + {expand, unicode:unicode_binary()} | + {insert, unicode:unicode_binary()} | + {delete, integer()} | + {move, integer()} | + clear | + beep. +-opaque state() :: #state{}. +-export_type([state/0]). + +-spec on_load() -> ok. +on_load() -> + on_load(#{}). + +-spec on_load(Extra) -> ok when + Extra :: map(). +on_load(Extra) -> + case erlang:load_nif(atom_to_list(?MODULE), Extra) of + ok -> ok; + {error,{reload,_}} -> + ok + end. + +-spec window_size(state()) -> {ok, {non_neg_integer(), non_neg_integer()}} | {error, term()}. +window_size(State = #state{ tty = TTY }) -> + case tty_window_size(TTY) of + {error, enotsup} when map_get(tty, State#state.options) -> + %% When the TTY is enabled, we should return a "dummy" row and column + %% when we cannot find the proper size. + {ok, {State#state.cols, State#state.rows}}; + WinSz -> + WinSz + end. + +-spec init(options()) -> state(). +init(UserOptions) when is_map(UserOptions) -> + + Options = options(UserOptions), + {ok, TTY} = tty_create(), + + %% Initialize the locale to see if we support utf-8 or not + UnicodeMode = + case setlocale(TTY) of + primitive -> + lists:any( + fun(Key) -> + string:find(os:getenv(Key,""),"UTF-8") =/= nomatch + end, ["LC_ALL", "LC_CTYPE", "LANG"]); + UnicodeLocale when is_boolean(UnicodeLocale) -> + UnicodeLocale + end, + + init_term(#state{ tty = TTY, unicode = UnicodeMode, options = Options }). +init_term(State = #state{ tty = TTY, options = Options }) -> + TTYState = + case maps:get(tty, Options) of + true -> + ok = tty_init(TTY, stdout, Options), + NewState = init(State, os:type()), + ok = tty_set(TTY), + NewState; + false -> + State + end, + + WriterState = + if TTYState#state.writer =:= undefined -> + {ok, Writer} = proc_lib:start_link(?MODULE, writer, [State#state.tty]), + TTYState#state{ writer = Writer }; + true -> + TTYState + end, + ReaderState = + case {maps:get(input, Options), TTYState#state.reader} of + {true, undefined} -> + {ok, Reader} = proc_lib:start_link(?MODULE, reader, [[State#state.tty, self()]]), + WriterState#state{ reader = Reader }; + {true, _} -> + WriterState; + {false, undefined} -> + WriterState + end, + + update_geometry(ReaderState). + +-spec reinit(state(), options()) -> state(). +reinit(State, UserOptions) -> + init_term(State#state{ options = options(UserOptions) }). + +options(UserOptions) -> + maps:merge( + #{ input => true, + tty => true, + canon => false, + echo => false }, UserOptions). + +init(State, {unix,_}) -> + + case os:getenv("TERM") of + false -> + error(enotsup); + Term -> + case tgetent(Term) of + ok -> ok; + {error,_} -> error(enotsup) + end + end, + + %% See https://www.gnu.org/software/termutils/manual/termcap-1.3/html_mono/termcap.html#SEC23 + %% for a list of all possible termcap capabilities + Clear = case tgetstr("clear") of + {ok, C} -> C; + false -> (#state{})#state.clear + end, + Cols = case tgetnum("co") of + {ok, Cs} -> Cs; + _ -> (#state{})#state.cols + end, + Up = case tgetstr("up") of + {ok, U} -> U; + false -> error(enotsup) + end, + Down = case tgetstr("do") of + false -> (#state{})#state.down; + {ok, D} -> D + end, + Left = case {tgetflag("bs"),tgetstr("bc")} of + {true,_} -> (#state{})#state.left; + {_,false} -> (#state{})#state.left; + {_,{ok, L}} -> L + end, + + Right = case tgetstr("nd") of + {ok, R} -> R; + false -> error(enotsup) + end, + Insert = + case tgetstr("IC") of + {ok, IC} -> IC; + false -> (#state{})#state.insert + end, + + Tab = case tgetstr("ta") of + {ok, TA} -> TA; + false -> (#state{})#state.tab + end, + + Delete = case tgetstr("DC") of + {ok, DC} -> DC; + false -> (#state{})#state.delete + end, + + Position = case tgetstr("u7") of + {ok, <<"\e[6n">> = U7} -> + %% User 7 should contain the codes for getting + %% cursor position. + % User 6 should contain how to parse the reply + {ok, <<"\e[%i%d;%dR">>} = tgetstr("u6"), + <<"\e[6n">> = U7; + false -> (#state{})#state.position + end, + + %% According to the manual this should only be issued when the cursor + %% is at position 0, but until we encounter such a console we keep things + %% simple and issue this with the cursor anywhere + DeleteAfter = case tgetstr("cd") of + {ok, DA} -> + DA; + false -> + (#state{})#state.delete_after_cursor + end, + + State#state{ + cols = Cols, + clear = Clear, + xn = tgetflag("xn"), + up = Up, + down = Down, + left = Left, + right = Right, + insert = Insert, + delete = Delete, + tab = Tab, + position = Position, + delete_after_cursor = DeleteAfter + }; +init(State, {win32, _}) -> + State#state{ + %% position = false, + xn = true }. + +-spec handles(state()) -> #{ read := undefined | reference(), + write := reference() }. +handles(#state{ reader = undefined, + writer = {_WriterPid, WriterRef}}) -> + #{ read => undefined, write => WriterRef }; +handles(#state{ reader = {_ReaderPid, ReaderRef}, + writer = {_WriterPid, WriterRef}}) -> + #{ read => ReaderRef, write => WriterRef }. + +-spec unicode(state()) -> boolean(). +unicode(State) -> + State#state.unicode. + +-spec unicode(state(), boolean()) -> state(). +unicode(#state{ reader = Reader } = State, Bool) -> + case Reader of + {ReaderPid, _} -> + call(ReaderPid, {set_unicode_state, Bool}); + undefined -> + ok + end, + State#state{ unicode = Bool }. + +-spec reader_stop(state()) -> state(). +reader_stop(#state{ reader = {ReaderPid, _} } = State) -> + {error, _} = call(ReaderPid, stop), + State#state{ reader = undefined }. + +-spec handle_signal(state(), winch | cont) -> state(). +handle_signal(State, winch) -> + update_geometry(State); +handle_signal(State, cont) -> + tty_set(State#state.tty), + State. + +-spec disable_reader(state()) -> ok. +disable_reader(#state{ reader = {ReaderPid, _} }) -> + ok = call(ReaderPid, disable). + +-spec enable_reader(state()) -> ok. +enable_reader(#state{ reader = {ReaderPid, _} }) -> + ok = call(ReaderPid, enable). + +call(Pid, Msg) -> + Alias = erlang:monitor(process, Pid, [{alias, reply_demonitor}]), + Pid ! {Alias, Msg}, + receive + {Alias, Reply} -> + Reply; + {'DOWN',Alias,_,_,Reason} -> + {error, Reason} + end. + +reader([TTY, Parent]) -> + register(user_drv_reader, self()), + ReaderRef = make_ref(), + SignalRef = make_ref(), + ok = tty_select(TTY, SignalRef, ReaderRef), + proc_lib:init_ack({ok, {self(), ReaderRef}}), + FromEnc = case os:type() of + {unix, _} -> utf8; + {win32, _} -> + case isatty(stdin) of + true -> + {utf16, little}; + _ -> + %% When not reading from a console + %% the data read is utf8 encoded + utf8 + end + end, + reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, <<>>). + +reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, Acc) -> + receive + {DisableAlias, disable} -> + DisableAlias ! {DisableAlias, ok}, + receive + {EnableAlias, enable} -> + EnableAlias ! {EnableAlias, ok}, + reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, Acc) + end; + {select, TTY, SignalRef, ready_input} -> + {ok, Signal} = tty_read_signal(TTY, SignalRef), + Parent ! {ReaderRef,{signal,Signal}}, + reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, Acc); + {Alias, {set_unicode_state, _}} when FromEnc =:= {utf16, little} -> + %% Ignore requests on windows + Alias ! {Alias, true}, + reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, Acc); + {Alias, {set_unicode_state, Bool}} -> + Alias ! {Alias, FromEnc =/= latin1}, + NewFromEnc = if Bool -> utf8; not Bool -> latin1 end, + reader_loop(TTY, Parent, SignalRef, ReaderRef, NewFromEnc, Acc); + {_Alias, stop} -> + ok; + {select, TTY, ReaderRef, ready_input} -> + case read_nif(TTY, ReaderRef) of + {error, closed} -> + Parent ! {ReaderRef, eof}, + ok; + {ok, <<>>} -> + %% EAGAIN or EINTR + reader_loop(TTY, Parent, SignalRef, ReaderRef, FromEnc, Acc); + {ok, UtfXBytes} -> + + {Bytes, NewAcc, NewFromEnc} = + case unicode:characters_to_binary([Acc, UtfXBytes], FromEnc, utf8) of + {error, B, Error} -> + %% We should only be able to get incorrect encoded data when + %% using utf8 (i.e. we are on unix) + FromEnc = utf8, + Parent ! {self(), set_unicode_state, false}, + receive + {Alias, {set_unicode_state, false}} -> + Alias ! {Alias, true} + end, + receive + {Parent, set_unicode_state, true} -> ok + end, + Latin1Chars = unicode:characters_to_binary(Error, latin1, utf8), + {<<B/binary,Latin1Chars/binary>>, <<>>, latin1}; + {incomplete, B, Inc} -> + {B, Inc, FromEnc}; + B when is_binary(B) -> + {B, <<>>, FromEnc} + end, + Parent ! {ReaderRef, {data, Bytes}}, + reader_loop(TTY, Parent, SignalRef, ReaderRef, NewFromEnc, NewAcc) + end + end. + +writer(TTY) -> + register(user_drv_writer, self()), + WriterRef = make_ref(), + proc_lib:init_ack({ok, {self(), WriterRef}}), + writer_loop(TTY, WriterRef). + +-spec write(state(), unicode:chardata()) -> ok. +write(#state{ writer = {WriterPid, _}}, Chars) -> + WriterPid ! {write, erlang:iolist_to_iovec(Chars)}, ok. +-spec write(state(), unicode:chardata(), From :: pid()) -> {ok, reference()}. +write(#state{ writer = {WriterPid, _WriterRef}}, Chars, From) -> + Ref = erlang:monitor(process, WriterPid), + WriterPid ! {write, From, erlang:iolist_to_iovec(Chars)}, + {ok, Ref}. + +writer_loop(TTY, WriterRef) -> + receive + {write, []} -> + writer_loop(TTY, WriterRef); + {write, Chars} -> + _ = write_nif(TTY, Chars), + writer_loop(TTY, WriterRef); + {write, From, []} -> + From ! {WriterRef, ok}, + writer_loop(TTY, WriterRef); + {write, From, Chars} -> + case write_nif(TTY, Chars) of + ok -> + From ! {WriterRef, ok}, + writer_loop(TTY, WriterRef); + {error, Reason} -> + exit(self(), Reason) + end + end. + +-spec handle_request(state(), request()) -> {erlang:iovec(), state()}. +handle_request(State = #state{ options = #{ tty := false } }, Request) -> + case Request of + {putc, Binary} -> + {encode(Binary, State#state.unicode), State}; + beep -> + {<<7>>, State}; + _Ignore -> + {<<>>, State} + end; +%% Clear the expand buffer after the cursor when we handle any request. +handle_request(State = #state{ buffer_expand = Expand, unicode = U }, Request) + when Expand =/= undefined -> + BBCols = cols(State#state.buffer_before, U), + BACols = cols(State#state.buffer_after, U), + ClearExpand = [move_cursor(State, BBCols, BBCols + BACols), + State#state.delete_after_cursor, + move_cursor(State, BBCols + BACols, BBCols)], + {Output, NewState} = handle_request(State#state{ buffer_expand = undefined }, Request), + {[ClearExpand, encode(Output, U)], NewState}; +%% Print characters in the expandbuffer after the cursor +handle_request(State = #state{ unicode = U }, {expand, Binary}) -> + BBCols = cols(State#state.buffer_before, U), + BACols = cols(State#state.buffer_after, U), + Expand = iolist_to_binary(["\r\n",string:trim(Binary, both)]), + MoveToEnd = move_cursor(State, BBCols, BBCols + BACols), + {ExpandBuffer, NewState} = insert_buf(State#state{ buffer_expand = [] }, Expand), + BECols = cols(NewState#state.cols, BBCols + BACols, NewState#state.buffer_expand, U), + MoveToOrig = move_cursor(State, BECols, BBCols), + {[MoveToEnd, encode(ExpandBuffer, U), MoveToOrig], NewState}; +%% putc prints Binary and overwrites any existing characters +handle_request(State = #state{ unicode = U }, {putc, Binary}) -> + %% Todo should handle invalid unicode? + {PutBuffer, NewState} = insert_buf(State, Binary), + if NewState#state.buffer_after =:= [] -> + {encode(PutBuffer, U), NewState}; + true -> + %% Delete any overwritten characters after current the cursor + OldLength = logical(State#state.buffer_before), + NewLength = logical(NewState#state.buffer_before), + {_, _, _, NewBA} = split(NewLength - OldLength, NewState#state.buffer_after, U), + {encode(PutBuffer, U), NewState#state{ buffer_after = NewBA }} + end; +handle_request(State = #state{ unicode = U }, {delete, N}) when N > 0 -> + {_DelNum, DelCols, _, NewBA} = split(N, State#state.buffer_after, U), + BBCols = cols(State#state.buffer_before, U), + NewBACols = cols(NewBA, U), + {[encode(NewBA, U), + lists:duplicate(DelCols, $\s), + xnfix(State, BBCols + NewBACols + DelCols), + move_cursor(State, + BBCols + NewBACols + DelCols, + BBCols)], + State#state{ buffer_after = NewBA }}; +handle_request(State = #state{ unicode = U }, {delete, N}) when N < 0 -> + {_DelNum, DelCols, _, NewBB} = split(-N, State#state.buffer_before, U), + NewBBCols = cols(NewBB, U), + BACols = cols(State#state.buffer_after, U), + {[move_cursor(State, NewBBCols + DelCols, NewBBCols), + encode(State#state.buffer_after,U), + lists:duplicate(DelCols, $\s), + xnfix(State, NewBBCols + BACols + DelCols), + move_cursor(State, NewBBCols + BACols + DelCols, NewBBCols)], + State#state{ buffer_before = NewBB } }; +handle_request(State, {delete, 0}) -> + {"",State}; +handle_request(State = #state{ unicode = U }, {move, N}) when N < 0 -> + {_DelNum, DelCols, NewBA, NewBB} = split(-N, State#state.buffer_before, U), + NewBBCols = cols(NewBB, U), + Moves = move_cursor(State, NewBBCols + DelCols, NewBBCols), + {Moves, State#state{ buffer_before = NewBB, + buffer_after = NewBA ++ State#state.buffer_after} }; +handle_request(State = #state{ unicode = U }, {move, N}) when N > 0 -> + {_DelNum, DelCols, NewBB, NewBA} = split(N, State#state.buffer_after, U), + BBCols = cols(State#state.buffer_before, U), + {move_cursor(State, BBCols, BBCols + DelCols), + State#state{ buffer_after = NewBA, + buffer_before = NewBB ++ State#state.buffer_before} }; +handle_request(State, {move, 0}) -> + {"",State}; +handle_request(State = #state{ xn = OrigXn, unicode = U }, {insert, Chars}) -> + {InsertBuffer, NewState0} = insert_buf(State#state{ xn = false }, Chars), + NewState = NewState0#state{ xn = OrigXn }, + BBCols = cols(NewState#state.buffer_before, U), + BACols = cols(NewState#state.buffer_after, U), + {[ encode(InsertBuffer, U), + encode(NewState#state.buffer_after, U), + xnfix(State, BBCols + BACols), + move_cursor(State, BBCols + BACols, BBCols) ], + NewState}; +handle_request(State, beep) -> + {<<7>>, State}; +handle_request(State, clear) -> + {State#state.clear, State}; +handle_request(State, Req) -> + erlang:display({unhandled_request, Req}), + {"", State}. + +%% Split the buffer after N logical characters returning +%% the number of real characters deleted and the column length +%% of those characters +split(N, Buff, Unicode) -> + ?dbg({?FUNCTION_NAME, N, Buff, Unicode}), + split(N, Buff, [], 0, 0, Unicode). +split(0, Buff, Acc, Chars, Cols, _Unicode) -> + ?dbg({?FUNCTION_NAME, {Chars, Cols, Acc, Buff}}), + {Chars, Cols, Acc, Buff}; +split(N, _Buff, _Acc, _Chars, _Cols, _Unicode) when N < 0 -> + ok = N; +split(_N, [], Acc, Chars, Cols, _Unicode) -> + {Chars, Cols, Acc, []}; +split(N, [Char | T], Acc, Cnt, Cols, Unicode) when is_integer(Char) -> + split(N - 1, T, [Char | Acc], Cnt + 1, Cols + npwcwidth(Char, Unicode), Unicode); +split(N, [Chars | T], Acc, Cnt, Cols, Unicode) when is_list(Chars) -> + split(N - length(Chars), T, [Chars | Acc], + Cnt + length(Chars), Cols + cols(Chars, Unicode), Unicode); +split(N, [SkipChars | T], Acc, Cnt, Cols, Unicode) when is_binary(SkipChars) -> + split(N, T, [SkipChars | Acc], Cnt, Cols, Unicode). + +logical([]) -> + 0; +logical([Char | T]) when is_integer(Char) -> + 1 + logical(T); +logical([Chars | T]) when is_list(Chars) -> + length(Chars) + logical(T); +logical([SkipChars | T]) when is_binary(SkipChars) -> + logical(T). + +move_cursor(#state{ cols = W } = State, FromCol, ToCol) -> + ?dbg({?FUNCTION_NAME, FromCol, ToCol}), + [case (ToCol div W) - (FromCol div W) of + 0 -> ""; + N when N < 0 -> + ?dbg({move, up, -N}), + move(up, State, -N); + N -> + ?dbg({move, down, N}), + move(down, State, N) + end, + case (ToCol rem W) - (FromCol rem W) of + 0 -> ""; + N when N < 0 -> + ?dbg({down, left, -N}), + move(left, State, -N); + N -> + ?dbg({down, right, N}), + move(right, State, N) + end]. + +move(up, #state{ up = Up }, N) -> + lists:duplicate(N, Up); +move(down, #state{ down = Down }, N) -> + lists:duplicate(N, Down); +move(left, #state{ left = Left }, N) -> + lists:duplicate(N, Left); +move(right, #state{ right = Right }, N) -> + lists:duplicate(N, Right). + +cols([],_Unicode) -> + 0; +cols([Char | T], Unicode) when is_integer(Char) -> + npwcwidth(Char, Unicode) + cols(T, Unicode); +cols([Chars | T], Unicode) when is_list(Chars) -> + cols(Chars, Unicode) + cols(T, Unicode); +cols([SkipSeq | T], Unicode) when is_binary(SkipSeq) -> + %% Any binary should be an ANSI escape sequence + %% so we skip that + cols(T, Unicode). + +cols(ColsPerLine, CurrCols, Chars, Unicode) when CurrCols > ColsPerLine -> + ColsPerLine + cols(ColsPerLine, CurrCols - ColsPerLine, Chars, Unicode); +cols(_ColsPerLine, CurrCols, [], _Unicode) -> + CurrCols; +cols(ColsPerLine, CurrCols, ["\r\n" | T], Unicode) -> + CurrCols + (ColsPerLine - CurrCols) + cols(ColsPerLine, 0, T, Unicode); +cols(ColsPerLine, CurrCols, [H | T], Unicode) -> + cols(ColsPerLine, CurrCols + cols([H], Unicode), T, Unicode). + +update_geometry(State) -> + case tty_window_size(State#state.tty) of + {ok, {Cols, Rows}} when Cols > 0 -> + ?dbg({?FUNCTION_NAME, Cols}), + State#state{ cols = Cols, rows = Rows }; + _Error -> + ?dbg({?FUNCTION_NAME, _Error}), + State + end. + +npwcwidthstring(String) when is_list(String) -> + npwcwidthstring(unicode:characters_to_binary(String)); +npwcwidthstring(String) -> + case string:next_grapheme(String) of + [] -> 0; + [$\e | Rest] -> + case re:run(String, ?ANSI_REGEXP, [unicode]) of + {match, [{0, N}]} -> + <<_Ansi:N/binary, AnsiRest/binary>> = String, + npwcwidthstring(AnsiRest); + _ -> + npwcwidth($\e) + npwcwidthstring(Rest) + end; + [H|Rest] -> npwcwidth(H) + npwcwidthstring(Rest) + end. + +npwcwidth(Char) -> + npwcwidth(Char, true). +npwcwidth(Char, true) -> + case wcwidth(Char) of + {error, not_printable} -> 0; + {error, enotsup} -> + case unicode_util:is_wide(Char) of + true -> 2; + false -> 1 + end; + C -> C + end; +npwcwidth(Char, false) -> + byte_size(char_to_latin1(Char)). + + +%% Return the xn fix for the current cursor position. +%% We use get_position to figure out if we need to calculate the current columns +%% or not. +%% +%% We need to know the actual column because get_position will return the last +%% column number when the cursor is: +%% * in the last column +%% * off screen +%% +%% and it is when the cursor is off screen that we should do the xnfix. +xnfix(#state{ position = _, unicode = U } = State) -> + xnfix(State, cols(State#state.buffer_before, U)). +%% Return the xn fix for CurrCols location. +xnfix(#state{ xn = true, cols = Cols } = State, CurrCols) + when CurrCols =/= 0, CurrCols rem Cols == 0 -> + [<<"\s">>,move(left, State, 1)]; +xnfix(_, _CurrCols) -> + ?dbg({xnfix, _CurrCols}), + []. + +characters_to_output(Chars) -> + try unicode:characters_to_binary(Chars) of + Binary -> + Binary + catch error:badarg -> + unicode:characters_to_binary( + lists:map( + fun({ansi, Ansi}) -> + Ansi; + (Char) -> + Char + end, Chars) + ) + end. +characters_to_buffer(Chars) -> + lists:flatmap( + fun({ansi, _Ansi}) -> + ""; + (Char) -> + [Char] + end, Chars). + +insert_buf(State, Binary) when is_binary(Binary) -> + insert_buf(State, Binary, [], []). +insert_buf(State, Bin, LineAcc, Acc) -> + case string:next_grapheme(Bin) of + [] when State#state.buffer_expand =/= undefined -> + Expand = lists:reverse(LineAcc), + {[Acc, characters_to_output(Expand)], + State#state{ buffer_expand = characters_to_buffer(Expand) }}; + [] -> + NewBB = characters_to_buffer(LineAcc) ++ State#state.buffer_before, + NewState = State#state{ buffer_before = NewBB }, + {[Acc, characters_to_output(lists:reverse(LineAcc)), xnfix(NewState)], + NewState}; + [$\t | Rest] -> + insert_buf(State, Rest, [State#state.tab | LineAcc], Acc); + [$\e | Rest] -> + case re:run(Bin, State#state.ansi_regexp, [unicode]) of + {match, [{0, N}]} -> + <<Ansi:N/binary, AnsiRest/binary>> = Bin, + case re:run(Bin, State#state.ansi_sgr, [unicode]) of + {match, [{0, N}]} -> + %% We include the graphics ansi sequences in the + %% buffer that we step over + insert_buf(State, AnsiRest, [Ansi | LineAcc], Acc); + _ -> + %% Any other ansi sequences are just printed and + %% then dropped as they "should" not effect rendering + insert_buf(State, AnsiRest, [{ansi, Ansi} | LineAcc], Acc) + end; + _ -> + insert_buf(State, Rest, [$\e | LineAcc], Acc) + end; + [NLCR | Rest] when NLCR =:= $\n; NLCR =:= $\r -> + Tail = + if NLCR =:= $\n -> + <<$\r,$\n>>; + true -> + <<$\r>> + end, + if State#state.buffer_expand =:= undefined -> + insert_buf(State#state{ buffer_before = [], buffer_after = [] }, Rest, [], + [Acc, [characters_to_output(lists:reverse(LineAcc)), Tail]]); + true -> + insert_buf(State, Rest, [binary_to_list(Tail) | LineAcc], Acc) + end; + [Cluster | Rest] when is_list(Cluster) -> + insert_buf(State, Rest, [Cluster | LineAcc], Acc); + %% We have gotten a code point that may be part of the previous grapheme cluster. + [Char | Rest] when Char >= 128, LineAcc =:= [], State#state.buffer_before =/= [], + State#state.buffer_expand =:= undefined -> + [PrevChar | BB] = State#state.buffer_before, + case string:next_grapheme([PrevChar | Bin]) of + [PrevChar | _] -> + %% It was not part of the previous cluster, so just insert + %% it as a normal character + insert_buf(State, Rest, [Char | LineAcc], Acc); + [Cluster | ClusterRest] -> + %% It was part of the previous grapheme cluster, so we output + %% it and insert it into the before_buffer + %% TODO: If an xnfix was done on PrevChar, + %% then we should rewrite the entire grapheme cluster. + {_, ToWrite} = lists:split(length(lists:flatten([PrevChar])), Cluster), + insert_buf(State#state{ buffer_before = [Cluster | BB] }, + ClusterRest, LineAcc, + [Acc, unicode:characters_to_binary(ToWrite)]) + end; + [Char | Rest] when Char >= 128 -> + insert_buf(State, Rest, [Char | LineAcc], Acc); + [Char | Rest] -> + case {isprint(Char), Char} of + {true,_} -> + insert_buf(State, Rest, [Char | LineAcc], Acc); + {false, 8#177} -> %% DEL + insert_buf(State, Rest, ["^?" | LineAcc], Acc); + {false, _} -> + insert_buf(State, Rest, ["^" ++ [Char bor 8#40] | LineAcc], Acc) + end + end. + +-spec to_latin1(erlang:binary()) -> erlang:iovec(). +to_latin1(Bin) -> + case is_usascii(Bin) of + true -> [Bin]; + false -> lists:flatten([binary_to_latin1(Bin)]) + end. + +is_usascii(<<Char/utf8,T/binary>>) when Char < 128 -> + is_usascii(T); +is_usascii(<<>>) -> + true; +is_usascii(_) -> + false. + +binary_to_latin1(Buffer) -> + [char_to_latin1(CP) || CP <- unicode:characters_to_list(Buffer)]. +char_to_latin1(UnicodeChar) when UnicodeChar >= 512 -> + <<"\\x{",(integer_to_binary(UnicodeChar, 16))/binary,"}">>; +char_to_latin1(UnicodeChar) when UnicodeChar >= 128 -> + <<"\\",(integer_to_binary(UnicodeChar, 8))/binary>>; +char_to_latin1(UnicodeChar) -> + <<UnicodeChar>>. + +encode(UnicodeChars, true) -> + unicode:characters_to_binary(UnicodeChars); +encode(UnicodeChars, false) -> + to_latin1(unicode:characters_to_binary(UnicodeChars)). + +%% Using get_position adds about 10ms of latency +%% get_position(#state{ position = false }) -> +%% unknown; +%% get_position(State) -> +%% [] = write(State, State#state.position), +%% get_position(State, <<>>). +%% get_position(State, Acc) -> +%% receive +%% {select,TTY,Ref,ready_input} +%% when TTY =:= State#state.tty, +%% Ref =:= State#state.read -> +%% {Bytes, <<>>} = read_input(State#state{ acc = Acc }), +%% case re:run(Bytes, State#state.position_reply, [unicode]) of +%% {match,[{Start,Length},Row,Col]} -> +%% <<Before:Start/binary,_:Length/binary,After/binary>> = Bytes, +%% %% This should be put in State in order to not screw up the +%% %% message order... +%% [State#state.parent ! {{self(), State#state.tty}, {data, <<Before/binary, After/binary>>}} +%% || Before =/= <<>>, After =/= <<>>], +%% {binary_to_integer(binary:part(Bytes,Row)), +%% binary_to_integer(binary:part(Bytes,Col))}; +%% nomatch -> +%% get_position(State, Bytes) +%% end +%% after 1000 -> +%% unknown +%% end. + +-ifdef(debug). +dbg(_) -> + ok. +-endif. + +%% Nif functions +-spec isatty(stdin | stdout | stderr) -> boolean() | ebadf. +isatty(_Fd) -> + erlang:nif_error(undef). +tty_create() -> + erlang:nif_error(undef). +tty_init(_TTY, _Fd, _Options) -> + erlang:nif_error(undef). +tty_set(_TTY) -> + erlang:nif_error(undef). +setlocale(_TTY) -> + erlang:nif_error(undef). +tty_select(_TTY, _SignalRef, _ReadRef) -> + erlang:nif_error(undef). +write_nif(_TTY, _IOVec) -> + erlang:nif_error(undef). +read_nif(_TTY, _Ref) -> + erlang:nif_error(undef). +tty_window_size(_TTY) -> + erlang:nif_error(undef). +isprint(_Char) -> + erlang:nif_error(undef). +wcwidth(_Char) -> + erlang:nif_error(undef). +sizeof_wchar() -> + erlang:nif_error(undef). +wcswidth(_Char) -> + erlang:nif_error(undef). +tgetent(Char) -> + tgetent_nif([Char,0]). +tgetnum(Char) -> + tgetnum_nif([Char,0]). +tgetflag(Char) -> + tgetflag_nif([Char,0]). +tgetstr(Char) -> + tgetstr_nif([Char,0]). +tgoto(Char, Arg) -> + tgoto_nif([Char,0], Arg). +tgoto(Char, Arg1, Arg2) -> + tgoto_nif([Char,0], Arg1, Arg2). +tgetent_nif(_Char) -> + erlang:nif_error(undef). +tgetnum_nif(_Char) -> + erlang:nif_error(undef). +tgetflag_nif(_Char) -> + erlang:nif_error(undef). +tgetstr_nif(_Char) -> + erlang:nif_error(undef). +tgoto_nif(_Ent, _Arg) -> + erlang:nif_error(undef). +tgoto_nif(_Ent, _Arg1, _Arg2) -> + erlang:nif_error(undef). +tty_read_signal(_TTY, _Ref) -> + erlang:nif_error(undef). + diff --git a/lib/kernel/src/standard_error.erl b/lib/kernel/src/standard_error.erl index 1aad064392..58831f0ba7 100644 --- a/lib/kernel/src/standard_error.erl +++ b/lib/kernel/src/standard_error.erl @@ -66,6 +66,7 @@ server(PortName,PortSettings) -> run(P) -> put(encoding, latin1), + put(onlcr, false), server_loop(P). server_loop(Port) -> @@ -161,7 +162,7 @@ io_request({get_geometry,rows},Port) -> io_request(getopts, _Port) -> getopts(); io_request({setopts,Opts}, _Port) when is_list(Opts) -> - setopts(Opts); + do_setopts(Opts); io_request({requests,Reqs}, Port) -> io_requests(Reqs, {ok,ok}, Port); io_request(R, _Port) -> %Unknown request @@ -203,19 +204,27 @@ put_chars(Chars, Port) when is_binary(Chars) -> {ok,ok}. %% setopts -setopts(Opts0) -> +do_setopts(Opts0) -> Opts = expand_encoding(Opts0), case check_valid_opts(Opts) of true -> - do_setopts(Opts); + lists:foreach( + fun({encoding, Enc}) -> + put(encoding, Enc); + ({onlcr, Bool}) -> + put(onlcr, Bool) + end, Opts), + {ok, ok}; false -> {error,{error,enotsup}} end. check_valid_opts([]) -> true; -check_valid_opts([{encoding,Valid}|T]) when Valid =:= unicode; - Valid =:= utf8; Valid =:= latin1 -> +check_valid_opts([{encoding,Valid}|T]) when Valid =:= unicode; Valid =:= utf8; + Valid =:= latin1 -> + check_valid_opts(T); +check_valid_opts([{onlcr,Bool}|T]) when is_boolean(Bool) -> check_valid_opts(T); check_valid_opts(_) -> false. @@ -226,27 +235,21 @@ expand_encoding([latin1 | T]) -> [{encoding,latin1} | expand_encoding(T)]; expand_encoding([unicode | T]) -> [{encoding,unicode} | expand_encoding(T)]; +expand_encoding([utf8 | T]) -> + [{encoding,unicode} | expand_encoding(T)]; +expand_encoding([{encoding,utf8} | T]) -> + [{encoding,unicode} | expand_encoding(T)]; expand_encoding([H|T]) -> [H|expand_encoding(T)]. -do_setopts(Opts) -> - case proplists:get_value(encoding, Opts) of - Valid when Valid =:= unicode; Valid =:= utf8 -> - put(encoding, unicode); - latin1 -> - put(encoding, latin1); - undefined -> - ok - end, - {ok,ok}. - getopts() -> Uni = {encoding,get(encoding)}, - {ok,[Uni]}. + Onlcr = {onlcr, get(onlcr)}, + {ok,[Uni, Onlcr]}. wrap_characters_to_binary(Chars,From,To) -> - TrNl = (whereis(user_drv) =/= undefined), - Limit = case To of + TrNl = get(onlcr), + Limit = case To of latin1 -> 255; _Else -> diff --git a/lib/kernel/src/user.erl b/lib/kernel/src/user.erl deleted file mode 100644 index 67c2eafdbe..0000000000 --- a/lib/kernel/src/user.erl +++ /dev/null @@ -1,769 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2020. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(user). --compile(inline). - -%% Basic standard i/o server for user interface port. - --export([start/0, start/1, start_out/0]). --export([interfaces/1]). - --define(NAME, user). - -%% Defines for control ops --define(ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER, 16#018b0900). --define(CTRL_OP_GET_WINSIZE, (100 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). - -%% -%% The basic server and start-up. -%% - -start() -> - start_port([eof,binary]). - -start([Mod,Fun|Args]) -> - %% Mod,Fun,Args should return a pid. That process is supposed to act - %% as the io port. - Pid = apply(Mod, Fun, Args), % This better work! - Id = spawn(fun() -> server(Pid) end), - register(?NAME, Id), - Id. - -start_out() -> - %% Output-only version of start/0 - start_port([out,binary]). - -start_port(PortSettings) -> - Id = spawn(fun() -> server({fd,0,1}, PortSettings) end), - register(?NAME, Id), - Id. - -%% Return the pid of the shell process. -%% Note: We can't ask the user process for this info since it -%% may be busy waiting for data from the port. -interfaces(User) -> - case process_info(User, dictionary) of - {dictionary,Dict} -> - case lists:keysearch(shell, 1, Dict) of - {value,Sh={shell,Shell}} when is_pid(Shell) -> - [Sh]; - _ -> - [] - end; - _ -> - [] - end. - -server(Pid) when is_pid(Pid) -> - process_flag(trap_exit, true), - link(Pid), - run(Pid). - -server(PortName,PortSettings) -> - process_flag(trap_exit, true), - Port = open_port(PortName,PortSettings), - run(Port). - -run(P) -> - put(read_mode,list), - put(encoding,latin1), - case init:get_argument(noshell) of - %% non-empty list -> noshell - {ok, [_|_]} -> - put(shell, noshell), - server_loop(P, queue:new()); - _ -> - group_leader(self(), self()), - catch_loop(P, start_init_shell()) - end. - -catch_loop(Port, Shell) -> - catch_loop(Port, Shell, queue:new()). - -catch_loop(Port, Shell, Q) -> - case catch server_loop(Port, Q) of - new_shell -> - exit(Shell, kill), - catch_loop(Port, start_new_shell()); - {unknown_exit,{Shell,Reason},_} -> % shell has exited - case Reason of - normal -> - put_port(<<"*** ">>, Port); - _ -> - put_port(<<"*** ERROR: ">>, Port) - end, - put_port(<<"Shell process terminated! ***\n">>, Port), - catch_loop(Port, start_new_shell()); - {unknown_exit,_,Q1} -> - catch_loop(Port, Shell, Q1); - {'EXIT',R} -> - exit(R) - end. - -link_and_save_shell(Shell) -> - link(Shell), - put(shell, Shell), - Shell. - -start_init_shell() -> - link_and_save_shell(shell:start(init)). - -start_new_shell() -> - link_and_save_shell(shell:start()). - -server_loop(Port, Q) -> - receive - {io_request,From,ReplyAs,Request} when is_pid(From) -> - server_loop(Port, do_io_request(Request, From, ReplyAs, Port, Q)); - {Port,{data,Bytes}} -> - case get(shell) of - noshell -> - server_loop(Port, queue:snoc(Q, Bytes)); - _ -> - case contains_ctrl_g_or_ctrl_c(Bytes) of - false -> - server_loop(Port, queue:snoc(Q, Bytes)); - _ -> - throw(new_shell) - end - end; - {Port, eof} -> - put(eof, true), - server_loop(Port, Q); - - %% Ignore messages from port here. - {'EXIT',Port,badsig} -> % Ignore badsig errors - server_loop(Port, Q); - {'EXIT',Port,What} -> % Port has exited - exit(What); - - %% Check if shell has exited - {'EXIT',SomePid,What} -> - case get(shell) of - noshell -> - server_loop(Port, Q); % Ignore - _ -> - throw({unknown_exit,{SomePid,What},Q}) - end; - - _Other -> % Ignore other messages - server_loop(Port, Q) - end. - - -get_fd_geometry(Port) -> - case (catch port_control(Port,?CTRL_OP_GET_WINSIZE,[])) of - List when length(List) =:= 8 -> - <<W:32/native,H:32/native>> = list_to_binary(List), - {W,H}; - _ -> - error - end. - - -%% NewSaveBuffer = io_request(Request, FromPid, ReplyAs, Port, SaveBuffer) - -do_io_request(Req, From, ReplyAs, Port, Q0) -> - case io_request(Req, Port, Q0) of - {_Status,Reply,Q1} -> - _ = io_reply(From, ReplyAs, Reply), - Q1; - {exit,What} -> - ok = send_port(Port, close), - exit(What) - end. - -%% New in R13B -%% Encoding option (unicode/latin1) -io_request({put_chars,unicode,Chars}, Port, Q) -> % Binary new in R9C - case wrap_characters_to_binary(Chars, unicode, get(encoding)) of - error -> - {error,{error,put_chars},Q}; - Bin -> - put_chars(Bin, Port, Q) - end; -io_request({put_chars,unicode,Mod,Func,Args}, Port, Q) -> - case catch apply(Mod,Func,Args) of - Data when is_list(Data); is_binary(Data) -> - case wrap_characters_to_binary(Data, unicode, get(encoding)) of - Bin when is_binary(Bin) -> - put_chars(Bin, Port, Q); - error -> - {error,{error,put_chars},Q} - end; - Undef -> - put_chars(Undef, Port, Q) - end; -io_request({put_chars,latin1,Chars}, Port, Q) -> % Binary new in R9C - case catch unicode:characters_to_binary(Chars, latin1, get(encoding)) of - Data when is_binary(Data) -> - put_chars(Data, Port, Q); - _ -> - {error,{error,put_chars},Q} - end; -io_request({put_chars,latin1,Mod,Func,Args}, Port, Q) -> - case catch apply(Mod,Func,Args) of - Data when is_list(Data); is_binary(Data) -> - case - catch unicode:characters_to_binary(Data,latin1,get(encoding)) - of - Bin when is_binary(Bin) -> - put_chars(Bin, Port, Q); - _ -> - {error,{error,put_chars},Q} - end; - Undef -> - put_chars(Undef, Port, Q) - end; -io_request({get_chars,Enc,Prompt,N}, Port, Q) -> % New in R9C - get_chars(Prompt, io_lib, collect_chars, N, Port, Q, Enc); -io_request({get_line,Enc,Prompt}, Port, Q) -> - case get(read_mode) of - binary -> - get_line_bin(Prompt,Port,Q,Enc); - _ -> - get_chars(Prompt, io_lib, collect_line, [], Port, Q, Enc) - end; -io_request({get_until,Enc,Prompt,M,F,As}, Port, Q) -> - get_chars(Prompt, io_lib, get_until, {M,F,As}, Port, Q, Enc); -%% End New in R13B -io_request(getopts, Port, Q) -> - getopts(Port, Q); -io_request({setopts,Opts}, Port, Q) when is_list(Opts) -> - setopts(Opts, Port, Q); -io_request({requests,Reqs}, Port, Q) -> - io_requests(Reqs, {ok,ok,Q}, Port); - -%% New in R12 -io_request({get_geometry,columns},Port,Q) -> - case get_fd_geometry(Port) of - {W,_H} -> - {ok,W,Q}; - _ -> - {error,{error,enotsup},Q} - end; -io_request({get_geometry,rows},Port,Q) -> - case get_fd_geometry(Port) of - {_W,H} -> - {ok,H,Q}; - _ -> - {error,{error,enotsup},Q} - end; -%% BC with pre-R13 nodes -io_request({put_chars,Chars}, Port, Q) -> - io_request({put_chars,latin1,Chars}, Port, Q); -io_request({put_chars,Mod,Func,Args}, Port, Q) -> - io_request({put_chars,latin1,Mod,Func,Args}, Port, Q); -io_request({get_chars,Prompt,N}, Port, Q) -> - io_request({get_chars,latin1,Prompt,N}, Port, Q); -io_request({get_line,Prompt}, Port, Q) -> - io_request({get_line,latin1,Prompt}, Port, Q); -io_request({get_until,Prompt,M,F,As}, Port, Q) -> - io_request({get_until,latin1,Prompt,M,F,As}, Port, Q); - -io_request(R, _Port, Q) -> %Unknown request - {error,{error,{request,R}},Q}. %Ignore but give error (?) - -%% Status = io_requests(RequestList, PrevStat, Port) -%% Process a list of output requests as long as the previous status is 'ok'. - -io_requests([R|Rs], {ok,_Res,Q}, Port) -> - io_requests(Rs, io_request(R, Port, Q), Port); -io_requests([_|_], Error, _) -> - Error; -io_requests([], Stat, _) -> - Stat. - -%% put_port(DeepList, Port) -%% Take a deep list of characters, flatten and output them to the -%% port. - -put_port(List, Port) -> - true = port_command(Port, List), - ok. - -%% send_port(Port, Command) - -send_port(Port, Command) -> - Port ! {self(),Command}, - ok. - -%% io_reply(From, ReplyAs, Reply) -%% The function for sending i/o command acknowledgement. -%% The ACK contains the return value. - -io_reply(From, ReplyAs, Reply) -> - From ! {io_reply,ReplyAs,Reply}. - -%% put_chars -put_chars(Chars, Port, Q) when is_binary(Chars) -> - ok = put_port(Chars, Port), - {ok,ok,Q}; -put_chars(Chars, Port, Q) -> - case catch list_to_binary(Chars) of - Binary when is_binary(Binary) -> - put_chars(Binary, Port, Q); - _ -> - {error,{error,put_chars},Q} - end. - -expand_encoding([]) -> - []; -expand_encoding([latin1 | T]) -> - [{encoding,latin1} | expand_encoding(T)]; -expand_encoding([unicode | T]) -> - [{encoding,unicode} | expand_encoding(T)]; -expand_encoding([H|T]) -> - [H|expand_encoding(T)]. - -%% setopts -setopts(Opts0,Port,Q) -> - Opts = proplists:unfold( - proplists:substitute_negations( - [{list,binary}], - expand_encoding(Opts0))), - case check_valid_opts(Opts) of - true -> - do_setopts(Opts,Port,Q); - false -> - {error,{error,enotsup},Q} - end. -check_valid_opts([]) -> - true; -check_valid_opts([{binary,_}|T]) -> - check_valid_opts(T); -check_valid_opts([{encoding,Valid}|T]) when Valid =:= latin1; Valid =:= utf8; Valid =:= unicode -> - check_valid_opts(T); -check_valid_opts(_) -> - false. - -do_setopts(Opts, _Port, Q) -> - case proplists:get_value(encoding,Opts) of - Valid when Valid =:= unicode; Valid =:= utf8 -> - put(encoding,unicode); - latin1 -> - put(encoding,latin1); - undefined -> - ok - end, - case proplists:get_value(binary, Opts) of - true -> - put(read_mode,binary), - {ok,ok,Q}; - false -> - put(read_mode,list), - {ok,ok,Q}; - _ -> - {ok,ok,Q} - end. - -getopts(_Port,Q) -> - Bin = {binary, get(read_mode) =:= binary}, - Uni = {encoding, get(encoding)}, - {ok,[Bin,Uni],Q}. - -get_line_bin(Prompt,Port,Q, Enc) -> - case prompt(Port, Prompt) of - error -> - {error,{error,get_line},Q}; - ok -> - case {get(eof),queue:is_empty(Q)} of - {true,true} -> - {ok,eof,Q}; - _ -> - get_line(Prompt,Port, Q, [], Enc) - end - end. - -get_line(Prompt, Port, Q, Acc, Enc) -> - case queue:is_empty(Q) of - true -> - receive - {Port,{data,Bytes}} -> - get_line_bytes(Prompt, Port, Q, Acc, Bytes, Enc); - {Port, eof} -> - put(eof, true), - {ok, eof, queue:new()}; - {io_request,From,ReplyAs,{get_geometry,_}=Req} when is_pid(From) -> - do_io_request(Req, From, ReplyAs, Port, - queue:new()), - %% No prompt. - get_line(Prompt, Port, Q, Acc, Enc); - {io_request,From,ReplyAs,Request} when is_pid(From) -> - do_io_request(Request, From, ReplyAs, Port, queue:new()), - case prompt(Port, Prompt) of - error -> - {error,{error,get_line},Q}; - ok -> - get_line(Prompt, Port, Q, Acc, Enc) - end; - {'EXIT',From,What} when node(From) =:= node() -> - {exit,What} - end; - false -> - get_line_doit(Prompt, Port, Q, Acc, Enc) - end. - -get_line_bytes(Prompt, Port, Q, Acc, Bytes, Enc) -> - case get(shell) of - noshell -> - get_line_doit(Prompt, Port, queue:snoc(Q, Bytes),Acc,Enc); - _ -> - case contains_ctrl_g_or_ctrl_c(Bytes) of - false -> - get_line_doit(Prompt, Port, queue:snoc(Q, Bytes), Acc, Enc); - _ -> - throw(new_shell) - end - end. -is_cr_at(Pos,Bin) -> - case Bin of - <<_:Pos/binary,$\r,_/binary>> -> - true; - _ -> - false - end. -srch(<<>>,_,_) -> - nomatch; -srch(<<X:8,_/binary>>,X,N) -> - {match,[{N,1}]}; -srch(<<_:8,T/binary>>,X,N) -> - srch(T,X,N+1). - -get_line_doit(Prompt, Port, Q, Accu, Enc) -> - case queue:is_empty(Q) of - true -> - case get(eof) of - true -> - case Accu of - [] -> - {ok,eof,Q}; - _ -> - {ok,binrev(Accu,[]),Q} - end; - _ -> - get_line(Prompt, Port, Q, Accu, Enc) - end; - false -> - Bin = queue:head(Q), - case srch(Bin,$\n,0) of - nomatch -> - X = byte_size(Bin)-1, - case is_cr_at(X,Bin) of - true -> - <<D:X/binary,_/binary>> = Bin, - get_line_doit(Prompt, Port, queue:tail(Q), - [<<$\r>>,D|Accu], Enc); - false -> - get_line_doit(Prompt, Port, queue:tail(Q), - [Bin|Accu], Enc) - end; - {match,[{Pos,1}]} -> - %% We are done - PosPlus = Pos + 1, - case Accu of - [] -> - {Head,Tail} = - case is_cr_at(Pos - 1,Bin) of - false -> - <<H:PosPlus/binary, - T/binary>> = Bin, - {H,T}; - true -> - PosMinus = Pos - 1, - <<H:PosMinus/binary, - _,_,T/binary>> = Bin, - {binrev([],[H,$\n]),T} - end, - case Tail of - <<>> -> - {ok, cast(Head,Enc), queue:tail(Q)}; - _ -> - {ok, cast(Head,Enc), - queue:cons(Tail, queue:tail(Q))} - end; - [<<$\r>>|Stack1] when Pos =:= 0 -> - <<_:PosPlus/binary,Tail/binary>> = Bin, - case Tail of - <<>> -> - {ok, cast(binrev(Stack1, [$\n]),Enc), - queue:tail(Q)}; - _ -> - {ok, cast(binrev(Stack1, [$\n]),Enc), - queue:cons(Tail, queue:tail(Q))} - end; - _ -> - {Head,Tail} = - case is_cr_at(Pos - 1,Bin) of - false -> - <<H:PosPlus/binary, - T/binary>> = Bin, - {H,T}; - true -> - PosMinus = Pos - 1, - <<H:PosMinus/binary, - _,_,T/binary>> = Bin, - {[H,$\n],T} - end, - case Tail of - <<>> -> - {ok, cast(binrev(Accu,[Head]),Enc), - queue:tail(Q)}; - _ -> - {ok, cast(binrev(Accu,[Head]),Enc), - queue:cons(Tail, queue:tail(Q))} - end - end - end - end. - -binrev(L, T) -> - list_to_binary(lists:reverse(L, T)). - -%% get_chars(Prompt, Module, Function, XtraArg, Port, Queue, Encoding) -%% Gets characters from the input port until the applied function -%% returns {stop,Result,RestBuf}. Does not block output until input -%% has been received. Encoding is the encoding of the data sent to -%% the client and to Function. -%% Returns: -%% {Status,Result,NewQueue} -%% {exit,Reason} - -%% Entry function. -get_chars(Prompt, M, F, Xa, Port, Q, Enc) -> - case prompt(Port, Prompt) of - error -> - {error,{error,get_chars},Q}; - ok -> - case {get(eof),queue:is_empty(Q)} of - {true,true} -> - {ok,eof,Q}; - _ -> - get_chars(Prompt, M, F, Xa, Port, Q, start, Enc) - end - end. - -%% First loop. Wait for port data. Respond to output requests. -get_chars(Prompt, M, F, Xa, Port, Q, State, Enc) -> - case queue:is_empty(Q) of - true -> - receive - {Port,{data,Bytes}} -> - get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc); - {Port, eof} -> - put(eof, true), - {ok, eof, queue:new()}; - {io_request,From,ReplyAs,{get_geometry,_}=Req} when is_pid(From) -> - do_io_request(Req, From, ReplyAs, Port, - queue:new()), %Keep Q over this call - %% No prompt. - get_chars(Prompt, M, F, Xa, Port, Q, State, Enc); - {io_request,From,ReplyAs,Request} when is_pid(From) -> - get_chars_req(Prompt, M, F, Xa, Port, Q, State, - Request, From, ReplyAs, Enc); - {'EXIT',From,What} when node(From) =:= node() -> - {exit,What} - end; - false -> - get_chars_apply(State, M, F, Xa, Port, Q, Enc) - end. - -get_chars_req(Prompt, M, F, XtraArg, Port, Q, State, - Req, From, ReplyAs, Enc) -> - do_io_request(Req, From, ReplyAs, Port, queue:new()), %Keep Q over this call - case prompt(Port, Prompt) of - error -> - {error,{error,get_chars},Q}; - ok -> - get_chars(Prompt, M, F, XtraArg, Port, Q, State, Enc) - end. - -%% Second loop. Pass data to client as long as it wants more. -%% A ^G in data interrupts loop if 'noshell' is not undefined. -get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc) -> - case get(shell) of - noshell -> - get_chars_apply(State, M, F, Xa, Port, queue:snoc(Q, Bytes),Enc); - _ -> - case contains_ctrl_g_or_ctrl_c(Bytes) of - false -> - get_chars_apply(State, M, F, Xa, Port, - queue:snoc(Q, Bytes),Enc); - _ -> - throw(new_shell) - end - end. - -get_chars_apply(State0, M, F, Xa, Port, Q, Enc) -> - case catch M:F(State0, cast(queue:head(Q),Enc), Enc, Xa) of - {stop,Result,<<>>} -> - {ok,Result,queue:tail(Q)}; - {stop,Result,[]} -> - {ok,Result,queue:tail(Q)}; - {stop,Result,eof} -> - {ok,Result,queue:tail(Q)}; - {stop,Result,Buf} -> - {ok,Result,queue:cons(Buf, queue:tail(Q))}; - {'EXIT',_Why} -> - {error,{error,err_func(M, F, Xa)},queue:new()}; - State1 -> - get_chars_more(State1, M, F, Xa, Port, queue:tail(Q), Enc) - end. - -get_chars_more(State, M, F, Xa, Port, Q, Enc) -> - case queue:is_empty(Q) of - true -> - case get(eof) of - undefined -> - receive - {Port,{data,Bytes}} -> - get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc); - {Port,eof} -> - put(eof, true), - get_chars_apply(State, M, F, Xa, Port, - queue:snoc(Q, eof), Enc); - {'EXIT',From,What} when node(From) =:= node() -> - {exit,What} - end; - _ -> - get_chars_apply(State, M, F, Xa, Port, queue:snoc(Q, eof), Enc) - end; - false -> - get_chars_apply(State, M, F, Xa, Port, Q, Enc) - end. - - -%% prompt(Port, Prompt) -%% Print Prompt onto Port - -%% common case, reduces execution time by 20% -prompt(_Port, '') -> ok; -prompt(Port, Prompt) -> - Encoding = get(encoding), - PromptString = io_lib:format_prompt(Prompt, Encoding), - case wrap_characters_to_binary(PromptString, unicode, Encoding) of - Bin when is_binary(Bin) -> - put_port(Bin, Port); - error -> - error - end. - -%% Convert error code to make it look as before -err_func(io_lib, get_until, {_,F,_}) -> - F; -err_func(_, F, _) -> - F. - -%% using regexp reduces execution time by >50% compared to old code -%% running two regexps in sequence is much faster than \\x03|\\x07 -contains_ctrl_g_or_ctrl_c(BinOrList)-> - case {re:run(BinOrList, <<3>>),re:run(BinOrList, <<7>>)} of - {nomatch, nomatch} -> false; - _ -> true - end. - -%% Convert a buffer between list and binary -cast(Data, _Encoding) when is_atom(Data) -> - Data; -cast(Data, Encoding) -> - IoEncoding = get(encoding), - cast(Data, get(read_mode), IoEncoding, Encoding). - -cast(B, binary, latin1, latin1) when is_binary(B) -> - B; -cast(L, binary, latin1, latin1) -> - case catch erlang:iolist_to_binary(L) of - Bin when is_binary(Bin) -> Bin; - _ -> exit({no_translation, latin1, latin1}) - end; -cast(Data, binary, unicode, latin1) when is_binary(Data); is_list(Data) -> - case catch unicode:characters_to_binary(Data, unicode, latin1) of - Bin when is_binary(Bin) -> Bin; - _ -> exit({no_translation, unicode, latin1}) - end; -cast(Data, binary, latin1, unicode) when is_binary(Data); is_list(Data) -> - case catch unicode:characters_to_binary(Data, latin1, unicode) of - Bin when is_binary(Bin) -> Bin; - _ -> exit({no_translation, latin1, unicode}) - end; -cast(B, binary, unicode, unicode) when is_binary(B) -> - B; -cast(L, binary, unicode, unicode) -> - case catch unicode:characters_to_binary(L, unicode) of - Bin when is_binary(Bin) -> Bin; - _ -> exit({no_translation, unicode, unicode}) - end; -cast(B, list, latin1, latin1) when is_binary(B) -> - binary_to_list(B); -cast(L, list, latin1, latin1) -> - case catch erlang:iolist_to_binary(L) of - Bin when is_binary(Bin) -> binary_to_list(Bin); - _ -> exit({no_translation, latin1, latin1}) - end; -cast(Data, list, unicode, latin1) when is_binary(Data); is_list(Data) -> - case catch unicode:characters_to_list(Data, unicode) of - Chars when is_list(Chars) -> - [ case X of - High when High > 255 -> - exit({no_translation, unicode, latin1}); - Low -> - Low - end || X <- Chars ]; - _ -> - exit({no_translation, unicode, latin1}) - end; -cast(Data, list, latin1, unicode) when is_binary(Data); is_list(Data) -> - case catch unicode:characters_to_list(Data, latin1) of - Chars when is_list(Chars) -> Chars; - _ -> exit({no_translation, latin1, unicode}) - end; -cast(Data, list, unicode, unicode) when is_binary(Data); is_list(Data) -> - case catch unicode:characters_to_list(Data, unicode) of - Chars when is_list(Chars) -> Chars; - _ -> exit({no_translation, unicode, unicode}) - end. - -wrap_characters_to_binary(Chars, unicode, latin1) -> - case catch unicode:characters_to_binary(Chars, unicode, latin1) of - Bin when is_binary(Bin) -> - Bin; - _ -> - case catch unicode:characters_to_list(Chars, unicode) of - L when is_list(L) -> - list_to_binary( - [ case X of - High when High > 255 -> - ["\\x{",erlang:integer_to_list(X, 16),$}]; - Low -> - Low - end || X <- L ]); - _ -> - error - end - end; -wrap_characters_to_binary(Bin, From, From) when is_binary(Bin) -> - Bin; -wrap_characters_to_binary(Chars, From, To) -> - case catch unicode:characters_to_binary(Chars, From, To) of - Bin when is_binary(Bin) -> - Bin; - _ -> - error - end. diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index fa7687bf2a..11eab9cca8 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -19,591 +19,849 @@ %% -module(user_drv). -%% Basic interface to a port. - --export([start/0,start/1,start/2,start/3,server/2,server/3]). - --export([interfaces/1]). +%% Basic interface to stdin/stdout. +%% +%% This is responsible for a couple of things: +%% - Dispatching I/O messages when erl is running +%% The messages are listed in the type message/0. +%% - Any data received from the terminal is sent to the current group like this: +%% `{DrvPid :: pid(), {data, UnicodeCharacters :: list()}}` +%% - It serves as the job control manager (i.e. what happens when you type ^G) +%% - Starts potential -remsh sessions to other nodes +%% +-type message() :: + %% I/O requests that modify the terminal + {Sender :: pid(), request()} | + %% Query the server of the current dimensions of the terminal. + %% `Sender` will be sent the message: + %% `{DrvPid :: pid(), tty_geometry, {Width :: integer(), Height :: integer()}}` + {Sender :: pid(), tty_geometry} | + %% Query the server if it supports unicode characters + %% `Sender` will be sent the message: + %% `{DrvPid :: pid(), get_unicode_state, SupportUnicode :: boolean()}` + {Sender :: pid(), get_unicode_state} | + %% Change whether the server supports unicode characters or not. The reply + %% contains the previous unicode state. + %% `Sender` will be sent the message: + %% `{DrvPid :: pid(), set_unicode_state, SupportedUnicode :: boolean()}` + {Sender :: pid(), set_unicode_state, boolean()}. +-type request() :: + %% Put characters at current cursor position, + %% overwriting any characters it encounters. + {put_chars, unicode, binary()} | + %% Same as put_chars/3, but sends Reply to From when the characters are + %% guaranteed to have been written to the terminal + {put_chars_sync, unicode, binary(), {From :: pid(), Reply :: term()}} | + %% Move the cursor X characters left or right (negative is left) + {move_rel, -32768..32767} | + %% Insert characters at current cursor position moving any + %% characters after the cursor. + {insert_chars, unicode, binary()} | + %% Delete X chars before or after the cursor adjusting any test remaining + %% to the right of the cursor. + {delete_chars, -32768..32767} | + %% Trigger a terminal "bell" + beep | + %% Clears the screen + clear | + %% Execute multiple request() actions + {requests, [request()]} | + %% Open external editor + {open_editor, string()}. + +-export_type([message/0]). +-export([start/0, start/1, start_shell/0, start_shell/1, whereis_group/0]). + +%% gen_statem state callbacks +-behaviour(gen_statem). +-export([init/3,server/3,switch_loop/3]). + +%% gen_statem callbacks +-export([init/1, callback_mode/0]). -include_lib("kernel/include/logger.hrl"). --define(OP_PUTC,0). --define(OP_MOVE,1). --define(OP_INSC,2). --define(OP_DELC,3). --define(OP_BEEP,4). --define(OP_PUTC_SYNC,5). -% Control op --define(ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER, 16#018b0900). --define(CTRL_OP_GET_WINSIZE, (100 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). --define(CTRL_OP_GET_UNICODE_STATE, (101 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). --define(CTRL_OP_SET_UNICODE_STATE, (102 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). - -%% start() -%% start(ArgumentList) -%% start(PortName, Shell) -%% start(InPortName, OutPortName, Shell) -%% Start the user driver server. The arguments to start/1 are slightly -%% strange as this may be called both at start up from the command line -%% and explicitly from other code. - +-record(editor, { port :: port(), file :: file:name(), requester :: pid() }). +-record(state, { tty :: prim_tty:state() | undefined, + write :: reference() | undefined, + read :: reference() | undefined, + shell_started = new :: new | old | false, + editor :: #editor{} | undefined, + user :: pid(), + current_group :: pid() | undefined, + groups, queue }). + +-type shell() :: {module(), atom(), arity()} | {node(), module(), atom(), arity()}. +-type arguments() :: #{ initial_shell => noshell | shell() | + {remote, unicode:charlist()} | {remote, unicode:charlist(), mfa()}, + input => boolean() }. + +%% Default line editing shell -spec start() -> pid(). +start() -> + case init:get_argument(remsh) of + {ok,[[Node]]} -> + start(#{ initial_shell => {remote, Node} }); + {ok,[[Node]|_]} -> + ?LOG_WARNING("Multiple -remsh given to erl, using the first, ~p", [Node]), + start(#{ initial_shell => {remote, Node} }); + E when E =:= error ; E =:= {ok,[[]]} -> + start(#{ }) + end. -start() -> %Default line editing shell - spawn(user_drv, server, ['tty_sl -c -e',{shell,start,[init]}]). - -start([Pname]) -> - spawn(user_drv, server, [Pname,{shell,start,[init]}]); -start([Pname|Args]) -> - spawn(user_drv, server, [Pname|Args]); -start(Pname) -> - spawn(user_drv, server, [Pname,{shell,start,[init]}]). +-spec start_shell() -> ok | {error, Reason :: term()}. +start_shell() -> + start_shell(#{ }). +-spec start_shell(arguments()) -> ok | {error, already_started}. +start_shell(Args) -> + gen_statem:call(?MODULE, {start_shell, Args}). + +-spec whereis_group() -> pid() | undefined. +whereis_group() -> + {dictionary, Dict} = + erlang:process_info(whereis(?MODULE), dictionary), + proplists:get_value(current_group, Dict). + +%% Backwards compatibility with pre OTP-26 for Elixir/LFE etc +-spec start(['tty_sl -c -e'| shell()]) -> pid(); + (arguments()) -> pid(). +start(['tty_sl -c -e', Shell]) -> + start(#{ initial_shell => Shell }); +start(Args) when is_map(Args) -> + case gen_statem:start({local, ?MODULE}, ?MODULE, Args, []) of + {ok, Pid} -> Pid; + {error, Reason} -> + {error, Reason} + end. -start(Pname, Shell) -> - spawn(user_drv, server, [Pname,Shell]). +callback_mode() -> state_functions. -start(Iname, Oname, Shell) -> - spawn(user_drv, server, [Iname,Oname,Shell]). +-spec init(arguments()) -> gen_statem:init_result(init). +init(Args) -> + process_flag(trap_exit, true), + %% When running in embedded mode we need to call prim_tty:on_load manually here + %% as the automatic call happens after user is started. + ok = init:run_on_load_handlers([prim_tty]), + + IsTTY = prim_tty:isatty(stdin) =:= true andalso prim_tty:isatty(stdout) =:= true, + StartShell = maps:get(initial_shell, Args, undefined) =/= noshell, + OldShell = maps:get(initial_shell, Args, undefined) =:= oldshell, + try + if + not IsTTY andalso StartShell; OldShell -> + error(enotsup); + IsTTY, StartShell -> + TTYState = prim_tty:init(#{}), + init_standard_error(TTYState, true), + {ok, init, {Args, #state{ user = start_user() } }, + {next_event, internal, TTYState}}; + true -> + TTYState = prim_tty:init(#{input => maps:get(input, Args, true), + tty => false}), + init_standard_error(TTYState, false), + {ok, init, {Args,#state{ user = start_user() } }, + {next_event, internal, TTYState}} + end + catch error:enotsup -> + %% This is thrown by prim_tty:init when + %% it could not start the terminal, + %% probably because TERM=dumb was set. + %% + %% The oldshell mode is important as it is + %% the mode used when running erlang in an + %% emacs buffer. + CatchTTYState = prim_tty:init(#{tty => false}), + init_standard_error(CatchTTYState, false), + {ok, init, {Args,#state{ shell_started = old, user = start_user() } }, + {next_event, internal, CatchTTYState}} + end. +%% Initialize standard_error +init_standard_error(TTY, NewlineCarriageReturn) -> + Encoding = case prim_tty:unicode(TTY) of + true -> unicode; + false -> latin1 + end, + ok = io:setopts(standard_error, [{encoding, Encoding}, + {onlcr, NewlineCarriageReturn}]). + +init(internal, TTYState, {Args, State = #state{ user = User }}) -> + + %% Cleanup ancestors so that observer looks nice + put('$ancestors',[User|get('$ancestors')]), + + #{ read := ReadHandle, write := WriteHandle } = prim_tty:handles(TTYState), + + NewState = State#state{ tty = TTYState, + read = ReadHandle, write = WriteHandle, + user = User, queue = {false, queue:new()}, + groups = gr_add_cur(gr_new(), User, {}) + }, + + case Args of + #{ initial_shell := noshell } -> + init_noshell(NewState); + #{ initial_shell := {remote, Node} } -> + InitialShell = {shell,start,[]}, + exit_on_remote_shell_error( + Node, InitialShell, init_remote_shell(NewState, Node, InitialShell)); + #{ initial_shell := {remote, Node, InitialShell} } -> + exit_on_remote_shell_error( + Node, InitialShell, init_remote_shell(NewState, Node, InitialShell)); + #{ initial_shell := oldshell } -> + old = State#state.shell_started, + init_local_shell(NewState, {shell,start,[]}); + #{ initial_shell := InitialShell } -> + init_local_shell(NewState, InitialShell); + _ -> + init_local_shell(NewState, {shell,start,[init]}) + end. -%% Return the pid of the active group process. -%% Note: We can't ask the user_drv process for this info since it -%% may be busy waiting for data from the port. +exit_on_remote_shell_error(RemoteNode, _, {error, noconnection}) -> + io:format(standard_error, "Could not connect to ~p\n", [RemoteNode]), + erlang:halt(1); +exit_on_remote_shell_error(RemoteNode, {M, _, _}, {error, Reason}) -> + io:format(standard_error, "Could not load ~p on ~p (~p)\n", [RemoteNode, M, Reason]), + erlang:halt(1); +exit_on_remote_shell_error(_, _, Result) -> + Result. + +%% We have been started with -noshell. In this mode the current_group is +%% the `user` group process. +init_noshell(State) -> + init_shell(State#state{ shell_started = false }, ""). + +init_remote_shell(State, Node, {M, F, A}) -> + + case net_kernel:get_state() of + #{ started := no } -> + {ok, _} = net_kernel:start([undefined, shortnames]), + ok; + _ -> + ok + end, --spec interfaces(pid()) -> [{'current_group', pid()}]. + LocalNode = + case net_kernel:get_state() of + #{ name_type := dynamic } -> + net_kernel:nodename(); + #{ name_type := static } -> + node() + end, + + RemoteNode = + case string:find(Node,"@") of + nomatch -> + list_to_atom(Node ++ string:find(atom_to_list(LocalNode),"@")); + _ -> + list_to_atom(Node) + end, + + case net_kernel:connect_node(RemoteNode) of + true -> + + case erpc:call(RemoteNode, code, ensure_loaded, [M]) of + {error, Reason} when Reason =/= embedded -> + {error, Reason}; + _ -> + + %% Setup correct net tick times + case erpc:call(RemoteNode, net_kernel, get_net_ticktime, []) of + {ongoing_change_to, NetTickTime} -> + _ = net_kernel:set_net_ticktime(NetTickTime), + ok; + NetTickTime -> + _ = net_kernel:set_net_ticktime(NetTickTime), + ok + end, -interfaces(UserDrv) -> - case process_info(UserDrv, dictionary) of - {dictionary,Dict} -> - case lists:keysearch(current_group, 1, Dict) of - {value,Gr={_,Group}} when is_pid(Group) -> - [Gr]; - _ -> - [] - end; - _ -> - [] + RShell = {RemoteNode, M, F, A}, + + %% We fetch the shell slogan from the remote node + Slogan = + case erpc:call(RemoteNode, application, get_env, + [stdlib, shell_slogan, + erpc:call(RemoteNode, erlang, system_info, [system_version])]) of + Fun when is_function(Fun, 0) -> + erpc:call(RemoteNode, Fun); + SloganEnv -> + SloganEnv + end, + + Group = group:start(self(), RShell, + [{echo,State#state.shell_started =:= new}] ++ + group_opts(RemoteNode)), + + Gr = gr_add_cur(State#state.groups, Group, RShell), + + init_shell(State#state{ groups = Gr }, [Slogan,$\n]) + end; + false -> + {error, noconnection} end. -%% server(Pid, Shell) -%% server(Pname, Shell) -%% server(Iname, Oname, Shell) -%% The initial calls to run the user driver. These start the port(s) -%% then call server1/3 to set everything else up. +init_local_shell(State, InitialShell) -> -server(Pid, Shell) when is_pid(Pid) -> - server1(Pid, Pid, Shell); -server(Pname, Shell) -> - process_flag(trap_exit, true), - case catch open_port({spawn,Pname}, [eof]) of - {'EXIT', _} -> - %% Let's try a dumb user instead - user:start(); - Port -> - server1(Port, Port, Shell) - end. + Slogan = + case application:get_env( + stdlib, shell_slogan, + fun() -> erlang:system_info(system_version) end) of + Fun when is_function(Fun, 0) -> + Fun(); + SloganEnv -> + SloganEnv + end, -server(Iname, Oname, Shell) -> - process_flag(trap_exit, true), - case catch open_port({spawn,Iname}, [eof]) of - {'EXIT', _} -> %% It might be a dumb terminal lets start dumb user - user:start(); - Iport -> - Oport = open_port({spawn,Oname}, [eof]), - server1(Iport, Oport, Shell) - end. + Gr = gr_add_cur(State#state.groups, + group:start(self(), InitialShell, + group_opts() ++ [{echo,State#state.shell_started =:= new}]), + InitialShell), -server1(Iport, Oport, Shell) -> - put(eof, false), - %% Start user and initial shell. - User = start_user(), - Gr1 = gr_add_cur(gr_new(), User, {}), - - {Curr,Shell1} = - case init:get_argument(remsh) of - {ok,[[Node]]} -> - ANode = - if - node() =:= nonode@nohost -> - %% We try to connect to the node if the current node is not - %% a distributed node yet. If this succeeds it means that we - %% are running using "-sname undefined". - _ = net_kernel:start([undefined, shortnames]), - NodeName = append_hostname(Node, net_kernel:nodename()), - case net_kernel:connect_node(NodeName) of - true -> - NodeName; - _Else -> - ?LOG_ERROR("Could not connect to ~p",[Node]) - end; - true -> - append_hostname(Node, node()) - end, + init_shell(State#state{ groups = Gr }, [Slogan,$\n]). - RShell = {ANode,shell,start,[]}, - RGr = group:start(self(), RShell, rem_sh_opts(ANode)), - {RGr,RShell}; - E when E =:= error ; E =:= {ok,[[]]} -> - {group:start(self(), Shell),Shell} - end, +init_shell(State, Slogan) -> + init_standard_error(State#state.tty, State#state.shell_started =:= new), + Curr = gr_cur_pid(State#state.groups), put(current_group, Curr), - Gr = gr_add_cur(Gr1, Curr, Shell1), - %% Print some information. - io_request({put_chars, unicode, - flatten(io_lib:format("~ts\n", - [erlang:system_info(system_version)]))}, - Iport, Oport), - - %% Enter the server loop. - server_loop(Iport, Oport, Curr, User, Gr, {false, queue:new()}). - -append_hostname(Node, LocalNode) -> - case string:find(Node,"@") of - nomatch -> - list_to_atom(Node ++ string:find(atom_to_list(LocalNode),"@")); - _ -> - list_to_atom(Node) - end. - -rem_sh_opts(Node) -> - [{expand_fun,fun(B)-> rpc:call(Node,edlin_expand,expand,[B]) end}]. + {next_state, server, State#state{ current_group = gr_cur_pid(State#state.groups) }, + {next_event, info, + {gr_cur_pid(State#state.groups), + {put_chars, unicode, + unicode:characters_to_binary(io_lib:format("~ts", [Slogan]))}}}}. %% start_user() %% Start a group leader process and register it as 'user', unless, %% of course, a 'user' already exists. - start_user() -> - case whereis(user_drv) of - undefined -> - register(user_drv, self()); - _ -> - ok - end, case whereis(user) of undefined -> - User = group:start(self(), {}), + User = group:start(self(), {}, [{echo,false}]), register(user, User), User; User -> User end. - -server_loop(Iport, Oport, User, Gr, IOQueue) -> - Curr = gr_cur_pid(Gr), - put(current_group, Curr), - server_loop(Iport, Oport, Curr, User, Gr, IOQueue). - -server_loop(Iport, Oport, Curr, User, Gr, {Resp, IOQ} = IOQueue) -> - receive - {Iport,{data,Bs}} -> - BsBin = list_to_binary(Bs), - Unicode = unicode:characters_to_list(BsBin,utf8), - port_bytes(Unicode, Iport, Oport, Curr, User, Gr, IOQueue); - {Iport,eof} -> - Curr ! {self(),eof}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - - %% We always handle geometry and unicode requests - {Requester,tty_geometry} -> - Requester ! {self(),tty_geometry,get_tty_geometry(Iport)}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - {Requester,get_unicode_state} -> - Requester ! {self(),get_unicode_state,get_unicode_state(Iport)}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - {Requester,set_unicode_state, Bool} -> - Requester ! {self(),set_unicode_state,set_unicode_state(Iport,Bool)}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - - Req when element(1,Req) =:= User orelse element(1,Req) =:= Curr, - tuple_size(Req) =:= 2 orelse tuple_size(Req) =:= 3 -> - %% We match {User|Curr,_}|{User|Curr,_,_} - NewQ = handle_req(Req, Iport, Oport, IOQueue), - server_loop(Iport, Oport, Curr, User, Gr, NewQ); - {Oport,ok} -> - %% We get this ok from the port, in io_request we store - %% info about where to send reply at head of queue - {Origin,Reply} = Resp, - Origin ! {reply,Reply}, - NewQ = handle_req(next, Iport, Oport, {false, IOQ}), - server_loop(Iport, Oport, Curr, User, Gr, NewQ); - {'EXIT',Iport,_R} -> - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - {'EXIT',Oport,_R} -> - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - {'EXIT',User,shutdown} -> % force data to port - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - {'EXIT',User,_R} -> % keep 'user' alive - NewU = start_user(), - server_loop(Iport, Oport, Curr, NewU, gr_set_num(Gr, 1, NewU, {}), IOQueue); - {'EXIT',Pid,R} -> % shell and group leader exit - case gr_cur_pid(Gr) of - Pid when R =/= die , - R =/= terminated -> % current shell exited - if R =/= normal -> - io_requests([{put_chars,unicode,"*** ERROR: "}], Iport, Oport); - true -> % exit not caused by error - io_requests([{put_chars,unicode,"*** "}], Iport, Oport) - end, - io_requests([{put_chars,unicode,"Shell process terminated! "}], Iport, Oport), - Gr1 = gr_del_pid(Gr, Pid), - case gr_get_info(Gr, Pid) of - {Ix,{shell,start,Params}} -> % 3-tuple == local shell - io_requests([{put_chars,unicode,"***\n"}], Iport, Oport), - %% restart group leader and shell, same index - Pid1 = group:start(self(), {shell,start,Params}), - {ok,Gr2} = gr_set_cur(gr_set_num(Gr1, Ix, Pid1, - {shell,start,Params}), Ix), - put(current_group, Pid1), - server_loop(Iport, Oport, Pid1, User, Gr2, IOQueue); - _ -> % remote shell - io_requests([{put_chars,unicode,"(^G to start new job) ***\n"}], - Iport, Oport), - server_loop(Iport, Oport, Curr, User, Gr1, IOQueue) - end; - _ -> % not current, just remove it - server_loop(Iport, Oport, Curr, User, gr_del_pid(Gr, Pid), IOQueue) - end; - {Requester, {put_chars_sync, _, _, Reply}} -> - %% We need to ack the Req otherwise originating process will hang forever - %% Do discard the output to non visible shells (as was done previously) - Requester ! {reply, Reply}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); - _X -> - %% Ignore unknown messages. - server_loop(Iport, Oport, Curr, User, Gr, IOQueue) - end. -handle_req(next,Iport,Oport,{false,IOQ}=IOQueue) -> - case queue:out(IOQ) of - {empty,_} -> - IOQueue; - {{value,{Origin,Req}},ExecQ} -> - case io_request(Req, Iport, Oport) of - ok -> - handle_req(next,Iport,Oport,{false,ExecQ}); - Reply -> - {{Origin,Reply}, ExecQ} +server({call, From}, {start_shell, Args}, + State = #state{ tty = TTY, shell_started = false }) -> + IsTTY = prim_tty:isatty(stdin) =:= true andalso prim_tty:isatty(stdout) =:= true, + StartShell = maps:get(initial_shell, Args, undefined) =/= noshell, + OldShell = maps:get(initial_shell, Args, undefined) =:= oldshell, + NewState = + try + if + not IsTTY andalso StartShell; OldShell -> + error(enotsup); + IsTTY, StartShell -> + NewTTY = prim_tty:reinit(TTY, #{ }), + State#state{ tty = NewTTY, + shell_started = new }; + true -> + NewTTY = prim_tty:reinit(TTY, #{ tty => false }), + State#state{ tty = NewTTY, shell_started = false } end + catch error:enotsup -> + NewTTYState = prim_tty:reinit(TTY, #{ tty => false }), + State#state{ tty = NewTTYState, shell_started = old } + end, + #{ read := ReadHandle, write := WriteHandle } = prim_tty:handles(NewState#state.tty), + NewHandleState = NewState#state { + read = ReadHandle, + write = WriteHandle + }, + {Result, Reply} + = case maps:get(initial_shell, Args, undefined) of + noshell -> + {init_noshell(NewHandleState), ok}; + {remote, Node} -> + case init_remote_shell(NewHandleState, Node, {shell, start, []}) of + {error, _} = Error -> + {init_noshell(NewHandleState), Error}; + R -> + {R, ok} + end; + {remote, Node, InitialShell} -> + case init_remote_shell(NewHandleState, Node, InitialShell) of + {error, _} = Error -> + {init_noshell(NewHandleState), Error}; + R -> + {R, ok} + end; + undefined -> + case NewHandleState#state.shell_started of + old -> + {init_local_shell(NewHandleState, {shell,start,[]}), ok}; + new -> + {init_local_shell(NewHandleState, {shell,start,[init]}), ok}; + false -> + %% This can never happen, but dialyzer complains so we add + %% this clause. + {keep_state_and_data, ok} + end; + InitialShell -> + {init_local_shell(NewHandleState, InitialShell), ok} + end, + gen_statem:reply(From, Reply), + Result; +server({call, From}, {start_shell, _Args}, _State) -> + gen_statem:reply(From, {error, already_started}), + keep_state_and_data; +server(info, {ReadHandle,{data,UTF8Binary}}, State = #state{ read = ReadHandle }) + when State#state.current_group =:= State#state.user -> + State#state.current_group ! + {self(), {data, unicode:characters_to_list(UTF8Binary, utf8)}}, + keep_state_and_data; +server(info, {ReadHandle,{data,UTF8Binary}}, State = #state{ read = ReadHandle }) -> + case contains_ctrl_g_or_ctrl_c(UTF8Binary) of + ctrl_g -> {next_state, switch_loop, State, {next_event, internal, init}}; + ctrl_c -> + case gr_get_info(State#state.groups, State#state.current_group) of + undefined -> ok; + _ -> exit(State#state.current_group, interrupt) + end, + keep_state_and_data; + none -> + State#state.current_group ! + {self(), {data, unicode:characters_to_list(UTF8Binary, utf8)}}, + keep_state_and_data end; -handle_req(Msg,Iport,Oport,{false,IOQ}=IOQueue) -> - empty = queue:peek(IOQ), - {Origin,Req} = Msg, - case io_request(Req, Iport, Oport) of - ok -> - IOQueue; - Reply -> - {{Origin,Reply}, IOQ} - end; -handle_req(Msg,_Iport,_Oport,{Resp, IOQ}) -> - %% All requests are queued when we have outstanding sync put_chars - {Resp, queue:in(Msg,IOQ)}. - -%% port_bytes(Bytes, InPort, OutPort, CurrentProcess, UserProcess, Group) -%% Check the Bytes from the port to see if it contains a ^G. If so, -%% either escape to switch_loop or restart the shell. Otherwise send -%% the bytes to Curr. - -port_bytes([$\^G|_Bs], Iport, Oport, _Curr, User, Gr, IOQueue) -> - handle_escape(Iport, Oport, User, Gr, IOQueue); - -port_bytes([$\^C|_Bs], Iport, Oport, Curr, User, Gr, IOQueue) -> - interrupt_shell(Iport, Oport, Curr, User, Gr, IOQueue); - -port_bytes([B], Iport, Oport, Curr, User, Gr, IOQueue) -> - Curr ! {self(),{data,[B]}}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue); -port_bytes(Bs, Iport, Oport, Curr, User, Gr, IOQueue) -> - case member($\^G, Bs) of - true -> - handle_escape(Iport, Oport, User, Gr, IOQueue); - false -> - Curr ! {self(),{data,Bs}}, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue) - end. - -interrupt_shell(Iport, Oport, Curr, User, Gr, IOQueue) -> - case gr_get_info(Gr, Curr) of - undefined -> - ok; % unknown - _ -> - exit(Curr, interrupt) +server(info, {ReadHandle,eof}, State = #state{ read = ReadHandle }) -> + State#state.current_group ! {self(), eof}, + keep_state_and_data; +server(info,{ReadHandle,{signal,Signal}}, State = #state{ tty = TTYState, read = ReadHandle }) -> + {keep_state, State#state{ tty = prim_tty:handle_signal(TTYState, Signal) }}; + +server(info, {Requester, tty_geometry}, #state{ tty = TTYState }) -> + case prim_tty:window_size(TTYState) of + {ok, Geometry} -> + Requester ! {self(), tty_geometry, Geometry}, + ok; + Error -> + Requester ! {self(), tty_geometry, Error}, + ok end, - server_loop(Iport, Oport, Curr, User, Gr, IOQueue). - -handle_escape(Iport, Oport, User, Gr, IOQueue) -> - case application:get_env(stdlib, shell_esc) of - {ok,abort} -> - Pid = gr_cur_pid(Gr), - exit(Pid, die), + keep_state_and_data; +server(info, {Requester, get_unicode_state}, #state{ tty = TTYState }) -> + Requester ! {self(), get_unicode_state, prim_tty:unicode(TTYState) }, + keep_state_and_data; +server(info, {Requester, set_unicode_state, Bool}, #state{ tty = TTYState } = State) -> + OldUnicode = prim_tty:unicode(TTYState), + NewTTYState = prim_tty:unicode(TTYState, Bool), + ok = io:setopts(standard_error,[{encoding, if Bool -> unicode; true -> latin1 end}]), + Requester ! {self(), set_unicode_state, OldUnicode}, + {keep_state, State#state{ tty = NewTTYState }}; +server(info, {Requester, get_terminal_state}, _State) -> + Requester ! {self(), get_terminal_state, prim_tty:isatty(stdout) }, + keep_state_and_data; +server(info, {Requester, {open_editor, Buffer}}, #state{tty = TTYState } = State) -> + case open_editor(TTYState, Buffer) of + false -> + Requester ! {self(), {editor_data, Buffer}}, + keep_state_and_data; + {EditorPort, TmpPath} -> + {keep_state, State#state{ editor = #editor{ port = EditorPort, + file = TmpPath, + requester = Requester }}} + end; +server(info, Req, State = #state{ user = User, current_group = Curr, editor = undefined }) + when element(1,Req) =:= User orelse element(1,Req) =:= Curr, + tuple_size(Req) =:= 2 orelse tuple_size(Req) =:= 3 -> + %% We match {User|Curr,_}|{User|Curr,_,_} + {NewTTYState, NewQueue} = handle_req(Req, State#state.tty, State#state.queue), + {keep_state, State#state{ tty = NewTTYState, queue = NewQueue }}; +server(info, {WriteRef, ok}, State = #state{ write = WriteRef, + queue = {{Origin, MonitorRef, Reply}, IOQ} }) -> + %% We get this ok from the user_drv_writer, in io_request we store + %% info about where to send reply at head of queue + Origin ! {reply, Reply, ok}, + erlang:demonitor(MonitorRef, [flush]), + {NewTTYState, NewQueue} = handle_req(next, State#state.tty, {false, IOQ}), + {keep_state, State#state{ tty = NewTTYState, queue = NewQueue }}; +server(info, {'DOWN', MonitorRef, _, _, Reason}, + #state{ queue = {{Origin, MonitorRef, Reply}, _IOQ} }) -> + %% The writer process died, we send the correct error to the caller and + %% then stop this process. This will bring down all linked groups (including 'user'). + %% All writes from now on will throw badarg terminated. + Origin ! {reply, Reply, {error, Reason}}, + ?LOG_INFO("Failed to write to standard out (~p)", [Reason]), + stop; +server(info,{Requester, {put_chars_sync, _, _, Reply}}, _State) -> + %% This is a sync request from an unknown or inactive group. + %% We need to ack the Req otherwise originating process will hang forever. + %% We discard the output to non visible shells + Requester ! {reply, Reply, ok}, + keep_state_and_data; + +server(info,{'EXIT',User, shutdown}, #state{ user = User }) -> + keep_state_and_data; +server(info,{'EXIT',User, _Reason}, State = #state{ user = User }) -> + NewUser = start_user(), + {keep_state, State#state{ user = NewUser, + groups = gr_set_num(State#state.groups, 1, NewUser, {})}}; +server(info, {'EXIT', EditorPort, _R}, + State = #state{tty = TTYState, + editor = #editor{ requester = Requester, + port = EditorPort, + file = PathTmp}}) -> + {ok, Content} = file:read_file(PathTmp), + _ = file:del_dir_r(PathTmp), + Unicode = case unicode:characters_to_list(Content,unicode) of + {error, _, _} -> unicode:characters_to_list( + unicode:characters_to_list(Content,latin1), unicode); + U -> U + end, + Requester ! {self(), {editor_data, string:chomp(Unicode)}}, + ok = prim_tty:enable_reader(TTYState), + {keep_state, State#state{editor = undefined}}; +server(info,{'EXIT', Group, Reason}, State) -> % shell and group leader exit + case gr_cur_pid(State#state.groups) of + Group when Reason =/= die, Reason =/= terminated -> % current shell exited + Reqs = [if + Reason =/= normal -> + {put_chars,unicode,<<"*** ERROR: ">>}; + true -> % exit not caused by error + {put_chars,unicode,<<"*** ">>} + end, + {put_chars,unicode,<<"Shell process terminated! ">>}], + Gr1 = gr_del_pid(State#state.groups, Group), + case gr_get_info(State#state.groups, Group) of + {Ix,{shell,start,Params}} -> % 3-tuple == local shell + NewTTyState = io_requests(Reqs ++ [{put_chars,unicode,<<"***\n">>}], + State#state.tty), + %% restart group leader and shell, same index + NewGroup = group:start(self(), {shell,start,Params}), + {ok,Gr2} = gr_set_cur(gr_set_num(Gr1, Ix, NewGroup, + {shell,start,Params}), Ix), + {keep_state, State#state{ tty = NewTTyState, + current_group = NewGroup, + groups = Gr2 }}; + _ -> % remote shell + NewTTYState = io_requests( + Reqs ++ [{put_chars,unicode,<<"(^G to start new job) ***\n">>}], + State#state.tty), + {keep_state, State#state{ tty = NewTTYState, groups = Gr1 }} + end; + _ -> % not current, just remove it + {keep_state, State#state{ groups = gr_del_pid(State#state.groups, Group) }} + end; +server(_, _, _) -> + keep_state_and_data. + +contains_ctrl_g_or_ctrl_c(<<$\^G,_/binary>>) -> + ctrl_g; +contains_ctrl_g_or_ctrl_c(<<$\^C,_/binary>>) -> + ctrl_c; +contains_ctrl_g_or_ctrl_c(<<_/utf8,T/binary>>) -> + contains_ctrl_g_or_ctrl_c(T); +contains_ctrl_g_or_ctrl_c(<<>>) -> + none. + +switch_loop(internal, init, State) -> + case application:get_env(stdlib, shell_esc, jcl) of + abort -> + CurrGroup = gr_cur_pid(State#state.groups), + exit(CurrGroup, die), Gr1 = - case gr_get_info(Gr, Pid) of - {_Ix,{}} -> % no shell - Gr; + case gr_get_info(State#state.groups, CurrGroup) of + {_Ix,{}} -> % no shell + State#state.groups; _ -> - receive {'EXIT',Pid,_} -> - gr_del_pid(Gr, Pid) + receive {'EXIT',CurrGroup,_} -> + gr_del_pid(State#state.groups, CurrGroup) after 1000 -> - Gr + State#state.groups end end, - Pid1 = group:start(self(), {shell,start,[]}), - io_request({put_chars,unicode,"\n"}, Iport, Oport), - server_loop(Iport, Oport, User, - gr_add_cur(Gr1, Pid1, {shell,start,[]}), IOQueue); - - _ -> % {ok,jcl} | undefined - io_request({put_chars,unicode,"\nUser switch command\n"}, Iport, Oport), + NewGroup = group:start(self(), {shell,start,[]}), + NewTTYState = io_requests([{put_chars,unicode,<<"\n">>}], State#state.tty), + {next_state, server, + State#state{ tty = NewTTYState, + groups = gr_add_cur(Gr1, NewGroup, {shell,start,[]})}}; + jcl -> + NewTTYState = + io_requests([{put_chars,unicode,<<"\nUser switch command (type h for help)\n">>}], + State#state.tty), %% init edlin used by switch command and have it copy the %% text buffer from current group process - edlin:init(gr_cur_pid(Gr)), - server_loop(Iport, Oport, User, switch_loop(Iport, Oport, Gr), IOQueue) - end. - -switch_loop(Iport, Oport, Gr) -> - Line = get_line(edlin:start(" --> "), Iport, Oport), - switch_cmd(erl_scan:string(Line), Iport, Oport, Gr). - -switch_cmd({ok,[{atom,_,c},{integer,_,I}],_}, Iport, Oport, Gr0) -> - case gr_set_cur(Gr0, I) of - {ok,Gr} -> Gr; - undefined -> unknown_group(Iport, Oport, Gr0) + edlin:init(gr_cur_pid(State#state.groups)), + {keep_state, State#state{ tty = NewTTYState }, + {next_event, internal, line}} end; -switch_cmd({ok,[{atom,_,c}],_}, Iport, Oport, Gr) -> - case gr_get_info(Gr, gr_cur_pid(Gr)) of - undefined -> - unknown_group(Iport, Oport, Gr); - _ -> - Gr +switch_loop(internal, line, State) -> + {more_chars, Cont, Rs} = edlin:start(" --> "), + {keep_state, {Cont, State#state{ tty = io_requests(Rs, State#state.tty) }}}; +switch_loop(internal, {line, Line}, State) -> + case erl_scan:string(Line) of + {ok, Tokens, _} -> + case switch_cmd(Tokens, State#state.groups) of + {ok, Groups} -> + Curr = gr_cur_pid(Groups), + put(current_group, Curr), + {next_state, server, + State#state{ current_group = Curr, groups = Groups } }; + {retry, Requests} -> + {keep_state, State#state{ tty = io_requests(Requests, State#state.tty) }, + {next_event, internal, line}}; + {retry, Requests, Groups} -> + Curr = gr_cur_pid(Groups), + put(current_group, Curr), + {keep_state, State#state{ + tty = io_requests(Requests, State#state.tty), + current_group = Curr, + groups = Groups }, + {next_event, internal, line}} + end; + {error, _, _} -> + NewTTYState = + io_requests([{put_chars,unicode,<<"Illegal input\n">>}], State#state.tty), + {keep_state, State#state{ tty = NewTTYState }, + {next_event, internal, line}} end; -switch_cmd({ok,[{atom,_,i},{integer,_,I}],_}, Iport, Oport, Gr) -> +switch_loop(info,{ReadHandle,{data,Cs}}, {Cont, #state{ read = ReadHandle } = State}) -> + case edlin:edit_line(unicode:characters_to_list(Cs), Cont) of + {done,Line,_Rest, Rs} -> + {keep_state, State#state{ tty = io_requests(Rs, State#state.tty) }, + {next_event, internal, {line, Line}}}; + {undefined,_Char,MoreCs,NewCont,Rs} -> + {keep_state, + {NewCont, State#state{ tty = io_requests(Rs ++ [beep], State#state.tty)}}, + {next_event, info, {ReadHandle,{data,MoreCs}}}}; + {more_chars,NewCont,Rs} -> + {keep_state, + {NewCont, State#state{ tty = io_requests(Rs, State#state.tty)}}}; + {blink,NewCont,Rs} -> + {keep_state, + {NewCont, State#state{ tty = io_requests(Rs, State#state.tty)}}, + 1000} + end; +switch_loop(timeout, _, {_Cont, State}) -> + {keep_state_and_data, + {next_event, info, {State#state.read,{data,[]}}}}; +switch_loop(info, _Unknown, _State) -> + {keep_state_and_data, postpone}. + +switch_cmd([{atom,_,Key},{Type,_,Value}], Gr) + when Type =:= atom; Type =:= integer -> + switch_cmd({Key, Value}, Gr); +switch_cmd([{atom,_,Key},{atom,_,V1},{atom,_,V2}], Gr) -> + switch_cmd({Key, V1, V2}, Gr); +switch_cmd([{atom,_,Key}], Gr) -> + switch_cmd(Key, Gr); +switch_cmd([{'?',_}], Gr) -> + switch_cmd(h, Gr); + +switch_cmd(Cmd, Gr) when Cmd =:= c; Cmd =:= i; Cmd =:= k -> + switch_cmd({Cmd, gr_cur_index(Gr)}, Gr); +switch_cmd({c, I}, Gr0) -> + case gr_set_cur(Gr0, I) of + {ok,Gr} -> {ok, Gr}; + undefined -> unknown_group() + end; +switch_cmd({i, I}, Gr) -> case gr_get_num(Gr, I) of {pid,Pid} -> exit(Pid, interrupt), - switch_loop(Iport, Oport, Gr); + {retry, []}; undefined -> - unknown_group(Iport, Oport, Gr) + unknown_group() end; -switch_cmd({ok,[{atom,_,i}],_}, Iport, Oport, Gr) -> - Pid = gr_cur_pid(Gr), - case gr_get_info(Gr, Pid) of - undefined -> - unknown_group(Iport, Oport, Gr); - _ -> - exit(Pid, interrupt), - switch_loop(Iport, Oport, Gr) - end; -switch_cmd({ok,[{atom,_,k},{integer,_,I}],_}, Iport, Oport, Gr) -> +switch_cmd({k, I}, Gr) -> case gr_get_num(Gr, I) of {pid,Pid} -> exit(Pid, die), case gr_get_info(Gr, Pid) of {_Ix,{}} -> % no shell - switch_loop(Iport, Oport, Gr); + retry; _ -> - Gr1 = - receive {'EXIT',Pid,_} -> - gr_del_pid(Gr, Pid) - after 1000 -> - Gr - end, - switch_loop(Iport, Oport, Gr1) + receive {'EXIT',Pid,_} -> + {retry,[],gr_del_pid(Gr, Pid)} + after 1000 -> + {retry,[],Gr} + end end; undefined -> - unknown_group(Iport, Oport, Gr) + unknown_group() end; -switch_cmd({ok,[{atom,_,k}],_}, Iport, Oport, Gr) -> - Pid = gr_cur_pid(Gr), - Info = gr_get_info(Gr, Pid), - case Info of - undefined -> - unknown_group(Iport, Oport, Gr); - {_Ix,{}} -> % no shell - switch_loop(Iport, Oport, Gr); - _ -> - exit(Pid, die), - Gr1 = - receive {'EXIT',Pid,_} -> - gr_del_pid(Gr, Pid) - after 1000 -> - Gr - end, - switch_loop(Iport, Oport, Gr1) - end; -switch_cmd({ok,[{atom,_,j}],_}, Iport, Oport, Gr) -> - io_requests(gr_list(Gr), Iport, Oport), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{atom,_,s},{atom,_,Shell}],_}, Iport, Oport, Gr0) -> +switch_cmd(j, Gr) -> + {retry, gr_list(Gr)}; +switch_cmd({s, Shell}, Gr0) when is_atom(Shell) -> Pid = group:start(self(), {Shell,start,[]}), Gr = gr_add_cur(Gr0, Pid, {Shell,start,[]}), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{atom,_,s}],_}, Iport, Oport, Gr0) -> - Pid = group:start(self(), {shell,start,[]}), - Gr = gr_add_cur(Gr0, Pid, {shell,start,[]}), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{atom,_,r}],_}, Iport, Oport, Gr0) -> + {retry, [], Gr}; +switch_cmd(s, Gr) -> + switch_cmd({s, shell}, Gr); +switch_cmd(r, Gr0) -> case is_alive() of true -> Node = pool:get_node(), - Pid = group:start(self(), {Node,shell,start,[]}), + Pid = group:start(self(), {Node,shell,start,[]}, group_opts(Node)), Gr = gr_add_cur(Gr0, Pid, {Node,shell,start,[]}), - switch_loop(Iport, Oport, Gr); + {retry, [], Gr}; false -> - io_request({put_chars,unicode,"Not alive\n"}, Iport, Oport), - switch_loop(Iport, Oport, Gr0) + {retry, [{put_chars,unicode,<<"Node is not alive\n">>}]} + end; +switch_cmd({r, Node}, Gr) when is_atom(Node)-> + switch_cmd({r, Node, shell}, Gr); +switch_cmd({r,Node,Shell}, Gr0) when is_atom(Node), is_atom(Shell) -> + case is_alive() of + true -> + Pid = group:start(self(), {Node,Shell,start,[]}, group_opts(Node)), + Gr = gr_add_cur(Gr0, Pid, {Node,Shell,start,[]}), + {retry, [], Gr}; + false -> + {retry, [{put_chars,unicode,"Node is not alive\n"}]} end; -switch_cmd({ok,[{atom,_,r},{atom,_,Node}],_}, Iport, Oport, Gr0) -> - Pid = group:start(self(), {Node,shell,start,[]}), - Gr = gr_add_cur(Gr0, Pid, {Node,shell,start,[]}), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{atom,_,r},{atom,_,Node},{atom,_,Shell}],_}, - Iport, Oport, Gr0) -> - Pid = group:start(self(), {Node,Shell,start,[]}), - Gr = gr_add_cur(Gr0, Pid, {Node,Shell,start,[]}), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{atom,_,q}],_}, Iport, Oport, Gr) -> + +switch_cmd(q, _Gr) -> case erlang:system_info(break_ignored) of true -> % noop - io_request({put_chars,unicode,"Unknown command\n"}, Iport, Oport), - switch_loop(Iport, Oport, Gr); + {retry, [{put_chars,unicode,<<"Unknown command\n">>}]}; false -> halt() end; -switch_cmd({ok,[{atom,_,h}],_}, Iport, Oport, Gr) -> - list_commands(Iport, Oport), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[{'?',_}],_}, Iport, Oport, Gr) -> - list_commands(Iport, Oport), - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,[],_}, Iport, Oport, Gr) -> - switch_loop(Iport, Oport, Gr); -switch_cmd({ok,_Ts,_}, Iport, Oport, Gr) -> - io_request({put_chars,unicode,"Unknown command\n"}, Iport, Oport), - switch_loop(Iport, Oport, Gr); -switch_cmd(_Ts, Iport, Oport, Gr) -> - io_request({put_chars,unicode,"Illegal input\n"}, Iport, Oport), - switch_loop(Iport, Oport, Gr). - -unknown_group(Iport, Oport, Gr) -> - io_request({put_chars,unicode,"Unknown job\n"}, Iport, Oport), - switch_loop(Iport, Oport, Gr). - -list_commands(Iport, Oport) -> +switch_cmd(h, _Gr) -> + {retry, list_commands()}; +switch_cmd([], _Gr) -> + {retry,[]}; +switch_cmd(_Ts, _Gr) -> + {retry, [{put_chars,unicode,<<"Unknown command\n">>}]}. + +unknown_group() -> + {retry,[{put_chars,unicode,<<"Unknown job\n">>}]}. + +list_commands() -> QuitReq = case erlang:system_info(break_ignored) of - true -> + true -> []; false -> - [{put_chars, unicode," q - quit erlang\n"}] + [{put_chars, unicode,<<" q - quit erlang\n">>}] end, - io_requests([{put_chars, unicode," c [nn] - connect to job\n"}, - {put_chars, unicode," i [nn] - interrupt job\n"}, - {put_chars, unicode," k [nn] - kill job\n"}, - {put_chars, unicode," j - list all jobs\n"}, - {put_chars, unicode," s [shell] - start local shell\n"}, - {put_chars, unicode," r [node [shell]] - start remote shell\n"}] ++ - QuitReq ++ - [{put_chars, unicode," ? | h - this message\n"}], - Iport, Oport). - -get_line({done,Line,_Rest,Rs}, Iport, Oport) -> - io_requests(Rs, Iport, Oport), - Line; -get_line({undefined,_Char,Cs,Cont,Rs}, Iport, Oport) -> - io_requests(Rs, Iport, Oport), - io_request(beep, Iport, Oport), - get_line(edlin:edit_line(Cs, Cont), Iport, Oport); -get_line({What,Cont0,Rs}, Iport, Oport) -> - io_requests(Rs, Iport, Oport), - receive - {Iport,{data,Cs}} -> - get_line(edlin:edit_line(Cs, Cont0), Iport, Oport); - {Iport,eof} -> - get_line(edlin:edit_line(eof, Cont0), Iport, Oport) - after - get_line_timeout(What) -> - get_line(edlin:edit_line([], Cont0), Iport, Oport) - end. - -get_line_timeout(blink) -> 1000; -get_line_timeout(more_chars) -> infinity. - -% Let driver report window geometry, -% definitely outside of the common interface -get_tty_geometry(Iport) -> - case (catch port_control(Iport,?CTRL_OP_GET_WINSIZE,[])) of - List when length(List) =:= 8 -> - <<W:32/native,H:32/native>> = list_to_binary(List), - {W,H}; - _ -> - error - end. -get_unicode_state(Iport) -> - case (catch port_control(Iport,?CTRL_OP_GET_UNICODE_STATE,[])) of - [Int] when Int > 0 -> - true; - [Int] when Int =:= 0 -> - false; - _ -> - error - end. - -set_unicode_state(Iport, Bool) -> - Data = case Bool of - true -> [1]; - false -> [0] - end, - case (catch port_control(Iport,?CTRL_OP_SET_UNICODE_STATE,Data)) of - [Int] when Int > 0 -> - {unicode, utf8}; - [Int] when Int =:= 0 -> - {unicode, false}; - _ -> - error + [{put_chars, unicode,<<" c [nn] - connect to job\n">>}, + {put_chars, unicode,<<" i [nn] - interrupt job\n">>}, + {put_chars, unicode,<<" k [nn] - kill job\n">>}, + {put_chars, unicode,<<" j - list all jobs\n">>}, + {put_chars, unicode,<<" s [shell] - start local shell\n">>}, + {put_chars, unicode,<<" r [node [shell]] - start remote shell\n">>}] ++ + QuitReq ++ + [{put_chars, unicode,<<" ? | h - this message\n">>}]. + +group_opts(Node) -> + VersionString = erpc:call(Node, erlang, system_info, [otp_release]), + Version = list_to_integer(VersionString), + ExpandFun = + case Version > 25 of + true -> [{expand_fun,fun(B, Opts)-> erpc:call(Node,edlin_expand,expand,[B, Opts]) end}]; + false -> [{expand_fun,fun(B, _)-> erpc:call(Node,edlin_expand,expand,[B]) end}] + end, + group_opts() ++ ExpandFun. +group_opts() -> + [{expand_below, application:get_env(stdlib, shell_expand_location, below) =:= below}]. + +-spec io_request(request(), prim_tty:state()) -> {noreply, prim_tty:state()} | + {term(), reference(), prim_tty:state()}. +io_request({requests,Rs}, TTY) -> + {noreply, io_requests(Rs, TTY)}; +io_request({put_chars, unicode, Chars}, TTY) -> + write(prim_tty:handle_request(TTY, {putc, unicode:characters_to_binary(Chars)})); +io_request({put_chars_sync, unicode, Chars, Reply}, TTY) -> + {Output, NewTTY} = prim_tty:handle_request(TTY, {putc, unicode:characters_to_binary(Chars)}), + {ok, MonitorRef} = prim_tty:write(NewTTY, Output, self()), + {Reply, MonitorRef, NewTTY}; +io_request({put_expand, unicode, Chars}, TTY) -> + write(prim_tty:handle_request(TTY, {expand, unicode:characters_to_binary(Chars)})); +io_request({move_rel, N}, TTY) -> + write(prim_tty:handle_request(TTY, {move, N})); +io_request({insert_chars, unicode, Chars}, TTY) -> + write(prim_tty:handle_request(TTY, {insert, unicode:characters_to_binary(Chars)})); +io_request({delete_chars, N}, TTY) -> + write(prim_tty:handle_request(TTY, {delete, N})); +io_request(clear, TTY) -> + write(prim_tty:handle_request(TTY, clear)); +io_request(beep, TTY) -> + write(prim_tty:handle_request(TTY, beep)). + +write({Output, TTY}) -> + ok = prim_tty:write(TTY, Output), + {noreply, TTY}. + +io_requests([{insert_chars, unicode, C1},{insert_chars, unicode, C2}|Rs], TTY) -> + io_requests([{insert_chars, unicode, [C1,C2]}|Rs], TTY); +io_requests([{put_chars, unicode, C1},{put_chars, unicode, C2}|Rs], TTY) -> + io_requests([{put_chars, unicode, [C1,C2]}|Rs], TTY); +io_requests([R|Rs], TTY) -> + {noreply, NewTTY} = io_request(R, TTY), + io_requests(Rs, NewTTY); +io_requests([], TTY) -> + TTY. + +open_editor(TTY, Buffer) -> + DefaultEditor = + case os:type() of + {win32, _} -> "notepad"; + {unix, _} -> "nano" + end, + Editor = os:getenv("VISUAL", os:getenv("EDITOR", DefaultEditor)), + TmpFile = string:chomp(mktemp()) ++ ".erl", + _ = file:write_file(TmpFile, unicode:characters_to_binary(Buffer, unicode)), + case filelib:is_file(TmpFile) of + true -> + ok = prim_tty:disable_reader(TTY), + try + EditorPort = + case os:type() of + {win32, _} -> + [Cmd | Args] = string:split(Editor," ", all), + open_port({spawn_executable, os:find_executable(Cmd)}, + [{args,Args ++ [TmpFile]}, nouse_stdio]); + {unix, _ } -> + open_port({spawn, Editor ++ " " ++ TmpFile}, [nouse_stdio]) + end, + {EditorPort, TmpFile} + catch error:enoent -> + ok = prim_tty:enable_reader(TTY), + io:format(standard_error, "Could not find EDITOR '~ts'.~n", [Editor]), + false + end; + false -> + io:format(standard_error, + "Could not find create temp file '~ts'.~n", + [TmpFile]), + false end. -%% io_request(Request, InPort, OutPort) -%% io_requests(Requests, InPort, OutPort) -%% Note: InPort is unused. -io_request({requests,Rs}, Iport, Oport) -> - io_requests(Rs, Iport, Oport); -io_request(Request, _Iport, Oport) -> - case io_command(Request) of - {Data, Reply} -> - true = port_command(Oport, Data), - Reply; - unhandled -> - ok +mktemp() -> + case os:type() of + {win32, _} -> + os:cmd("powershell \"write-host (& New-TemporaryFile | Select-Object -ExpandProperty FullName)\""); + {unix,_} -> + os:cmd("mktemp") end. -io_requests([R|Rs], Iport, Oport) -> - io_request(R, Iport, Oport), - io_requests(Rs, Iport, Oport); -io_requests([], _Iport, _Oport) -> - ok. - -put_int16(N, Tail) -> - [(N bsr 8)band 255,N band 255|Tail]. - -%% When a put_chars_sync command is used, user_drv guarantees that -%% the bytes have been put in the buffer of the port before an acknowledgement -%% is sent back to the process sending the request. This command was added in -%% OTP 18 to make sure that data sent from io:format is actually printed -%% to the console before the vm stops when calling erlang:halt(integer()). --dialyzer({no_improper_lists, io_command/1}). -io_command({put_chars_sync, unicode,Cs,Reply}) -> - {[?OP_PUTC_SYNC|unicode:characters_to_binary(Cs,utf8)], Reply}; -io_command({put_chars, unicode,Cs}) -> - {[?OP_PUTC|unicode:characters_to_binary(Cs,utf8)], ok}; -io_command({move_rel,N}) -> - {[?OP_MOVE|put_int16(N, [])], ok}; -io_command({insert_chars,unicode,Cs}) -> - {[?OP_INSC|unicode:characters_to_binary(Cs,utf8)], ok}; -io_command({delete_chars,N}) -> - {[?OP_DELC|put_int16(N, [])], ok}; -io_command(beep) -> - {[?OP_BEEP], ok}; -io_command(_) -> - unhandled. +handle_req(next, TTYState, {false, IOQ} = IOQueue) -> + case queue:out(IOQ) of + {empty, _} -> + {TTYState, IOQueue}; + {{value, {Origin, Req}}, ExecQ} -> + case io_request(Req, TTYState) of + {noreply, NewTTYState} -> + handle_req(next, NewTTYState, {false, ExecQ}); + {Reply, MonitorRef, NewTTYState} -> + {NewTTYState, {{Origin, MonitorRef, Reply}, ExecQ}} + end + end; +handle_req(Msg, TTYState, {false, IOQ} = IOQueue) -> + empty = queue:peek(IOQ), + {Origin, Req} = Msg, + case io_request(Req, TTYState) of + {noreply, NewTTYState} -> + {NewTTYState, IOQueue}; + {Reply, MonitorRef, NewTTYState} -> + {NewTTYState, {{Origin, MonitorRef, Reply}, IOQ}} + end; +handle_req(Msg,TTYState,{Resp, IOQ}) -> + %% All requests are queued when we have outstanding sync put_chars + {TTYState, {Resp, queue:in(Msg,IOQ)}}. %% gr_new() %% gr_get_num(Group, Index) @@ -611,100 +869,71 @@ io_command(_) -> %% gr_add_cur(Group, Pid, Shell) %% gr_set_cur(Group, Index) %% gr_cur_pid(Group) +%% gr_cur_index(Group) %% gr_del_pid(Group, Pid) %% Manage the group list. The group structure has the form: %% {NextIndex,CurrIndex,CurrPid,GroupList} %% %% where each element in the group list is: %% {Index,GroupPid,Shell} - +-record(group, { index, pid, shell }). +-record(gr, { next = 0, current = 0, pid = none, groups = []}). gr_new() -> - {0,0,none,[]}. - -gr_get_num({_Next,_CurI,_CurP,Gs}, I) -> - gr_get_num1(Gs, I). - -gr_get_num1([{I,_Pid,{}}|_Gs], I) -> - undefined; -gr_get_num1([{I,Pid,_S}|_Gs], I) -> - {pid,Pid}; -gr_get_num1([_G|Gs], I) -> - gr_get_num1(Gs, I); -gr_get_num1([], _I) -> - undefined. - -gr_get_info({_Next,_CurI,_CurP,Gs}, Pid) -> - gr_get_info1(Gs, Pid). - -gr_get_info1([{I,Pid,S}|_Gs], Pid) -> - {I,S}; -gr_get_info1([_G|Gs], I) -> - gr_get_info1(Gs, I); -gr_get_info1([], _I) -> - undefined. - -gr_add_cur({Next,_CurI,_CurP,Gs}, Pid, Shell) -> - {Next+1,Next,Pid,append(Gs, [{Next,Pid,Shell}])}. - -gr_set_cur({Next,_CurI,_CurP,Gs}, I) -> - case gr_get_num1(Gs, I) of - {pid,Pid} -> {ok,{Next,I,Pid,Gs}}; + #gr{}. +gr_new_group(I, P, S) -> + #group{ index = I, pid = P, shell = S }. + +gr_get_num(#gr{ groups = Gs }, I) -> + case lists:keyfind(I, #group.index, Gs) of + false -> undefined; + #group{ shell = {} } -> + undefined; + #group{ pid = Pid } -> + {pid, Pid} + end. + +gr_get_info(#gr{ groups = Gs }, Pid) -> + case lists:keyfind(Pid, #group.pid, Gs) of + false -> undefined; + #group{ index = I, shell = S } -> + {I, S} + end. + +gr_add_cur(#gr{ next = Next, groups = Gs}, Pid, Shell) -> + put(current_group, Pid), + #gr{ next = Next + 1, current = Next, pid = Pid, + groups = Gs ++ [gr_new_group(Next, Pid, Shell)] + }. + +gr_set_cur(Gr, I) -> + case gr_get_num(Gr, I) of + {pid,Pid} -> + put(current_group, Pid), + {ok, Gr#gr{ current = I, pid = Pid }}; undefined -> undefined end. -gr_set_num({Next,CurI,CurP,Gs}, I, Pid, Shell) -> - {Next,CurI,CurP,gr_set_num1(Gs, I, Pid, Shell)}. - -gr_set_num1([{I,_Pid,_Shell}|Gs], I, NewPid, NewShell) -> - [{I,NewPid,NewShell}|Gs]; -gr_set_num1([{I,Pid,Shell}|Gs], NewI, NewPid, NewShell) when NewI > I -> - [{I,Pid,Shell}|gr_set_num1(Gs, NewI, NewPid, NewShell)]; -gr_set_num1(Gs, NewI, NewPid, NewShell) -> - [{NewI,NewPid,NewShell}|Gs]. - -gr_del_pid({Next,CurI,CurP,Gs}, Pid) -> - {Next,CurI,CurP,gr_del_pid1(Gs, Pid)}. - -gr_del_pid1([{_I,Pid,_S}|Gs], Pid) -> - Gs; -gr_del_pid1([G|Gs], Pid) -> - [G|gr_del_pid1(Gs, Pid)]; -gr_del_pid1([], _Pid) -> - []. - -gr_cur_pid({_Next,_CurI,CurP,_Gs}) -> - CurP. - -gr_list({_Next,CurI,_CurP,Gs}) -> - gr_list(Gs, CurI, []). - -gr_list([{_I,_Pid,{}}|Gs], Cur, Jobs) -> - gr_list(Gs, Cur, Jobs); -gr_list([{Cur,_Pid,Shell}|Gs], Cur, Jobs) -> - gr_list(Gs, Cur, [{put_chars, unicode,flatten(io_lib:format("~4w* ~w\n", [Cur,Shell]))}|Jobs]); -gr_list([{I,_Pid,Shell}|Gs], Cur, Jobs) -> - gr_list(Gs, Cur, [{put_chars, unicode,flatten(io_lib:format("~4w ~w\n", [I,Shell]))}|Jobs]); -gr_list([], _Cur, Jobs) -> - lists:reverse(Jobs). - -append([H|T], X) -> - [H|append(T, X)]; -append([], X) -> - X. - -member(X, [X|_Rest]) -> true; -member(X, [_H|Rest]) -> - member(X, Rest); -member(_X, []) -> false. - -flatten(List) -> - flatten(List, [], []). - -flatten([H|T], Cont, Tail) when is_list(H) -> - flatten(H, [T|Cont], Tail); -flatten([H|T], Cont, Tail) -> - [H|flatten(T, Cont, Tail)]; -flatten([], [H|Cont], Tail) -> - flatten(H, Cont, Tail); -flatten([], [], Tail) -> - Tail. +gr_set_num(Gr = #gr{ groups = Groups }, I, Pid, Shell) -> + NewGroups = lists:keystore(I, #group.index, Groups, gr_new_group(I,Pid,Shell)), + Gr#gr{ groups = NewGroups }. + + +gr_del_pid(Gr = #gr{ groups = Groups }, Pid) -> + Gr#gr{ groups = lists:keydelete(Pid, #group.pid, Groups) }. + + +gr_cur_pid(#gr{ pid = Pid }) -> + Pid. +gr_cur_index(#gr{ current = Index }) -> + Index. + +gr_list(#gr{ current = Current, groups = Groups}) -> + lists:flatmap( + fun(#group{ shell = {} }) -> + []; + (#group{ index = I, shell = S }) -> + Marker = ["*" || Current =:= I], + [{put_chars, unicode, + unicode:characters_to_binary( + io_lib:format("~4w~.1ts ~w\n", [I,Marker,S]))}] + end, Groups). diff --git a/lib/kernel/src/user_sup.erl b/lib/kernel/src/user_sup.erl index c1fb1b1a48..c8ebf2f7cd 100644 --- a/lib/kernel/src/user_sup.erl +++ b/lib/kernel/src/user_sup.erl @@ -39,11 +39,13 @@ start() -> -spec init([]) -> 'ignore' | {'error', 'nouser'} | {'ok', pid(), pid()}. init([]) -> - case get_user() of + init(init:get_arguments()); +init(Flags) -> + case get_user(Flags) of nouser -> ignore; {master, Master} -> - Pid = start_slave(Master), + Pid = start_relay(Master), {ok, Pid, Pid}; {M, F, A} -> case start_user(M, F, A) of @@ -54,7 +56,7 @@ init([]) -> end end. -start_slave(Master) -> +start_relay(Master) -> case rpc:call(Master, erlang, whereis, [user]) of User when is_pid(User) -> spawn(?MODULE, relay, [User]); @@ -112,19 +114,23 @@ wait_for_user_p(N) -> wait_for_user_p(N-1) end. -get_user() -> - Flags = init:get_arguments(), - check_flags(Flags, {user_drv, start, []}). +get_user(Flags) -> + check_flags(Flags, lists:keymember(detached, 1, Flags), {user_drv, start, []}). %% These flags depend upon what arguments the erl script passes on %% to erl91. -check_flags([{nouser, []} |T], _) -> check_flags(T, nouser); -check_flags([{user, [User]} | T], _) -> - check_flags(T, {list_to_atom(User), start, []}); -check_flags([{noshell, []} | T], _) -> check_flags(T, {user, start, []}); -check_flags([{oldshell, []} | T], _) -> check_flags(T, {user, start, []}); -check_flags([{noinput, []} | T], _) -> check_flags(T, {user, start_out, []}); -check_flags([{master, [Node]} | T], _) -> - check_flags(T, {master, list_to_atom(Node)}); -check_flags([_H | T], User) -> check_flags(T, User); -check_flags([], User) -> User. +check_flags([{nouser, []} |T], Attached, _) -> check_flags(T, Attached, nouser); +check_flags([{user, [User]} | T], Attached, _) -> + check_flags(T, Attached, {list_to_atom(User), start, []}); +check_flags([{noshell, []} | T], Attached, _) -> + check_flags(T, Attached, {user_drv, start, [#{ initial_shell => noshell }]}); +check_flags([{oldshell, []} | T], false, _) -> + %% When running in detached mode, we ignore any -oldshell flags as we do not + %% want input => true to be set as they may halt the node (on bsd) + check_flags(T, false, {user_drv, start, [#{ initial_shell => oldshell }]}); +check_flags([{noinput, []} | T], Attached, _) -> + check_flags(T, Attached, {user_drv, start, [#{ initial_shell => noshell, input => false }]}); +check_flags([{master, [Node]} | T], Attached, _) -> + check_flags(T, Attached, {master, list_to_atom(Node)}); +check_flags([_H | T], Attached, User) -> check_flags(T, Attached, User); +check_flags([], _Attached, User) -> User. diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index f7776f44fe..0d88e3555e 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -106,6 +106,7 @@ MODULES= \ net_SUITE \ os_SUITE \ pg_SUITE \ + rtnode \ seq_trace_SUITE \ $(SOCKET_MODULES) \ wrap_log_reader_SUITE \ @@ -217,6 +218,7 @@ release_tests_spec: make_emakefile $(EMAKEFILE) $(COVERFILE) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + $(INSTALL_DIR) "$(RELSYSDIR)/kernel_SUITE_data" $(INSTALL_DATA) $(ERL_TOP)/make/otp_version_tickets "$(RELSYSDIR)/kernel_SUITE_data" release_docs_spec: diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 0668c28692..ae69d9874f 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -35,7 +35,7 @@ permit_false_start_local/1, permit_false_start_dist/1, script_start/1, nodedown_start/1, init2973/0, loop2973/0, loop5606/1, otp_16504/1]). --export([config_change/1, persistent_env/1, +-export([config_change/1, persistent_env/1, invalid_app_file/1, distr_changed_tc1/1, distr_changed_tc2/1, ensure_started/1, ensure_all_started/1, shutdown_func/1, do_shutdown/1, shutdown_timeout/1, shutdown_deadlock/1, @@ -62,7 +62,7 @@ all() -> {group, distr_changed}, config_change, shutdown_func, shutdown_timeout, shutdown_deadlock, config_relative_paths, optional_applications, persistent_env, handle_many_config_files, format_log_1, format_log_2, - configfd_bash, configfd_port_program]. + configfd_bash, configfd_port_program, invalid_app_file]. groups() -> [{reported_bugs, [], @@ -2190,17 +2190,16 @@ do_configfd_test_bash() -> case application:start(os_mon) of ok -> case total_memory() of Memory when is_integer(Memory), - Memory > 16 -> + Memory > 8 -> application:stop(os_mon), - true = - ("magic42" =/= - RunInBash( - "erl " - "-noshell " - "-configfd 3 " - "-eval " - "'io:format(\"magic42\"),erlang:halt()' " - "3< <(erl -noshell -eval '(fun W(D) -> io:put_chars(D), W([D,D]) end)(<<\"00000000000000000\">>)') ")); + Res = RunInBash( + "erl " + "-noshell " + "-configfd 3 " + "-eval " + "'io:format(\"magic42\"),erlang:halt()' " + "3< <(erl -noshell -eval '(fun W(D) -> io:put_chars(D), W([D,<<\"00000000000000000\">>]) end)([])') "), + {match, _} = re:run(Res,"Max size 134217728 bytes exceeded"); _ -> io:format("Skipped huge file check to avoid flaky test on machine with less than 8GB of memory") end; @@ -2454,6 +2453,19 @@ persistent_env(Conf) when is_list(Conf) -> %% Clean up ok = application:unload(appinc). +%% Test that application app file error handling works as it should +invalid_app_file(_Config) -> + + {error,{bad_application,{application,"name",[]}}} + = application:load({application, "name",[]}), + {error,{invalid_options,#{}}} + = application:load({application, name,#{}}), + {error, {invalid_options,_}} = + application:load({application,name,[{env,[{"key",value}]}]}), + {error, {invalid_options,_}} = + application:load({application,name,[{env,[key]}]}), + {error, {invalid_options,_}} = + application:load({application,name,[{env,[{key,value},{key,value}]}]}). %% Test more than one config file defined by one -config parameter: handle_many_config_files(Conf) when is_list(Conf) -> diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl index 7270e20168..1dc233f253 100644 --- a/lib/kernel/test/erl_distribution_wb_SUITE.erl +++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl @@ -56,7 +56,9 @@ -define(DFLAG_MAP_TAG, 16#20000). -define(DFLAG_BIG_CREATION, 16#40000). -define(DFLAG_HANDSHAKE_23, 16#1000000). +-define(DFLAG_UNLINK_ID, 16#2000000). -define(DFLAG_MANDATORY_25_DIGEST, 16#4000000). +-define(DFLAG_V4_NC, 16#400000000). %% From OTP R9 extended references are compulsory. %% From OTP R10 extended pids and ports are compulsory. @@ -64,7 +66,8 @@ %% From OTP 21 NEW_FUN_TAGS is compulsory (no more tuple fallback {fun, ...}). %% From OTP 23 BIG_CREATION is compulsory. %% From OTP 25 NEW_FLOATS, MAP_TAG, EXPORT_PTR_TAG, and BIT_BINARIES are compulsory. --define(COMPULSORY_DFLAGS, + +-define(DFLAGS_MANDATORY_25, (?DFLAG_EXTENDED_REFERENCES bor ?DFLAG_FUN_TAGS bor ?DFLAG_EXTENDED_PIDS_PORTS bor @@ -74,7 +77,18 @@ ?DFLAG_NEW_FLOATS bor ?DFLAG_MAP_TAG bor ?DFLAG_EXPORT_PTR_TAG bor - ?DFLAG_BIT_BINARIES)). + ?DFLAG_BIT_BINARIES bor + ?DFLAG_HANDSHAKE_23)). + +%% From OTP 26 V4_NC and UNLINK_ID are compulsory. + +-define(DFLAGS_MANDATORY_26, + (?DFLAG_V4_NC bor + ?DFLAG_UNLINK_ID)). + +-define(COMPULSORY_DFLAGS, + (?DFLAGS_MANDATORY_25 bor + ?DFLAGS_MANDATORY_26)). -define(PASS_THROUGH, $p). @@ -398,7 +412,8 @@ dflag_mandatory_25(_Config) -> PortNo, [{active,false},{packet,2}]), OtherNode = list_to_atom(?CT_PEER_NAME()++"@"++atom_to_list(NB)), - send_name(SocketA, OtherNode, ?DFLAG_MANDATORY_25_DIGEST), + send_name(SocketA, OtherNode, + ?DFLAG_MANDATORY_25_DIGEST bor ?DFLAGS_MANDATORY_26), ok = recv_status(SocketA), gen_tcp:close(SocketA), peer:stop(Peer), diff --git a/lib/kernel/test/init_SUITE.erl b/lib/kernel/test/init_SUITE.erl index bc3882b862..ea75f040f2 100644 --- a/lib/kernel/test/init_SUITE.erl +++ b/lib/kernel/test/init_SUITE.erl @@ -29,7 +29,7 @@ many_restarts/0, many_restarts/1, restart_with_mode/1, get_plain_arguments/1, reboot/1, stop_status/1, stop/1, get_status/1, script_id/1, - dot_erlang/1, + dot_erlang/1, unknown_module/1, find_system_processes/0]). -export([boot1/1, boot2/1]). @@ -48,7 +48,7 @@ all() -> [get_arguments, get_argument, boot_var, many_restarts, restart_with_mode, get_plain_arguments, restart, stop_status, get_status, script_id, - dot_erlang, {group, boot}]. + dot_erlang, unknown_module, {group, boot}]. groups() -> [{boot, [], [boot1, boot2]}]. @@ -689,6 +689,21 @@ dot_erlang(Config) -> ok. +unknown_module(Config) when is_list(Config) -> + Port = open_port({spawn, "erl -s unknown_module"}, + [exit_status, use_stdio, stderr_to_stdout]), + Error = "Error! Failed to load module 'unknown_module' because it cannot be found.", + [_ | _] = string:find(collect_until_exit_one(Port), Error), + ok. + +collect_until_exit_one(Port) -> + receive + {Port, {data, Msg}} -> Msg ++ collect_until_exit_one(Port); + {Port, {exit_status, 1}} -> [] + after + 30_000 -> ct:fail(erl_timeout) + end. + %% ------------------------------------------------ %% Start the slave system with -boot flag. %% ------------------------------------------------ diff --git a/lib/kernel/test/interactive_shell_SUITE.erl b/lib/kernel/test/interactive_shell_SUITE.erl index ec3887df1d..fe1060ee8f 100644 --- a/lib/kernel/test/interactive_shell_SUITE.erl +++ b/lib/kernel/test/interactive_shell_SUITE.erl @@ -19,6 +19,21 @@ %% -module(interactive_shell_SUITE). -include_lib("kernel/include/file.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%% Things to add tests for: +%% - TERM=dumb +%% - Editing line > MAXSIZE (1 << 16) +%% - \t tests (use io:format("\t")) +%% - xn fix after Delete and Backspace +%% - octal_to_hex > 255 length (is this possible?) +%% 1222 0 : } else if (lastput == 0) { /* A multibyte UTF8 character */ +%% 1223 0 : for (i = 0; i < ubytes; ++i) { +%% 1224 0 : outc(ubuf[i]); +%% 1225 : } +%% 1226 : } else { +%% 1227 0 : outc(lastput); +%% - $TERM set to > 1024 long value -export([all/0, suite/0, groups/0, init_per_suite/1, end_per_suite/1, init_per_group/2, end_per_group/2, @@ -31,27 +46,45 @@ shell_history_custom/1, shell_history_custom_errors/1, job_control_remote_noshell/1,ctrl_keys/1, get_columns_and_rows_escript/1, - remsh_basic/1, remsh_longnames/1, remsh_no_epmd/1]). + shell_navigation/1, shell_xnfix/1, shell_delete/1, + shell_transpose/1, shell_search/1, shell_insert/1, + shell_update_window/1, shell_huge_input/1, + shell_invalid_unicode/1, shell_support_ansi_input/1, + shell_invalid_ansi/1, shell_suspend/1, shell_full_queue/1, + shell_unicode_wrap/1, shell_delete_unicode_wrap/1, + shell_delete_unicode_not_at_cursor_wrap/1, + shell_expand_location_above/1, + shell_expand_location_below/1, + shell_update_window_unicode_wrap/1, + shell_standard_error_nlcr/1, shell_clear/1, + remsh_basic/1, remsh_error/1, remsh_longnames/1, remsh_no_epmd/1, + remsh_expand_compatibility_25/1, remsh_expand_compatibility_later_version/1, + external_editor/1, external_editor_visual/1, + external_editor_unicode/1, shell_ignore_pager_commands/1]). -%% For spawn --export([toerl_server/3]). %% Exports for custom shell history module -export([load/0, add/1]). +%% For custom prompt testing +-export([prompt/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,3}}]. all() -> - [get_columns_and_rows_escript,get_columns_and_rows, - exit_initial, job_control_local, - job_control_remote, job_control_remote_noshell, - ctrl_keys, stop_during_init, wrap, - {group, shell_history}, - {group, remsh}]. + [{group, to_erl}, + {group, tty}]. groups() -> - [{shell_history, [], + [{to_erl,[], + [get_columns_and_rows_escript,get_columns_and_rows, + exit_initial, job_control_local, + job_control_remote, job_control_remote_noshell, + ctrl_keys, stop_during_init, wrap, + shell_invalid_ansi, + {group, shell_history}, + {group, remsh}]}, + {shell_history, [], [shell_history, shell_history_resize, shell_history_eaccess, @@ -65,26 +98,60 @@ groups() -> shell_history_custom_errors]}, {remsh, [], [remsh_basic, + remsh_error, remsh_longnames, - remsh_no_epmd]} + remsh_no_epmd, + remsh_expand_compatibility_25, + remsh_expand_compatibility_later_version]}, + {tty,[], + [{group,tty_unicode}, + {group,tty_latin1}, + shell_suspend, + shell_full_queue, + external_editor, + external_editor_visual, + shell_ignore_pager_commands + ]}, + {tty_unicode,[parallel], + [{group,tty_tests}, + shell_invalid_unicode, + external_editor_unicode + %% unicode wrapping does not work right yet + %% shell_unicode_wrap, + %% shell_delete_unicode_wrap, + %% shell_delete_unicode_not_at_cursor_wrap, + %% shell_update_window_unicode_wrap + ]}, + {tty_latin1,[],[{group,tty_tests}]}, + {tty_tests, [parallel], + [shell_navigation, shell_xnfix, shell_delete, + shell_transpose, shell_search, shell_insert, + shell_update_window, shell_huge_input, + shell_support_ansi_input, + shell_standard_error_nlcr, + shell_expand_location_above, + shell_expand_location_below, + shell_clear]} ]. init_per_suite(Config) -> - case get_progs() of - {error, Error} -> - {skip, Error}; - _ -> - Term = os:getenv("TERM", "dumb"), - os:putenv("TERM", "vt100"), - DefShell = get_default_shell(), - [{default_shell,DefShell},{term,Term}|Config] - end. + Term = os:getenv("TERM", "dumb"), + os:putenv("TERM", "vt100"), + [{term,Term}|Config]. end_per_suite(Config) -> Term = proplists:get_value(term,Config), os:putenv("TERM",Term), ok. +init_per_group(to_erl, Config) -> + case rtnode:get_progs() of + {error, Error} -> + {skip, Error}; + _ -> + DefShell = rtnode:get_default_shell(), + [{default_shell,DefShell}|Config] + end; init_per_group(remsh, Config) -> case proplists:get_value(default_shell, Config) of old -> {skip, "Not supported in old shell"}; @@ -95,13 +162,37 @@ init_per_group(shell_history, Config) -> old -> {skip, "Not supported in old shell"}; new -> Config end; +init_per_group(tty, Config) -> + case string:split(tmux("-V")," ") of + ["tmux",[Num,$.|_]] when Num >= $3, Num =< $9 -> + tmux("kill-session"), + "" = tmux("-u new-session -x 50 -y 60 -d"), + ["" = tmux(["set-environment '",Name,"' '",Value,"'"]) + || {Name,Value} <- os:env()], + Config; + ["tmux", Vsn] -> + {skip, "invalid tmux version " ++ Vsn ++ ". Need vsn 3 or later"}; + Error -> + {skip, "tmux not installed " ++ Error} + end; +init_per_group(Group, Config) when Group =:= tty_unicode; + Group =:= tty_latin1 -> + [Lang,_] = + string:split( + os:getenv("LC_ALL", + os:getenv("LC_CTYPE", + os:getenv("LANG","en_US.UTF-8"))),"."), + case Group of + tty_unicode -> + [{encoding, unicode},{env,[{"LC_ALL",Lang++".UTF-8"}]}|Config]; + tty_latin1 -> + % [{encoding, latin1},{env,[{"LC_ALL",Lang++".ISO-8859-1"}]}|Config], + {skip, "latin1 tests not implemented yet"} + end; init_per_group(sh_custom, Config) -> - %% Ensure that ERL_AFLAGS will not override the value of the - %% shell_history variable. - Name = interactive_shell_sh_custom, - Args = "-noshell -kernel shell_history not_overridden", - {ok, Node} = test_server:start_node(Name, slave, [{args,Args}]), - try erpc:call(Node, application, get_env, [kernel, shell_history], timeout(normal)) of + %% Ensure that ERL_AFLAGS will not override the value of the shell_history variable. + {ok, Peer, Node} = ?CT_PEER(["-noshell","-kernel","shell_history","not_overridden"]), + try erpc:call(Node, application, get_env, [kernel, shell_history], rtnode:timeout(normal)) of {ok, not_overridden} -> Config; _ -> @@ -112,23 +203,45 @@ init_per_group(sh_custom, Config) -> io:format("~p\n~p\n~p\n", [C,R,Stk]), {skip, "Unexpected error"} after - test_server:stop_node(Node) + peer:stop(Peer) end; init_per_group(_GroupName, Config) -> Config. +end_per_group(tty, _Config) -> + Windows = string:split(tmux("list-windows"), "\n", all), + lists:foreach( + fun(W) -> + case string:split(W, " ", all) of + ["0:" | _] -> ok; + [No, _Name | _] -> + "" = os:cmd(["tmux select-window -t ", string:split(No,":")]), + ct:log("~ts~n~ts",[W, os:cmd(lists:concat(["tmux capture-pane -p -e"]))]) + end + end, Windows), +% "" = os:cmd("tmux kill-session") + ok; end_per_group(_GroupName, Config) -> Config. -init_per_testcase(_Func, Config) -> - Config. - -end_per_testcase(_Case, _Config) -> - %% Terminate any connected nodes. They may disturb test cases that follow. - lists:foreach(fun(Node) -> - catch erpc:call(Node, erlang, halt, []) - end, nodes()), - ok. +init_per_testcase(Func, Config) -> + Path = [Func, + [proplists:get_value(name,P) || + P <- [proplists:get_value(tc_group_properties,Config,[])] ++ + proplists:get_value(tc_group_path,Config,[])]], + [{tc_path, lists:concat(lists:join("-",lists:flatten(Path)))} | Config]. + +end_per_testcase(_Case, Config) -> + GroupProperties = proplists:get_value(tc_group_properties, Config), + case (GroupProperties =/= false) andalso proplists:get_value(parallel, GroupProperties, false) of + true -> ok; + false -> + %% Terminate any connected nodes. They may disturb test cases that follow. + lists:foreach(fun(Node) -> + catch erpc:call(Node, erlang, halt, []) + end, nodes()), + ok + end. %%-define(DEBUG,1). -ifdef(DEBUG). @@ -155,7 +268,7 @@ run_unbuffer_escript(Rows, Columns, EScript, NoTermStdIn, NoTermStdOut) -> {true, true} -> io_lib:format(" > ~s < ~s ; cat ~s", [TmpFile, TmpFile, TmpFile]) end, Command = io_lib:format("unbuffer -p bash -c \"stty rows ~p; stty columns ~p; escript ~s ~s\"", - [Rows, Columns, EScript, CommandModifier]), + [Rows, Columns, EScript, CommandModifier]), %% io:format("Command: ~s ~n", [Command]), Out = os:cmd(Command), %% io:format("Out: ~p ~n", [Out]), @@ -203,50 +316,1294 @@ get_columns_and_rows(Config) when is_list(Config) -> ok. test_columns_and_rows(old, Args) -> - rtnode([{putline, ""}, - {putline, "2."}, - {expect, "2\r\n"}, - {putline, "io:columns()."}, - {expect, "{error,enotsup}\r\n"}, - {putline, "io:rows()."}, - {expect, "{error,enotsup}\r\n"} - ], [], [], Args), - - rtnode([{putline, ""}, - {putline, "2."}, - {expect, "2\r\n"}, - {putline, "io:columns()."}, - {expect, "{ok,90}\r\n"}, - {putline,"io:rows()."}, - {expect, "{ok,40}\r\n"}], - [], - "stty rows 40; stty columns 90; ", - Args); + rtnode:run( + [{putline, ""}, + {putline, "2."}, + {expect, "2\r\n"}, + {putline, "io:columns()."}, + {expect, "{error,enotsup}\r\n"}, + {putline, "io:rows()."}, + {expect, "{error,enotsup}\r\n"} + ], [], [], Args), + + rtnode:run( + [{putline, ""}, + {putline, "2."}, + {expect, "2\r\n"}, + {putline, "io:columns()."}, + {expect, "{ok,90}\r\n"}, + {putline,"io:rows()."}, + {expect, "{ok,40}\r\n"}], + [], + "stty rows 40; stty columns 90; ", + Args); test_columns_and_rows(new, _Args) -> - rtnode([{putline, ""}, - {expect, "1> $"}, - {putline, "2."}, - {expect, "\r\n2\r\n"}, - {expect, "> $"}, - {putline, "io:columns()."}, - {expect, "{ok,80}\r\n"}, - {expect, "> $"}, - {putline, "io:rows()."}, - {expect, "\r\n{ok,24}\r\n"} - ]), - - rtnode([{putline, ""}, - {expect, "1> $"}, - {putline, "2."}, - {expect, "\r\n2\r\n"}, - {expect, "> $"}, - {putline, "io:columns()."}, - {expect, "\r\n{ok,90}\r\n"}, - {expect, "> $"}, - {putline, "io:rows()."}, - {expect, "\r\n{ok,40}\r\n"}], - [], - "stty rows 40; stty columns 90; "). + rtnode:run( + [{putline, ""}, + {expect, "1> $"}, + {putline, "2."}, + {expect, "\r\n2\r\n"}, + {expect, "> $"}, + {putline, "io:columns()."}, + {expect, "{ok,80}\r\n"}, + {expect, "> $"}, + {putline, "io:rows()."}, + {expect, "\r\n{ok,24}\r\n"} + ]), + + rtnode:run( + [{putline, ""}, + {expect, "1> $"}, + {putline, "2."}, + {expect, "\r\n2\r\n"}, + {expect, "> $"}, + {putline, "io:columns()."}, + {expect, "\r\n{ok,90}\r\n"}, + {expect, "> $"}, + {putline, "io:rows()."}, + {expect, "\r\n{ok,40}\r\n"}], + [], + "stty rows 40; stty columns 90; "). + +shell_navigation(Config) -> + + Term = start_tty(Config), + + try + [begin + send_tty(Term,"{aaa,'b"++U++"b',ccc}"), + check_location(Term, {0, 0}), %% Check that cursor jump backward + check_content(Term, "{aaa,'b"++U++"b',ccc}$"), + timer:sleep(1000), %% Wait for cursor to jump back + check_location(Term, {0, width("{aaa,'b"++U++"b',ccc}")}), + send_tty(Term,"Home"), + check_location(Term, {0, 0}), + send_tty(Term,"End"), + check_location(Term, {0, width("{aaa,'b"++U++"b',ccc}")}), + send_tty(Term,"Left"), + check_location(Term, {0, width("{aaa,'b"++U++"b',ccc")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, width("{aaa,'b"++U++"b',")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, width("{aaa,")}), + send_tty(Term,"C-Right"), + check_location(Term, {0, width("{aaa,'b"++U++"b'")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, width("{aaa,")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, width("{")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, 0}), + send_tty(Term,"C-E"), + check_location(Term, {0, width("{aaa,'b"++U++"b',ccc}")}), + send_tty(Term,"C-A"), + check_location(Term, {0, 0}), + send_tty(Term,"Enter") + end || U <- hard_unicode()], + ok + after + stop_tty(Term) + end. + +shell_clear(Config) -> + + Term = start_tty(Config), + {Rows, _Cols} = get_window_size(Term), + + try + send_tty(Term,"foobar."), + send_tty(Term,"Enter"), + check_content(Term, "foobar"), + check_location(Term, {0, 0}), + send_tty(Term,"bazbat"), + check_location(Term, {0, 6}), + send_tty(Term,"C-L"), + check_location(Term, {-Rows+1, 6}), + check_content(Term, "bazbat") + after + stop_tty(Term) + end. + +shell_xnfix(Config) -> + + Term = start_tty(Config), + + {_Rows, Cols} = get_window_size(Term), + {_Row, Col} = get_location(Term), + + As = lists:duplicate(Cols - Col - 1,"a"), + + try + [begin + check_location(Term, {0, 0}), + send_tty(Term,As), + check_content(Term,[As,$$]), + check_location(Term, {0, Cols - Col - 1}), + send_tty(Term,"a"), + check_location(Term, {0, -Col}), + send_tty(Term,"aaa"), + check_location(Term, {0, -Col + 3}), + [send_tty(Term,"Left") || _ <- lists:seq(1,3 + width(U))], + send_tty(Term,U), + %% a{Cols-1}U\naaaaa + check_content(Term,[lists:duplicate(Cols - Col - 1 - width(U),$a), + U,"\n",lists:duplicate(3+width(U), $a),"$"]), + check_location(Term, {0, -Col}), + send_tty(Term,"Left"), + send_tty(Term,U), + %% a{Cols-1}U\nUaaaaa + check_content(Term,[lists:duplicate(Cols - Col - 1 - width(U),$a), + U,"\n",U,lists:duplicate(3+width(U), $a),"$"]), + check_location(Term, {0, -Col}), + %% send_tty(Term,"Left"), + %% send_tty(Term,"BSpace"), + %% a{Cols-2}U\nUaaaaa + %% check_content(Term,[lists:duplicate(Cols - Col - 2 - width(U),$a), + %% U,"\n",U,lists:duplicate(3+width(U), $a),"$"]), + %% send_tty(Term,"BSpace"), + %% check_content(Term,[lists:duplicate(Cols - Col - 1 - width(U),$a), + %% U,U,"\n",lists:duplicate(3+width(U), $a),"$"]), + %% send_tty(Term,"aa"), + %% check_content(Term,[lists:duplicate(Cols - Col - 2 - width(U),$a), + %% U,"a\n",U,lists:duplicate(3+width(U), $a),"$"]), + %% check_location(Term, {0, -Col}), + send_tty(Term,"C-K"), + check_location(Term, {0, -Col}), + send_tty(Term,"C-A"), + check_location(Term, {-1, 0}), + send_tty(Term,"C-E"), + check_location(Term, {0, -Col}), + send_tty(Term,"Enter"), + ok + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + + +%% Characters that are larger than 2 wide need special handling when they +%% are at the end of the current line. +shell_unicode_wrap(Config) -> + + Term = start_tty(Config), + + {_Rows, Cols} = get_window_size(Term), + {_Row, Col} = get_location(Term), + + try + [begin + FirstLine = [U,lists:duplicate(Cols - Col - width(U)*2 + 1,"a")], + OtherLineA = [U,lists:duplicate(Cols - width(U) * 2+1,"a")], + OtherLineB = [U,lists:duplicate(Cols - width(U) * 2+1,"b")], + OtherLineC = [U,lists:duplicate(Cols - width(U) * 2+1,"c")], + OtherLineD = [U,lists:duplicate(Cols - width(U) * 2+1,"d")], + send_tty(Term,FirstLine), + check_content(Term, [FirstLine,$$]), + check_location(Term, {0, Cols - Col - width(U)+1}), + + send_tty(Term,OtherLineA), + check_content(Term, [OtherLineA,$$]), + check_location(Term, {0, Cols - Col - width(U)+1}), + + send_tty(Term,OtherLineB), + check_content(Term, [OtherLineB,$$]), + check_location(Term, {0, Cols - Col - width(U)+1}), + + send_tty(Term,OtherLineC), + check_content(Term, [OtherLineC,$$]), + check_location(Term, {0, Cols - Col - width(U)+1}), + + send_tty(Term,OtherLineD), + check_content(Term, [OtherLineD,$$]), + check_location(Term, {0, Cols - Col - width(U)+1}), + + send_tty(Term,"C-A"), + check_location(Term, {-4, 0}), %% Broken + send_tty(Term,"Right"), + check_location(Term, {-4, width(U)}), %% Broken + + send_tty(Term,"DC"), %% Broken + check_content(Term, ["a.*",U,"$"]), + check_content(Term, ["^b.*",U,"c$"]), + check_content(Term, ["^c.*",U,"dd$"]), + + send_tty(Term,"a"), + check_content(Term, [FirstLine,$$]), + check_content(Term, [OtherLineA,$$]), + check_content(Term, [OtherLineB,$$]), + check_content(Term, [OtherLineC,$$]), + check_content(Term, [OtherLineD,$$]), + + send_tty(Term,"Enter") + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + +shell_delete(Config) -> + + Term = start_tty(Config), + + try + + [ begin + send_tty(Term,"a"), + check_content(Term, "> a$"), + check_location(Term, {0, 1}), + send_tty(Term,"BSpace"), + check_location(Term, {0, 0}), + check_content(Term, ">$"), + send_tty(Term,"a"), + send_tty(Term,U), + check_location(Term, {0, width([$a, U])}), + send_tty(Term,"a"), + send_tty(Term,U), + check_location(Term, {0, width([$a,U,$a,U])}), + check_content(Term, ["> a",U,$a,U,"$"]), + send_tty(Term,"Left"), + send_tty(Term,"Left"), + send_tty(Term,"BSpace"), + check_location(Term, {0, width([$a])}), + check_content(Term, ["> aa",U,"$"]), + send_tty(Term,U), + check_location(Term, {0, width([$a,U])}), + send_tty(Term,"Left"), + send_tty(Term,"DC"), + check_location(Term, {0, width([$a])}), + check_content(Term, ["> aa",U,"$"]), + send_tty(Term,"DC"), + send_tty(Term,"DC"), + check_content(Term, ["> a$"]), + send_tty(Term,"C-E"), + check_location(Term, {0, width([$a])}), + send_tty(Term,"BSpace"), + check_location(Term, {0, width([])}) + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + +%% When deleting characters at the edge of the screen that are "large", +%% we need to take special care. +shell_delete_unicode_wrap(Config) -> + + Term = start_tty(Config), + + {_Rows, Cols} = get_window_size(Term), + {_Row, Col} = get_location(Term), + + try + [begin + send_tty(Term,lists:duplicate(Cols - Col,"a")), + check_content(Term,"> a*$"), + send_tty(Term,[U,U,"aaaaa"]), + check_content(Term,["\n",U,U,"aaaaa$"]), + [send_tty(Term,"Left") || _ <- lists:seq(1,5+2)], + check_location(Term,{0,-Col}), + send_tty(Term,"BSpace"), + check_content(Term,"> a* \n"), + check_location(Term,{-1,Cols - Col - 1}), + send_tty(Term,"BSpace"), + check_content(Term,["> a*",U,"\n"]), + check_location(Term,{-1,Cols - Col - 2}), + send_tty(Term,"BSpace"), + check_content(Term,["> a*",U," \n"]), + check_location(Term,{-1,Cols - Col - 3}), + send_tty(Term,"BSpace"), + check_content(Term,["> a*",U,U,"\n"]), + check_content(Term,["\naaaaa$"]), + check_location(Term,{-1,Cols - Col - 4}), + send_tty(Term,"BSpace"), + check_content(Term,["> a*",U,U,"a\n"]), + check_content(Term,["\naaaa$"]), + check_location(Term,{-1,Cols - Col - 5}), + send_tty(Term,"Enter") + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + +%% When deleting characters and a "large" characters is changing line we need +%% to take extra care +shell_delete_unicode_not_at_cursor_wrap(Config) -> + + Term = start_tty(Config), + + {_Rows, Cols} = get_window_size(Term), + {_Row, Col} = get_location(Term), + + try + [begin + send_tty(Term,lists:duplicate(Cols - Col,"a")), + check_content(Term,"> a*$"), + send_tty(Term,["a",U,"aaaaa"]), + check_content(Term,["\na",U,"aaaaa$"]), + send_tty(Term,"C-A"), + send_tty(Term,"DC"), + check_content(Term,["\n",U,"aaaaa$"]), + send_tty(Term,"DC"), + check_content(Term,["\n",U,"aaaaa$"]), + check_content(Term,["> a* \n"]), + send_tty(Term,"DC"), + check_content(Term,["\naaaaa$"]), + check_content(Term,["> a*",U,"\n"]), + send_tty(Term,"DC"), + check_content(Term,["\naaaa$"]), + check_content(Term,["> a*",U,"a\n"]), + send_tty(Term,"Enter") + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + +%% When deleting characters and a "large" characters is changing line we need +%% to take extra care +shell_update_window_unicode_wrap(Config) -> + + Term = start_tty(Config), + + {_Rows, Cols} = get_window_size(Term), + {_Row, Col} = get_location(Term), + + try + [begin + send_tty(Term,lists:duplicate(Cols - Col - width(U) + 1,"a")), + check_content(Term,"> a*$"), + send_tty(Term,[U,"aaaaa"]), + check_content(Term,["> a* ?\n",U,"aaaaa$"]), + tmux(["resize-window -t ",tty_name(Term)," -x ",Cols+1]), + check_content(Term,["> a*",U,"\naaaaa$"]), + tmux(["resize-window -t ",tty_name(Term)," -x ",Cols]), + check_content(Term,["> a* ?\n",U,"aaaaa$"]), + send_tty(Term,"Enter") + end || U <- hard_unicode()] + after + stop_tty(Term) + end. + +shell_transpose(Config) -> + + Term = start_tty(Config), + + Unicode = [[$a]] ++ hard_unicode(), + + try + [ + begin + send_tty(Term,"a"), + [send_tty(Term,[CP]) || CP <- U], + send_tty(Term,"b"), + [[send_tty(Term,[CP]) || CP <- U2] || U2 <- Unicode], + send_tty(Term,"cde"), + check_content(Term, ["a",U,"b",Unicode,"cde$"]), + check_location(Term, {0, width(["a",U,"b",Unicode,"cde"])}), + send_tty(Term,"Home"), + check_location(Term, {0, 0}), + send_tty(Term,"Right"), + send_tty(Term,"Right"), + check_location(Term, {0, 1+width([U])}), + send_tty(Term,"C-T"), + check_content(Term, ["ab",U,Unicode,"cde$"]), + send_tty(Term,"C-T"), + check_content(Term, ["ab",hd(Unicode),U,tl(Unicode),"cde$"]), + [send_tty(Term,"C-T") || _ <- lists:seq(1,length(Unicode)-1)], + check_content(Term, ["ab",Unicode,U,"cde$"]), + send_tty(Term,"C-T"), + check_content(Term, ["ab",Unicode,"c",U,"de$"]), + check_location(Term, {0, width(["ab",Unicode,"c",U])}), + send_tty(Term,"End"), + check_location(Term, {0, width(["ab",Unicode,"c",U,"de"])}), + send_tty(Term,"Left"), + send_tty(Term,"Left"), + send_tty(Term,"BSpace"), + check_content(Term, ["ab",Unicode,"cde$"]), + send_tty(Term,"End"), + send_tty(Term,"Enter") + end || U <- Unicode], + ok + after + stop_tty(Term), + ok + end. + +shell_search(C) -> + + Term = start_tty(C), + {_Row, Cols} = get_location(Term), + + try + send_tty(Term,"a"), + send_tty(Term,"."), + send_tty(Term,"Enter"), + send_tty(Term,"'"), + send_tty(Term,"a"), + send_tty(Term,[16#1f600]), + send_tty(Term,"'"), + send_tty(Term,"."), + send_tty(Term,"Enter"), + check_location(Term, {0, 0}), + send_tty(Term,"C-r"), + check_location(Term, {0, - Cols + width(C, "(search)`': 'a😀'.") }), + send_tty(Term,"C-a"), + check_location(Term, {0, width(C, "'a😀'.")}), + send_tty(Term,"Enter"), + send_tty(Term,"C-r"), + check_location(Term, {0, - Cols + width(C, "(search)`': 'a😀'.") }), + send_tty(Term,"a"), + check_location(Term, {0, - Cols + width(C, "(search)`a': 'a😀'.") }), + send_tty(Term,"C-r"), + check_location(Term, {0, - Cols + width(C, "(search)`a': a.") }), + send_tty(Term,"BSpace"), + check_location(Term, {0, - Cols + width(C, "(search)`': 'a😀'.") }), + send_tty(Term,"BSpace"), + check_location(Term, {0, - Cols + width(C, "(search)`': 'a😀'.") }), + ok + after + stop_tty(Term), + ok + end. + +shell_insert(Config) -> + Term = start_tty(Config), + + try + send_tty(Term,"abcdefghijklm"), + check_content(Term, "abcdefghijklm$"), + check_location(Term, {0, 13}), + send_tty(Term,"Home"), + send_tty(Term,"Right"), + send_tty(Term,"C-T"), + send_tty(Term,"C-T"), + send_tty(Term,"C-T"), + send_tty(Term,"C-T"), + check_content(Term, "bcdeafghijklm$"), + send_tty(Term,"End"), + send_tty(Term,"Left"), + send_tty(Term,"Left"), + send_tty(Term,"BSpace"), + check_content(Term, "bcdeafghijlm$"), + ok + after + stop_tty(Term) + end. + +shell_update_window(Config) -> + Term = start_tty(Config), + + Text = lists:flatten(["abcdefghijklmabcdefghijklm"]), + {_Row, Col} = get_location(Term), + + try + send_tty(Term,Text), + check_content(Term,Text), + check_location(Term, {0, width(Text)}), + tmux(["resize-window -t ",tty_name(Term)," -x ",width(Text)+Col+1]), + send_tty(Term,"a"), + check_location(Term, {0, -Col}), + send_tty(Term,"BSpace"), + tmux(["resize-window -t ",tty_name(Term)," -x ",width(Text)+Col]), + %% xnfix bug! at least in tmux... seems to work in iTerm as it does not + %% need xnfix when resizing + check_location(Term, {0, -Col}), + tmux(["resize-window -t ",tty_name(Term)," -x ",width(Text) div 2 + Col]), + check_location(Term, {0, -Col + width(Text) div 2}), + ok + after + stop_tty(Term) + end. + +shell_huge_input(Config) -> + Term = start_tty(Config), + + ManyUnicode = lists:duplicate(100,hard_unicode()), + + try + send_tty(Term,ManyUnicode), + check_content(Term, hard_unicode_match(Config) ++ "$", + #{ replace => {"\n",""} }), + send_tty(Term,"Enter"), + ok + after + stop_tty(Term) + end. + +%% Test that the shell works when invalid utf-8 (aka latin1) is sent to it +shell_invalid_unicode(Config) -> + Term = start_tty(Config), + + InvalidUnicode = <<$å,$ä,$ö>>, %% åäö in latin1 + + try + send_tty(Term,hard_unicode()), + check_content(Term, hard_unicode() ++ "$"), + send_tty(Term,"Enter"), + check_content(Term, "illegal character"), + %% Send invalid utf-8 + send_stdin(Term,InvalidUnicode), + %% Check that the utf-8 was echoed + check_content(Term, "\\\\345\\\\344\\\\366$"), + send_tty(Term,"Enter"), + %% Check that the terminal entered "latin1" mode + send_tty(Term,"😀한."), + check_content(Term, "\\Q\\360\\237\\230\\200\\355\\225\\234.\\E$"), + send_tty(Term,"Enter"), + %% Check that we can reset the encoding to unicode + send_tty(Term,"io:setopts([{encoding,unicode}])."), + send_tty(Term,"Enter"), + check_content(Term, "\nok\n"), + send_tty(Term,"😀한"), + check_content(Term, "😀한$"), + ok + after + stop_tty(Term), + ok + end. + + +%% Test the we can handle ansi insert, navigation and delete +%% We currently can not so skip this test +shell_support_ansi_input(Config) -> + + Term = start_tty(Config), + + BoldText = "\e[;1m", + ClearText = "\e[0m", + + try + send_stdin(Term,["{",BoldText,"a😀b",ClearText,"}"]), + timer:sleep(1000), + try check_location(Term, {0, width("{1ma😀bm}")}) of + _ -> + throw({skip, "Do not support ansi input"}) + catch _:_ -> + ok + end, + check_location(Term, {0, width("{a😀b}")}), + check_content(fun() -> get_content(Term,"-e") end, + ["{", BoldText, "a😀b", ClearText, "}"]), + send_tty(Term,"Left"), + send_tty(Term,"Left"), + check_location(Term, {0, width("{a😀")}), + send_tty(Term,"C-Left"), + check_location(Term, {0, width("{")}), + send_tty(Term,"End"), + send_tty(Term,"BSpace"), + send_tty(Term,"BSpace"), + check_content(Term, ["{", BoldText, "a😀"]), + ok + after + stop_tty(Term), + ok + end. + +shell_expand_location_below(Config) -> + + Term = start_tty(Config), + + {Rows, _} = get_location(Term), + + NumFunctions = lists:seq(0, Rows*2), + FunctionName = "a_long_function_name", + + Module = lists:flatten( + ["-module(long_module).\n", + "-export([",lists:join($,,[io_lib:format("~s~p/0",[FunctionName,I]) || I <- NumFunctions]),"]).\n\n", + [io_lib:format("~s~p() -> ok.\n",[FunctionName,I]) || I <- NumFunctions]]), + + DoScan = fun F([]) -> + []; + F(Str) -> + {done,{ok,T,_},C} = erl_scan:tokens([],Str,0), + [ T | F(C) ] + end, + Forms = [ begin {ok,Y} = erl_parse:parse_form(X),Y end || X <- DoScan(Module) ], + {ok,_,Bin} = compile:forms(Forms, [debug_info]), + + try + tmux(["resize-window -t ",tty_name(Term)," -x 80"]), + + %% First check that basic completion works + send_stdin(Term, "escript:"), + send_stdin(Term, "\t"), + %% Cursor at correct place + check_location(Term, {-3, width("escript:")}), + %% Nothing after the start( completion + check_content(Term, "start\\($"), + + %% Check that completion is cleared when we type + send_stdin(Term, "s"), + check_location(Term, {-3, width("escript:s")}), + check_content(Term, "escript:s$"), + + %% Check that completion works when in the middle of a term + send_tty(Term, "Home"), + send_tty(Term, "["), + send_tty(Term, "End"), + send_tty(Term, ", test_after]"), + [send_tty(Term, "Left") || _ <- ", test_after]"], + send_stdin(Term, "\t"), + check_location(Term, {-3, width("[escript:s")}), + check_content(Term, "script_name\\([ ]+start\\($"), + send_tty(Term, "C-K"), + + %% Check that completion works when in the middle of a long term + send_tty(Term, ", "++ lists:duplicate(80*2, $a)++"]"), + [send_tty(Term, "Left") || _ <- ", "++ lists:duplicate(80*2, $a)++"]"], + send_stdin(Term, "\t"), + check_location(Term, {-4, width("[escript:s")}), + check_content(Term, "script_name\\([ ]+start\\($"), + send_tty(Term, "End"), + send_stdin(Term, ".\n"), + + %% Check that we behave as we should with very long completions + rpc(Term, fun() -> + {module, long_module} = code:load_binary(long_module, "long_module.beam", Bin) + end), + check_content(Term, "3>"), + send_stdin(Term, "long_module:" ++ FunctionName), + send_stdin(Term, "\t"), + %% Check that correct text is printed below expansion + check_content(Term, io_lib:format("Press tab to see all ~p expansions", + [length(NumFunctions)])), + send_stdin(Term, "\t"), + %% The expansion does not fit on screen, verify that + %% expand above mode is used + check_content(fun() -> get_content(Term, "-S -5") end, + "3> long_module:" ++ FunctionName ++ "\nfunctions"), + check_content(Term, "3> long_module:" ++ FunctionName ++ "$"), + + %% We resize the terminal to make everything fit and test that + %% expand below mode is used + tmux(["resize-window -t ", tty_name(Term), " -y ", integer_to_list(Rows+10)]), + timer:sleep(1000), %% Sleep to make sure window has resized + send_stdin(Term, "\t\t"), + check_content(Term, "3> long_module:" ++ FunctionName ++ "\nfunctions(\n|.)*a_long_function_name99\\($"), + ok + after + stop_tty(Term), + ok + end. + +shell_expand_location_above(Config) -> + + Term = start_tty([{args,["-stdlib","shell_expand_location","above"]}|Config]), + + try + tmux(["resize-window -t ",tty_name(Term)," -x 80"]), + send_stdin(Term, "escript:"), + send_stdin(Term, "\t"), + check_location(Term, {0, width("escript:")}), + check_content(Term, "start\\(\n"), + check_content(Term, "escript:$"), + send_stdin(Term, "s"), + send_stdin(Term, "\t"), + check_location(Term, {0, width("escript:s")}), + check_content(Term, "\nscript_name\\([ ]+start\\(\n"), + check_content(Term, "escript:s$"), + ok + after + stop_tty(Term), + ok + end. + +%% Test the we can handle invalid ansi escape chars. +%% tmux cannot handle this... so we test this using to_erl +shell_invalid_ansi(_Config) -> + + InvalidAnsiPrompt = + case proplists:get_value(encoding, io:getopts(user)) of + unicode -> + ["\e]94m",54620,44397,50612,47,51312,49440,47568,"\e]0m"]; + latin1 -> + ["\e]94minvalid_test\e]0m"] + end, + + rtnode:run( + [{eval, fun() -> application:set_env( + stdlib, shell_prompt_func_test, + fun() -> InvalidAnsiPrompt end) + end }, + {putline,"a."}, + {expect, "a[.]"}, + {expect, ["\\Q",InvalidAnsiPrompt,"\\E"]}], + "", "", + ["-pz",filename:dirname(code:which(?MODULE)), + "-connect_all","false", + "-kernel","logger_level","all", + "-kernel","shell_history","disabled", + "-kernel","prevent_overlapping_partitions","false", + "-eval","shell:prompt_func({interactive_shell_SUITE,prompt})." + ]). + +shell_ignore_pager_commands(Config) -> + Term = start_tty(Config), + case code:get_doc(file, #{sources=>[eep48]}) of + {error, _} -> {skip, "No documentation available"}; + _ -> + try + send_tty(Term, "h(file).\n"), + check_content(Term,"\\Qmore (y/n)? (y)\\E"), + send_tty(Term, "n\n"), + check_content(Term,"ok"), + send_tty(Term, "C-P"), + check_content(Term,"\\Qh(file).\\E"), + ok + after + stop_tty(Term), + ok + end + end. + +external_editor(Config) -> + case os:find_executable("nano") of + false -> {skip, "nano is not installed"}; + _ -> + Term = start_tty(Config), + try + tmux(["resize-window -t ",tty_name(Term)," -x 80"]), + send_tty(Term,"os:putenv(\"EDITOR\",\"nano\").\n"), + send_tty(Term, "\"some text with\nnewline in it\""), + check_content(Term,"3> \"some text with\\s*\n.+3>\\s*newline in it\""), + send_tty(Term, "C-O"), + check_content(Term,"GNU nano [\\d.]+"), + check_content(Term,"newline in it\""), + send_tty(Term, "still"), + send_tty(Term, "Enter"), + send_tty(Term, "C-O"), %% save in nano + send_tty(Term, "Enter"), + send_tty(Term, "C-X"), %% quit in nano + check_content(Term,"still\n.+3> newline in it\""), + send_tty(Term,".\n"), + check_content(Term,"\\Q\"some text with\\nstill\\nnewline in it\"\\E"), + ok + after + stop_tty(Term), + ok + end + end. + +external_editor_visual(Config) -> + case os:find_executable("vim") of + false -> + {skip, "vim is not installed"}; + _ -> + Term = start_tty(Config), + try + tmux(["resize-window -t ",tty_name(Term)," -x 80"]), + send_tty(Term,"os:putenv(\"EDITOR\",\"nano\").\n"), + check_content(Term, "3>"), + send_tty(Term,"os:putenv(\"VISUAL\",\"vim -u DEFAULTS -U NONE -i NONE\").\n"), + check_content(Term, "4>"), + send_tty(Term,"\"hello"), + send_tty(Term, "C-O"), %% Open vim + check_content(Term, "^\"hello"), + send_tty(Term, "$"), %% Set cursor at end + send_tty(Term, "a"), %% Enter insert mode at end + check_content(Term, "-- INSERT --"), + send_tty(Term, "\"."), + send_tty(Term,"Escape"), + send_tty(Term,":wq"), + send_tty(Term,"Enter"), + check_content(Term, "\"hello\"[.]$"), + send_tty(Term,"Enter"), + check_content(Term, "\"hello\""), + check_content(Term, "5>$") + after + stop_tty(Term), + ok + end + end. + +external_editor_unicode(Config) -> + NanoPath = os:find_executable("nano"), + case NanoPath of + false -> {skip, "nano is not installed"}; + _ -> + Term = start_tty(Config), + try + tmux(["resize-window -t ",tty_name(Term)," -x 80"]), + send_tty(Term,"os:putenv(\"EDITOR\",\"nano\").\n"), + send_tty(Term, hard_unicode()), + check_content(Term,"3> " ++ hard_unicode_match(Config)), + send_tty(Term, "C-O"), %% open external editor (nano) + check_content(Term,"GNU nano [\\d.]+"), + send_tty(Term, "still "), + check_content(Term,"\nstill "), + send_tty(Term, "C-O"), %% save in nano + send_tty(Term, "Enter"), + send_tty(Term, "C-X"), %% quit in nano + check_content(Term,"still "++hard_unicode_match(Config)), + ok + after + stop_tty(Term), + ok + end + end. + +%% There used to be a race condition when using shell:start_interactive where +%% the newline handling of standard_error did not change correctly to compensate +%% for the tty changing to canonical mode +shell_standard_error_nlcr(Config) -> + + [ + begin + Term = setup_tty([{env,[{"TERM",TERM}]},{args, ["-noshell"]} | Config]), + try + rpc(Term, io, format, [standard_error,"test~ntest~ntest", []]), + check_content(Term, "test\ntest\ntest$"), + rpc(Term, fun() -> shell:start_interactive(), + io:format(standard_error,"test~ntest~ntest", []) + end), + check_content(Term, "test\ntest\ntest(\n|.)*test\ntest\ntest") + after + stop_tty(Term) + end + end || TERM <- ["dumb",os:getenv("TERM")]]. + +%% We test that suspending of `erl` and then resuming restores the shell +shell_suspend(Config) -> + + Name = peer:random_name(proplists:get_value(tc_path,Config)), + %% In order to suspend `erl` we need it to run in a shell that has job control + %% so we start the peer within a tmux window instead of having it be the original + %% process. + os:cmd("tmux new-window -n " ++ Name ++ " -d -- bash --norc"), + + Peer = #{ name => Name, + post_process_args => + fun(["new-window","-n",_,"-d","--"|CmdAndArgs]) -> + FlatCmdAndArgs = + lists:join( + " ",[[$',A,$'] || A <- CmdAndArgs]), + ["send","-t",Name,lists:flatten(FlatCmdAndArgs),"Enter"] + end + }, + + + Term = start_tty([{peer, Peer}|Config]), + + try + send_tty(Term, hard_unicode()), + check_content(Term,["2> ",hard_unicode(),"$"]), + send_tty(Term, "C-Z"), + check_content(Term,"\\Q[1]+\\E\\s*Stopped"), + send_tty(Term, "fg"), + send_tty(Term, "Enter"), + send_tty(Term, "M-l"), + check_content(Term,["2> ",hard_unicode(),"$"]), + check_location(Term,{0,width(hard_unicode())}), + ok + after + stop_tty(Term), + ok + end. + +%% We test that suspending of `erl` and then resuming restores the shell +shell_full_queue(Config) -> + + [throw({skip,"Need unbuffered to run"}) || os:find_executable("unbuffered") =:= false], + + %% In order to fill the read buffer of the terminal we need to get a + %% bit creative. We first need to start erl in bash in order to be + %% able to get access to job control for suspended processes. + %% We then also wrap `erl` in `unbuffer -p` so that we can suspend + %% that program in order to block writing to stdout for a while. + + Name = peer:random_name(proplists:get_value(tc_path,Config)), + os:cmd("tmux new-window -n " ++ Name ++ " -d -- bash --norc"), + + Peer = #{ name => Name, + post_process_args => + fun(["new-window","-n",_,"-d","--"|CmdAndArgs]) -> + FlatCmdAndArgs = ["unbuffer -p "] ++ + lists:join( + " ",[[$',A,$'] || A <- CmdAndArgs]), + ["send","-t",Name,lists:flatten(FlatCmdAndArgs),"Enter"] + end + }, + + + Term = start_tty([{peer, Peer}|Config]), + + UnbufferedPid = os:cmd("ps -o ppid= -p " ++ rpc(Term,os,getpid,[])), + + WriteUntilStopped = + fun F(Char) -> + rpc(Term,io,format,[user,[Char],[]]), + put(bytes,get(bytes,0)+1), + receive + stop -> + rpc(Term,io,format,[user,[Char+1],[]]) + after 0 -> F(Char) + end + end, + + WaitUntilBlocked = + fun(Pid, Ref) -> + (fun F(Cnt) -> + receive + {'DOWN',Ref,_,_,_} = Down -> + ct:fail({io_format_did_not_block, Down}) + after 1000 -> + ok + end, + case process_info(Pid,dictionary) of + {dictionary,[{bytes,Cnt}]} -> + ct:log("Bytes until blocked: ~p~n",[Cnt]), + %% Add one extra byte as for + %% the current blocking call + Cnt + 1; + {dictionary,[{bytes,NewCnt}]} -> + F(NewCnt) + end + end)(0) + end, + + try + %% First test that we can suspend and then resume + os:cmd("kill -TSTP " ++ UnbufferedPid), + check_content(Term,"\\Q[1]+\\E\\s*Stopped"), + {Pid, Ref} = spawn_monitor(fun() -> WriteUntilStopped($a) end), + WaitUntilBlocked(Pid, Ref), + send_tty(Term, "fg"), + send_tty(Term, "Enter"), + Pid ! stop, + check_content(Term,"b$"), + + send_tty(Term, "."), + send_tty(Term, "Enter"), + + %% Then we test that all characters are written when system + %% is terminated just after writing + {ok,Cols} = rpc(Term,io,columns,[user]), + send_tty(Term, "Enter"), + os:cmd("kill -TSTP " ++ UnbufferedPid), + check_content(Term,"\\Q[1]+\\E\\s*Stopped"), + {Pid2, Ref2} = spawn_monitor(fun() -> WriteUntilStopped($c) end), + Bytes = WaitUntilBlocked(Pid2, Ref2) - 1, + stop_tty(Term), + send_tty(Term, "fg"), + send_tty(Term, "Enter"), + check_content( + fun() -> + tmux(["capture-pane -p -S - -E - -t ",tty_name(Term)]) + end, lists:flatten([lists:duplicate(Cols,$c) ++ "\n" || + _ <- lists:seq(1,(Bytes) div Cols)] + ++ [lists:duplicate((Bytes) rem Cols,$c)])), + ct:log("~ts",[tmux(["capture-pane -p -S - -E - -t ",tty_name(Term)])]), + ok + after + stop_tty(Term), + ok + end. + +get(Key,Default) -> + case get(Key) of + undefined -> + Default; + Value -> + Value + end. + +%% A list of unicode graphemes that are notoriously hard to render +hard_unicode() -> + ZWJ = + case os:type() of + %% macOS has very good rendering of ZWJ, + %% but the cursor does not agree with it.. + {unix, darwin} -> []; + _ -> [[16#1F91A,16#1F3FC]] % Hand with skintone 🤚🏼 + end, + [[16#1f600], % Smilie 😀 + "한", % Hangul + "Z̤͔ͧ̑̓","ä͖̭̈̇","lͮ̒ͫ","ǧ̗͚̚","o̙̔ͮ̇͐̇" %% Vertically stacked chars + %%"👩👩", % Zero width joiner + %%"👩👩👧👦" % Zero width joiner + | ZWJ]. + +hard_unicode_match(Config) -> + ["\\Q",[unicode_to_octet(Config, U) || U <- hard_unicode()],"\\E"]. + +unicode_to_octet(Config, U) -> + case ?config(encoding,Config) of + unicode -> U; + latin1 -> unicode_to_octet(U) + end. + +unicode_to_octet(U) -> + [if Byte >= 128 -> [$\\,integer_to_list(Byte,8)]; + true -> Byte + end || <<Byte>> <= unicode:characters_to_binary(U)]. + +unicode_to_hex(Config, U) -> + case ?config(encoding,Config) of + unicode -> U; + latin1 -> unicode_to_hex(U) + end. + +unicode_to_hex(U) when is_integer(U) -> + unicode_to_hex([U]); +unicode_to_hex(Us) -> + [if U < 128 -> U; + U < 512 -> ["\\",integer_to_list(U,8)]; + true -> ["\\x{",integer_to_list(U,16),"}"] + end || U <- Us]. + +width(C, Str) -> + case ?config(encoding, C) of + unicode -> width(Str); + latin1 -> width(unicode_to_octet(Str)) + end. +width(Str) -> + lists:sum( + [npwcwidth(CP) || CP <- lists:flatten(Str)]). + +npwcwidth(CP) -> + try prim_tty:npwcwidth(CP) + catch error:undef -> + if CP =:= 16#D55C -> + 2; %% 한 + CP =:= 16#1f91A -> + 2; %% hand + CP =:= 16#1F3Fc -> + 2; %% Skintone + CP =:= 16#1f600 -> + 2; %% smilie + true -> + case lists:member(CP, [775,776,780,785,786,787,788,791,793,794, + 804,813,848,852,854,858,871,875,878]) of + true -> + 0; + false -> + 1 + end + end + end. + +-record(tmux, {peer, node, name, orig_location }). + +tmux([Cmd|_] = Command) when is_list(Cmd) -> + tmux(lists:concat(Command)); +tmux(Command) -> + string:trim(os:cmd(["tmux ",Command])). + +rpc(#tmux{ node = N }, Fun) -> + erpc:call(N, Fun). +rpc(#tmux{ node = N }, M, F, A) -> + erpc:call(N, M, F, A). + +%% Setup a TTY, but do not type anything in terminal +setup_tty(Config) -> + Name = maps:get(name,proplists:get_value(peer, Config, #{}), + peer:random_name(proplists:get_value(tc_path, Config))), + + Envs = lists:flatmap(fun({Key,Value}) -> + ["-env",Key,Value] + end, proplists:get_value(env,Config,[])), + + ExtraArgs = proplists:get_value(args,Config,[]), + + ExecArgs = case os:getenv("TMUX_DEBUG") of + "strace" -> + STraceLog = filename:join(proplists:get_value(priv_dir,Config), + Name++".strace"), + ct:pal("Link to strace: file://~ts", [STraceLog]), + [os:find_executable("strace"),"-f", + "-o",STraceLog, + "-e","trace=all", + "-e","read=0,1,2", + "-e","write=0,1,2" + ] ++ string:split(ct:get_progname()," ",all); + "rr" -> + [os:find_executable("cerl"),"-rr"]; + _ -> + string:split(ct:get_progname()," ",all) + end, + DefaultPeerArgs = #{ name => Name, + exec => + {os:find_executable("tmux"), + ["new-window","-n",Name,"-d","--"] ++ ExecArgs }, + + args => ["-pz",filename:dirname(code:which(?MODULE)), + "-connect_all","false", +% "-kernel","logger_level","all", + "-kernel","shell_history","disabled", + "-kernel","prevent_overlapping_partitions","false", + "-eval","shell:prompt_func({interactive_shell_SUITE,prompt})." + ] ++ Envs ++ ExtraArgs, + detached => false + }, + + {ok, Peer, Node} = + ?CT_PEER(maps:merge(proplists:get_value(peer,Config,#{}), + DefaultPeerArgs)), + + Self = self(), + + %% By default peer links with the starter. For these TCs we however only + %% want the peer to die if we die, so we create a "unidirection link" using + %% monitors. + spawn(fun() -> + TCRef = erlang:monitor(process, Self), + PeerRef = erlang:monitor(process, Peer), + receive + {'DOWN',TCRef,_,_,Reason} -> + exit(Peer, Reason); + {'DOWN',PeerRef,_,_,_} -> + ok + end + end), + unlink(Peer), + + "" = tmux(["set-option -t ",Name," remain-on-exit on"]), + + %% We start tracing on the remote node in order to help debugging + TraceLog = filename:join(proplists:get_value(priv_dir,Config),Name++".trace"), + ct:log("Link to trace: file://~ts",[TraceLog]), + + spawn(Node, + fun() -> + {ok, _} = dbg:tracer(file,TraceLog), + %% dbg:p(whereis(user_drv),[c,m]), + %% dbg:p(whereis(user_drv_writer),[c,m]), + %% dbg:p(whereis(user_drv_reader),[c,m]), + %% dbg:tp(user_drv,x), + %% dbg:tp(prim_tty,x), + %% dbg:tpl(prim_tty,write_nif,x), + %% dbg:tpl(prim_tty,read_nif,x), + monitor(process, Self), + receive _ -> ok end + end), + + #tmux{ peer = Peer, node = Node, name = Name }. + +%% Start a tty, setup custom prompt and set cursor at bottom +start_tty(Config) -> + + Term = setup_tty(Config), + + Prompt = fun() -> ["\e[94m",54620,44397,50612,47,51312,49440,47568,"\e[0m"] end, + erpc:call(Term#tmux.node, application, set_env, + [stdlib, shell_prompt_func_test, + proplists:get_value(shell_prompt_func_test, Config, Prompt)]), + + {Rows, _} = get_window_size(Term), + + %% We send a lot of newlines here in order for the number of rows + %% in the window to be max so that we can predict what the cursor + %% position is. + [send_tty(Term,"\n") || _ <- lists:seq(1, Rows)], + + %% We enter an 'a' here so that we can get the correct orig position + %% with an alternative prompt. + send_tty(Term,"a.\n"), + check_content(Term,"2>$"), + OrigLocation = get_location(Term), + Term#tmux{ orig_location = OrigLocation }. + +prompt(L) -> + N = proplists:get_value(history, L, 0), + Fun = application:get_env(stdlib, shell_prompt_func_test, + fun() -> atom_to_list(node()) end), + io_lib:format("(~ts)~w> ",[Fun(),N]). + +stop_tty(Term) -> + catch peer:stop(Term#tmux.peer), + ct:log("~ts",[get_content(Term, "-e")]), +% "" = tmux("kill-window -t " ++ Term#tmux.name), + ok. + +tty_name(Term) -> + Term#tmux.name. + +send_tty(Term, "Home") -> + %% https://stackoverflow.com/a/55616731 + send_tty(Term,"Escape"), + send_tty(Term,"OH"); +send_tty(Term, "End") -> + send_tty(Term,"Escape"), + send_tty(Term,"OF"); +send_tty(#tmux{ name = Name } = _Term,Value) -> + [Head | Quotes] = string:split(Value, "'", all), + "" = tmux("send -t " ++ Name ++ " '" ++ Head ++ "'"), + [begin + "" = tmux("send -t " ++ Name ++ " \"'\""), + "" = tmux("send -t " ++ Name ++ " '" ++ V ++ "'") + end || V <- Quotes]. + +%% We use send_stdin for testing of things that we cannot sent via +%% the tmux send command, such as invalid unicode +send_stdin(Term, Chars) when is_binary(Chars) -> + rpc(Term,erlang,display_string,[stdin,Chars]); +send_stdin(Term, Chars) -> + send_stdin(Term, iolist_to_binary(unicode:characters_to_binary(Chars))). + +check_location(Term, Where) -> + check_location(Term, Where, 5). +check_location(#tmux{ orig_location = {OrigRow, OrigCol} = Orig } = Term, + {AdjRow, AdjCol} = Where, Attempt) -> + NewLocation = get_location(Term), + case {OrigRow+AdjRow,OrigCol+AdjCol} of + NewLocation -> NewLocation; + _ when Attempt =:= 0 -> + {NewRow, NewCol} = NewLocation, + ct:fail({wrong_location, {expected,{AdjRow, AdjCol}}, + {got,{NewRow - OrigRow, NewCol - OrigCol}, + {NewLocation, Orig}}}); + _ -> + timer:sleep(50), + check_location(Term, Where, Attempt -1) + end. + +get_location(Term) -> + RowAndCol = tmux("display -pF '#{cursor_y} #{cursor_x}' -t "++Term#tmux.name), + [Row, Col] = string:lexemes(string:trim(RowAndCol,both)," "), + {list_to_integer(Row), list_to_integer(Col)}. + +get_window_size(Term) -> + RowAndCol = tmux("display -pF '#{window_height} #{window_width}' -t "++Term#tmux.name), + [Row, Col] = string:lexemes(string:trim(RowAndCol,both)," "), + {list_to_integer(Row), list_to_integer(Col)}. + +check_content(Term, Match) -> + check_content(Term, Match, #{}). +check_content(Term, Match, Opts) when is_map(Opts) -> + check_content(Term, Match, Opts, 5). +check_content(Term, Match, Opts, Attempt) -> + OrigContent = case Term of + #tmux{} -> get_content(Term); + Fun when is_function(Fun,0) -> Fun() + end, + Content = case maps:find(replace, Opts) of + {ok, {RE,Repl} } -> + re:replace(OrigContent, RE, Repl, [global]); + error -> + OrigContent + end, + case re:run(string:trim(Content, both), lists:flatten(Match), [unicode]) of + {match,_} -> + ok; + _ when Attempt =:= 0 -> + io:format("Failed to find '~ts' in ~n'~ts'~n", + [unicode:characters_to_binary(Match), Content]), + io:format("Failed to find '~w' in ~n'~w'~n", + [unicode:characters_to_binary(Match), Content]), + ct:fail(nomatch); + _ -> + timer:sleep(500), + check_content(Term, Match, Opts, Attempt - 1) + end. + +get_content(Term) -> + get_content(Term, ""). +get_content(#tmux{ name = Name }, Args) -> + Content = unicode:characters_to_binary(tmux("capture-pane -p " ++ Args ++ " -t " ++ Name)), + case string:split(Content,"a.\na") of + [_Ignore,C] -> + C; + [C] -> + C + end. %% Tests that exit of initial shell restarts shell. exit_initial(Config) when is_list(Config) -> @@ -260,36 +1617,38 @@ exit_initial(Config) when is_list(Config) -> ok. test_exit_initial(old) -> - rtnode([{putline, ""}, - {putline, "2."}, - {expect, "2\r\n"}, - {putline, "exit()."}, - {expect, "Eshell"}, - {putline, ""}, - {putline, "35."}, - {expect, "35\r\n"}], - [], [], ["-oldshell"]); + rtnode:run( + [{putline, ""}, + {putline, "2."}, + {expect, "2\r\n"}, + {putline, "exit()."}, + {expect, "Eshell"}, + {putline, ""}, + {putline, "35."}, + {expect, "35\r\n"}], + [], [], ["-oldshell"]); test_exit_initial(new) -> - rtnode([{putline, ""}, - {expect, "1> $"}, - {putline, "2."}, - {expect, "2"}, - {putline,"exit()."}, - {expect, "Eshell"}, - {expect, "1> $"}, - {putline, "35."}, - {expect, "35\r\n"}]). + rtnode:run( + [{putline, ""}, + {expect, "1> $"}, + {putline, "2."}, + {expect, "2"}, + {putline,"exit()."}, + {expect, "Eshell"}, + {expect, "1> $"}, + {putline, "35."}, + {expect, "35\r\n"}]). stop_during_init(Config) when is_list(Config) -> - {RunErl,_ToErl,Erl} = get_progs(), - case create_tempdir() of + {RunErl,_ToErl,[Erl|ErlArgs]} = rtnode:get_progs(), + case rtnode:create_tempdir() of {error, Reason} -> {skip, Reason}; Tempdir -> XArg = " -kernel shell_history enabled -s init stop", - start_runerl_command(RunErl, Tempdir, "\\\""++Erl++"\\\""++XArg), - Logs = rtnode_read_logs(Tempdir), - rtnode_dump_logs(Logs), + rtnode:start_runerl_command(RunErl, Tempdir, "\\\""++Erl++"\\\""++ErlArgs++XArg), + Logs = rtnode:read_logs(Tempdir), + rtnode:dump_logs(Logs), nomatch = binary:match(map_get("erlang.log.1", Logs), <<"*** ERROR: Shell process terminated! ***">>), ok @@ -312,16 +1671,16 @@ wrap(Config) when is_list(Config) -> case proplists:get_value(default_shell, Config) of new -> As = lists:duplicate(20,"a"), - rtnode([{putline, "io:columns()."}, - {expect, "{ok,20}\r\n"}, - {putline, ["io:format(\"~s\",[lists:duplicate(20,\"a\")])."]}, - {expect, As ++ " \b"}, - {putline, ["io:format(\"~s~n~s\",[lists:duplicate(20,\"a\"),lists:duplicate(20,\"a\")])."]}, - {expect, As ++ "\r\n" ++ As ++ " \b"} - ], - [], - "stty rows 40; stty columns 20; ", - [""]); + rtnode:run( + [{putline, "io:columns()."}, + {expect, "{ok,20}\r\n"}, + {putline, ["io:format(\"~s\",[lists:duplicate(20,\"a\")])."]}, + {expect, As ++ " \b"}, + {putline, ["io:format(\"~s~n~s\",[lists:duplicate(20,\"a\"),lists:duplicate(20,\"a\")])."]}, + {expect, As ++ "\r\n" ++ As ++ " \b"} + ], + [], + "stty rows 40; stty columns 20; "); _ -> ok end, @@ -336,58 +1695,71 @@ wrap(Config) when is_list(Config) -> %% commands. shell_history(Config) when is_list(Config) -> Path = shell_history_path(Config, "basic"), - rtnode([ - {putline, "echo1."}, - {expect, "echo1\r\n"}, - {putline, "echo2."}, - {expect, "echo2\r\n"}, - {putline, "echo3."}, - {expect, "echo3\r\n"}, - {putline, "echo4."}, - {expect, "echo4\r\n"}, - {putline, "echo5."}, - {expect, "echo5\r\n"} - ], [], [], " -kernel shell_history enabled " ++ - "-kernel shell_history_drop '[\\\"init:stop().\\\"]' " ++ - mk_sh_param(Path)), + rtnode:run( + [{putline, "echo1."}, + {expect, "echo1\r\n"}, + {putline, "echo2."}, + {expect, "echo2\r\n"}, + {putline, "echo3."}, + {expect, "echo3\r\n"}, + {putline, "echo4."}, + {expect, "echo4\r\n"}, + {putline, "echo5."}, + {expect, "echo5\r\n"} + ], [], [], mk_history_param(Path)), receive after 1000 -> ok end, - rtnode([ - {putline, ""}, - %% the init:stop that stopped the node is dropped - {putdata, [$\^p]}, {expect, "echo5[.]$"}, - {putdata, [$\n]}, - {expect, "echo5\r\n"}, - {putdata, [$\^p]}, {expect, "echo5[.]$"}, - {putdata, [$\^p]}, {expect, "echo4[.]$"}, - {putdata, [$\^p]}, {expect, "echo3[.]$"}, - {putdata, [$\^p]}, {expect, "echo2[.]$"}, - {putdata, [$\^n]}, {expect, "echo3[.]$"}, - {putdata, [$\^n]}, {expect, "echo4[.]$"}, - {putdata, [$\^b]}, {sleep,50}, %% the echo4. (cursor moved one left) - {putline, ["ECHO"]}, - {expect, "echo4ECHO\r\n"} - ], [], [], " -kernel shell_history enabled " ++ mk_sh_param(Path)), + rtnode:run( + [{sleep,100}, + {putline, ""}, + %% the init:stop that stopped the node is dropped + {putdata, [$\^p]}, {expect, "echo5[.]$"}, + {putdata, [$\n]}, + {expect, "echo5\r\n"}, + {putdata, [$\^p]}, {expect, "echo5[.]$"}, + {putdata, [$\^p]}, {expect, "echo4[.]$"}, + {putdata, [$\^p]}, {expect, "echo3[.]$"}, + {putdata, [$\^p]}, {expect, "echo2[.]$"}, + {putdata, [$\^n]}, {expect, "echo3[.]$"}, + {putdata, [$\^n]}, {expect, "echo4[.]$"}, + {putdata, [$\^b]}, {sleep,50}, %% the echo4. (cursor moved one left) + {putline, ["ECHO"]}, + {expect, "echo4ECHO\r\n"} + ], [], [], + mk_history_param(Path)), + receive after 1000 -> ok end, + + %% ignore input given after io:get_line, and after h(module) pager + rtnode:run( + [{putdata, "io:get_line(\"getline>\").\n"}, + {expect, "getline>"}, + {putline, "hej"}, {expect, "hej\r\n"}, + {putdata, [$\^p]}, {expect, "\\Qio:get_line(\"getline>\")\\E[.]$"} + ], [], [], + mk_history_param(Path)), ok. shell_history_resize(Config) -> Path = shell_history_path(Config, "resize"), - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history_file_bytes 123456 " ++ - "-kernel shell_history enabled " ++ mk_sh_param(Path)), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"}, + {putline, "echo2."}, + {expect, "echo2\r\n"} + ], [], [], ["-kernel","shell_history_file_bytes","123456"] ++ + mk_history_param(Path)), {ok, Logs} = - rtnode([ - {putline, ""}, - {putdata, [$\^p]}, {expect, "init:stop\\(\\)[.]$"}, - {putdata, [$\^p]}, {expect, "echo[.]$"}, - {putdata, [$\n]}, - {expect, "echo"} - ], [], [], " -kernel shell_history_file_bytes 654321 " ++ - "-kernel shell_history enabled " ++ mk_sh_param(Path)), - - rtnode_check_logs( + rtnode:run( + [{sleep,100}, + {putline, ""}, + {putdata, [$\^p]}, {expect, "echo2[.]$$"}, + {putdata, [$\^p]}, {expect, "echo[.]$"}, + {putdata, [$\n]}, + {expect, "echo"} + ], [], [], ["-kernel","shell_history_file_bytes","654321"] ++ + mk_history_param(Path)), + + rtnode:check_logs( "erlang.log.1", "The configured log history file size is different from the size " "of the log file on disk", Logs), @@ -405,24 +1777,24 @@ shell_history_eaccess(Config) -> %% Cannot create history log in folder {ok, Logs1} = - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"} + ], [], [], mk_history_param(Path)), - rtnode_check_logs("erlang.log.1", "Error handling file", Logs1), + ct:pal("~p",[Logs1]), + rtnode:check_logs("erlang.log.1", "Error handling file", Logs1), %% shell_docs recursively creates the folder to store the %% logs. This test checks that erlang still starts if we %% cannot create the folders to the path. {ok, Logs2} = - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ - mk_sh_param(filename:join(Path,"logs"))), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"} + ], [], [], mk_history_param(filename:join(Path,"logs"))), - rtnode_check_logs("erlang.log.1", "Error handling file", Logs2) + rtnode:check_logs("erlang.log.1", "Error handling file", Logs2) after file:write_file_info(Path, Info) @@ -436,15 +1808,15 @@ shell_history_repair(Config) -> shell_history_halt(Path), {ok, Logs} = - rtnode([ - {putline, ""}, - {putdata, [$\^p]}, {expect, "echo[.]$"}, - {putdata, [$\n]}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + rtnode:run( + [{putline, ""}, + {putdata, [$\^p]}, {expect, "echo[.]$"}, + {putdata, [$\n]}, + {expect, "echo\r\n"} + ], [], [], mk_history_param(Path)), %% The regexp below checks that he string is NOT part of the log - rtnode_check_logs("erlang.log.1", + rtnode:check_logs("erlang.log.1", "The shell history log file was corrupted and was repaired", false, Logs), @@ -462,14 +1834,14 @@ shell_history_repair_corrupt(Config) -> ok = file:close(D), {ok, Logs} = - rtnode([ - {putline, ""}, - {putdata, [$\^p]}, {expect, "echo[.]$"}, - {putdata, [$\n]}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), - - rtnode_check_logs("erlang.log.1", + rtnode:run( + [{putline, ""}, + {putdata, [$\^p]}, {expect, "echo[.]$"}, + {putdata, [$\n]}, + {expect, "echo\r\n"} + ], [], [], mk_history_param(Path)), + + rtnode:check_logs("erlang.log.1", "The shell history log file was corrupted and was repaired.", Logs), ok. @@ -478,9 +1850,12 @@ shell_history_corrupt(Config) -> Path = shell_history_path(Config, "corrupt"), %% We initialize the shell history log with a known value. - rtnode([{putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"}, + {putline, "echo2."}, + {expect, "echo2\r\n"} + ], [], [], mk_history_param(Path)), %% We corrupt the disklog. {ok, D} = file:open(filename:join(Path,"erlang-shell-log.1"), [read, append]), @@ -488,98 +1863,104 @@ shell_history_corrupt(Config) -> ok = file:close(D), {ok, Logs} = - rtnode([ - {putline, ""}, - {putdata, [$\^p]}, {expect, "init:stop\\(\\)[.]$"}, - {putdata, [$\^p]}, {expect, "echo[.]$"}, - {putdata, [$\n]}, - {expect, "echo\r\n"} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)), - - rtnode_check_logs("erlang.log.1", "Invalid chunk in the file", Logs), + rtnode:run( + [{putline, ""}, + {putdata, [$\^p]}, {expect, "echo2[.]$"}, + {putdata, [$\^p]}, {expect, "echo[.]$"}, + {putdata, [$\n]}, + {expect, "echo\r\n"} + ], [], [], mk_history_param(Path)), + + rtnode:check_logs("erlang.log.1", "Invalid chunk in the file", Logs), ok. %% Stop the node without closing the log. shell_history_halt(Path) -> try - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"}, - {sleep, 2500}, % disk_log internal cache timer is 2000 ms - {putline, "halt(0)."} - ], [], [], "-kernel shell_history enabled " ++ mk_sh_param(Path)) + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"}, + {sleep, 2500}, % disk_log internal cache timer is 2000 ms + {putline, "halt(0)."}, + {expect, "\r\n"}, + {sleep, 1000} %% wait for node to terminate + ], [], [], mk_history_param(Path)) catch _:_ -> ok end. shell_history_path(Config, TestCase) -> - filename:join([proplists:get_value(priv_dir, Config), - "shell_history", TestCase]). + filename:join([proplists:get_value(priv_dir, Config), + "shell_history", TestCase]). -mk_sh_param(Path) -> - "-kernel shell_history_path '\\\"" ++ Path ++ "\\\"'". +mk_history_param(Path) -> + ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]" + ]. shell_history_custom(_Config) -> %% Up key: Ctrl + P = Cp=[$\^p] - rtnode([{expect, "1> $"}, - %% {putline, ""}, - {putdata, [$\^p]}, {expect, "0[.]"}, - {putdata, [$\n]}, - {expect, "0\r\n"}, - {putline, "echo."}, - {expect, "!echo\r\n"} % exclamation mark is printed by custom history module - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{expect, "1> $"}, + %% {putline, ""}, + {putdata, [$\^p]}, {expect, "0[.]"}, + {putdata, [$\n]}, + {expect, "0\r\n"}, + {putline, "echo."}, + {expect, "!echo\r\n"} % exclamation mark is printed by custom history module + ], [], [], ["-kernel","shell_history",atom_to_list(?MODULE), + "-pz",filename:dirname(code:which(?MODULE))]), ok. shell_history_custom_errors(_Config) -> %% Check that we can start with a node with an undefined %% provider module. - rtnode([{expect, "1> $"}, - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history very_broken " ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"} + ], [], [], ["-kernel","shell_history","very_broken", + "-pz",filename:dirname(code:which(?MODULE))]), %% Check that we can start with a node with a provider module %% that crashes in load/0. - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -kernel provider_load crash" ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"} + ], [], [], ["-kernel","shell_history",atom_to_list(?MODULE), + "-kernel","provider_load","crash", + "-pz",filename:dirname(code:which(?MODULE))]), %% Check that we can start with a node with a provider module %% that return incorrect in load/0. - rtnode([ - {putline, "echo."}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -kernel provider_load badreturn" ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{putline, "echo."}, + {expect, "echo\r\n"} + ], [], [], ["-kernel","shell_history",atom_to_list(?MODULE), + "-kernel","provider_load","badreturn", + "-pz",filename:dirname(code:which(?MODULE))]), %% Check that we can start with a node with a provider module %% that crashes in load/0. - rtnode([ - {putline, "echo."}, - {expect, "Disabling shell history logging.\r\n"}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -kernel provider_add crash" ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{putline, "echo."}, + {expect, "(Disabling shell history logging.|echo)"}, + {expect, "(Disabling shell history logging.|echo)"} + ], [], [], ["-kernel","shell_history",atom_to_list(?MODULE), + "-kernel","provider_add","crash", + "-pz",filename:dirname(code:which(?MODULE))]), %% Check that we can start with a node with a provider module %% that return incorrect in load/0. - rtnode([ - {putline, "echo."}, - {expect, "It returned {error,badreturn}.\r\n"}, - {expect, "echo\r\n"} - ], [], [], " -kernel shell_history " ++ atom_to_list(?MODULE) ++ - " -kernel provider_add badreturn" ++ - " -pz " ++ filename:dirname(code:which(?MODULE))), + rtnode:run( + [{putline, "echo."}, + {expect, "It returned {error,badreturn}.\r\n"}, + {expect, "echo\r\n"} + ], [], [], ["-kernel","shell_history",atom_to_list(?MODULE), + "-kernel","provider_add","badreturn", + "-pz",filename:dirname(code:which(?MODULE))]), ok. @@ -613,20 +1994,60 @@ job_control_local(Config) when is_list(Config) -> {skip,"No new shell found"}; new -> %% New shell tests - rtnode([{putline, ""}, - {expect, "1> $"}, - {putline, "2."}, - {expect, "\r\n2\r\n"}, - {putline, "\^g"}, - {expect, ["--> $"]}, - {putline, "s"}, - {expect, ["--> $"]}, - {putline, "c"}, - {expect, ["\r\nEshell"]}, - {expect, ["1> $"]}, - {putline, "35."}, - {expect, "\r\n35\r\n2> $"}], - []), + rtnode:run( + [{putline, ""}, + {expect, "1> $"}, + {putline, "2."}, + {expect, "\r\n2\r\n"}, + {putline, "\^g"}, + {expect, "--> $"}, + {putline, "s"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, "\r\nEshell"}, + {expect, "1> $"}, + {putline, "35."}, + {expect, "\r\n35\r\n"}, + {expect, "2> $"}, + {putline, "receive M -> M end.\r\n"}, + {putline, "\^g"}, + {expect, "--> $"}, + {putline, "i 3"}, + {expect, "Unknown job"}, + {expect, "--> $"}, + {putline, "i 2"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, "[*][*] exception exit: killed"}, + {expect, "[23]>"}, + {putline, "\^g"}, + {expect, "--> $"}, + {putline, "k 3"}, + {expect, "Unknown job"}, + {expect, "--> $"}, + {putline, "k 2"}, + {expect, "--> $"}, + {putline, "k"}, + {expect, "Unknown job"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, "Unknown job"}, + {expect, "--> $"}, + {putline, "i"}, + {expect, "Unknown job"}, + {expect, "--> $"}, + {putline, "?"}, + {expect, "this message"}, + {expect, "--> $"}, + {putline, "h"}, + {expect, "this message"}, + {expect, "--> $"}, + {putline, "c 1"}, + {expect, "\r\n"}, + {putline, "35."}, + {expect, "\r\n35\r\n"}, + {expect, "[23]> $"} + ]), ok end. @@ -636,11 +2057,12 @@ job_control_remote(Config) when is_list(Config) -> old -> {skip,"No new shell found"}; _ -> - NSNode = start_node(?FUNCTION_NAME, []), + {ok, Peer, NSNode} = ?CT_PEER(#{ args => ["-connect_all","false"], + peer_down => continue }), try test_remote_job_control(NSNode) after - test_server:stop_node(NSNode) + peer:stop(Peer) end end. @@ -651,48 +2073,71 @@ job_control_remote_noshell(Config) when is_list(Config) -> old -> {skip,"No new shell found"}; _ -> - NSNode = start_node(?FUNCTION_NAME, ["-noshell"]), + {ok, Peer, NSNode} = ?CT_PEER(#{ name => peer:random_name(test_remote_job_control), + args => ["-connect_all","false","-noshell"], + peer_down => continue }), try test_remote_job_control(NSNode) after - test_server:stop_node(NSNode) + peer:stop(Peer) end end. test_remote_job_control(Node) -> - RemNode = create_nodename(), + RemNode = peer:random_name(test_remote_job_control), Pid = spawn_link(Node, fun() -> receive die -> ok end - end), + end), PidStr = erpc:call(Node, erlang, pid_to_list, [Pid]), true = erpc:call(Node, erlang, register, [kalaskula,Pid]), PrintedNode = printed_atom(Node), CookieString = printed_atom(erlang:get_cookie()), - rtnode([{putline, ""}, - {putline, "erlang:get_cookie()."}, - {expect, "\r\n\\Q" ++ CookieString ++ "\\E"}, - {putdata, "\^g"}, - {expect, " --> $"}, - {putline, "r " ++ PrintedNode}, - {expect, "\r\n"}, - {putline, "c"}, - {expect, "\r\n"}, - {expect, "Eshell"}, - {expect, "\\Q(" ++ atom_to_list(Node) ++")1> \\E$"}, - {putline, "whereis(kalaskula)."}, - {expect, PidStr}, - {putline, "exit()."}, - {expect, "[*][*][*] Shell process terminated!"}, - {putdata, "\^g"}, - {expect, " --> $"}, - {putline, "c 1"}, - {expect, "\r\n"}, - {putline, ""}, - {expect, "\\Q("++RemNode++")\\E[12]> $"} - ], RemNode), + rtnode:run( + [{putline, ""}, + {putline, "erlang:get_cookie()."}, + {expect, "\r\n\\Q" ++ CookieString ++ "\\E"}, + {putdata, "\^g"}, + {expect, " --> $"}, + {putline, "r " ++ PrintedNode}, + {expect, "\r\n"}, + {putline, "j"}, + {expect, "1 {shell,start,\\[init]}"}, + {expect, "2[*] {\\Q"++PrintedNode++"\\E,shell,start,\\[]}"}, + {expect, " --> $"}, + {putline, "c"}, + {expect, "\r\n"}, + {expect, "Eshell"}, + {expect, "\\Q(" ++ atom_to_list(Node) ++")1> \\E$"}, + {putline, "whereis(kalaskula)."}, + {expect, PidStr}, + {putline, "kalaskula ! die."}, + {putline, "exit()."}, + {expect, "[*][*][*] Shell process terminated!"}, + {putdata, "\^g"}, + {expect, " --> $"}, + {putline, "j"}, + {expect, "1 {shell,start,\\[init]}"}, + {expect, " --> $"}, + {putline, "c"}, + {expect, "Unknown job"}, + {expect, " --> $"}, + {putline, "c 1"}, + {expect, "\r\n"}, + {putline, ""}, + {expect, "\\Q("++RemNode++"@\\E[^)]*\\)[12]> $"}, + {putdata, "\^g"}, + {expect, " --> $"}, + {putline, "j"}, + {expect, "1[*] {shell,start,\\[init]}"}, + {putline, "c"}, + {expect, "\r\n"}, + {sleep, 100}, + {putline, "35."}, + {expect, "\\Q("++RemNode++"@\\E[^)]*\\)[123]> $"} + ], RemNode), Pid ! die, ok. @@ -703,20 +2148,21 @@ ctrl_keys(_Config) -> Cy = [$\^y], Home = [27,$O,$H], End = [27,$O,$F], - rtnode([{putline,""}, - {putline,"2."}, - {expect,"2"}, - {putline,"\"hello "++Cw++"world\"."}, % test <CTRL>+W - {expect,"\"world\""}, - {putline,"\"hello "++Cu++"\"world\"."}, % test <CTRL>+U - {expect,"\"world\""}, - {putline,"world\"."++Home++"\"hello "}, % test <HOME> - {expect,"\"hello world\""}, - {putline,"world"++Home++"\"hello "++End++"\"."}, % test <END> - {expect,"\"hello world\""}, - {putline,"\"hello world\""++Cu++Cy++"."}, - {expect,"\"hello world\""}] ++ - wordLeft() ++ wordRight(), []), + rtnode:run( + [{putline,""}, + {putline,"2."}, + {expect,"2"}, + {putline,"\"hello "++Cw++"world\"."}, % test <CTRL>+W + {expect,"\"world\""}, + {putline,"\"hello "++Cu++"\"world\"."}, % test <CTRL>+U + {expect,"\"world\""}, + {putline,"world\"."++Home++"\"hello "}, % test <HOME> + {expect,"\"hello world\""}, + {putline,"world"++Home++"\"hello "++End++"\"."}, % test <END> + {expect,"\"hello world\""}, + {putline,"\"hello world\""++Cu++Cy++"."}, + {expect,"\"hello world\""}] ++ + wordLeft() ++ wordRight()), ok. wordLeft() -> @@ -743,7 +2189,7 @@ wordRight(Chars) -> %% Test that -remsh works remsh_basic(Config) when is_list(Config) -> - TargetNode = start_node(?FUNCTION_NAME, []), + {ok, Peer, TargetNode} = ?CT_PEER(), TargetNodeStr = printed_atom(TargetNode), [_Name,Host] = string:split(atom_to_list(node()), "@"), @@ -756,19 +2202,30 @@ remsh_basic(Config) when is_list(Config) -> %% Test that remsh works with explicit -sname. HostNode = atom_to_list(?FUNCTION_NAME) ++ "_host", HostNodeStr = printed_atom(list_to_atom(HostNode ++ "@" ++ Host)), - rtnode(PreCmds ++ - [{putline,"nodes()."}, - {expect, "\\Q" ++ HostNodeStr ++ "\\E"}] ++ - PostCmds, - HostNode, [], "-remsh " ++ TargetNodeStr), + rtnode:run( + PreCmds ++ + [{putline,"nodes()."}, + {expect, "\\Q" ++ HostNodeStr ++ "\\E"}] ++ + PostCmds, + HostNode, " ", "-remsh " ++ TargetNodeStr), %% Test that remsh works without -sname. - rtnode(PreCmds ++ PostCmds, [], [], " -remsh " ++ TargetNodeStr), + rtnode:run(PreCmds ++ PostCmds, [], " ", "-remsh " ++ TargetNodeStr), + + %% Test that if multiple remsh are given, we select the first + rtnode:run([{expect, "Multiple"}] ++ PreCmds ++ PostCmds, + [], " ", + "-remsh " ++ TargetNodeStr ++ " -remsh invalid_node"), - test_server:stop_node(TargetNode), + peer:stop(Peer), ok. +%% Test that if we cannot connect to a node, we get a correct error +remsh_error(_Config) -> + "Could not connect to \"invalid_node\"\n" = + os:cmd(ct:get_progname() ++ " -remsh invalid_node"). + quit_hosting_node() -> %% Command sequence for entering a shell on the hosting node. [{putdata, "\^g"}, @@ -788,529 +2245,108 @@ remsh_longnames(Config) when is_list(Config) -> "@127.0.0.1"; _ -> "" end, - case rtstart(" -name " ++ atom_to_list(?FUNCTION_NAME)++Domain) of - {ok, _SRPid, STPid, SState} -> + Name = peer:random_name(?FUNCTION_NAME), + case rtnode:start(" -name " ++ Name ++ Domain) of + {ok, _SRPid, STPid, SNode, SState} -> try - {ok, _CRPid, CTPid, CState} = - rtstart("-name undefined" ++ Domain ++ - " -remsh " ++ atom_to_list(?FUNCTION_NAME)), + {ok, _CRPid, CTPid, CNode, CState} = + rtnode:start("-name undefined" ++ Domain ++ + " -remsh " ++ Name), try - ok = send_commands( + ok = rtnode:send_commands( + SNode, STPid, [{putline, ""}, {putline, "node()."}, - {expect, "\\Q" ++ atom_to_list(?FUNCTION_NAME) ++ "\\E"}], 1), - ok = send_commands( + {expect, "\\Q" ++ Name ++ "\\E"}], 1), + ok = rtnode:send_commands( + CNode, CTPid, [{putline, ""}, {putline, "node()."}, - {expect, "\\Q" ++ atom_to_list(?FUNCTION_NAME) ++ "\\E"} | quit_hosting_node()], 1) + {expect, "\\Q" ++ Name ++ "\\E"} | quit_hosting_node()], 1) after - rtnode_dump_logs(rtstop(CState)) + rtnode:dump_logs(rtnode:stop(CState)) end after - rtnode_dump_logs(rtstop(SState)) + rtnode:dump_logs(rtnode:stop(SState)) end; - Else -> + {skip, _} = Else -> Else end. %% Test that -remsh works without epmd. remsh_no_epmd(Config) when is_list(Config) -> - EPMD_ARGS = "-start_epmd false -erl_epmd_port 12345 ", - case rtstart([],"ERL_EPMD_PORT=12345 ", - EPMD_ARGS ++ " -sname " ++ atom_to_list(?FUNCTION_NAME)) of - {ok, _SRPid, STPid, SState} -> + EPMD_ARGS = "-start_epmd false -erl_epmd_port 12346 ", + Name = ?CT_PEER_NAME(), + case rtnode:start([],"ERL_EPMD_PORT=12345 ", + EPMD_ARGS ++ " -sname " ++ Name) of + {ok, _SRPid, STPid, SNode, SState} -> try - ok = send_commands( + ok = rtnode:send_commands( + SNode, STPid, [{putline, ""}, {putline, "node()."}, - {expect, "\\Q" ++ atom_to_list(?FUNCTION_NAME) ++ "\\E"}], 1), - {ok, _CRPid, CTPid, CState} = - rtstart([],"ERL_EPMD_PORT=12345 ", - EPMD_ARGS ++ " -remsh "++atom_to_list(?FUNCTION_NAME)), + {expect, "\\Q" ++ Name ++ "\\E"}], 1), + {ok, _CRPid, CTPid, CNode, CState} = + rtnode:start([],"ERL_EPMD_PORT=12345 ", + EPMD_ARGS ++ " -remsh "++Name), try - ok = send_commands( + ok = rtnode:send_commands( + CNode, CTPid, [{putline, ""}, {putline, "node()."}, - {expect, "\\Q" ++ atom_to_list(?FUNCTION_NAME) ++ "\\E"} | quit_hosting_node()], 1) + {expect, "\\Q" ++ Name ++ "\\E"} | quit_hosting_node()], 1) after - rtstop(CState) + rtnode:dump_logs(rtnode:stop(CState)) end after - rtstop(SState) + rtnode:dump_logs(rtnode:stop(SState)) end; - Else -> + {skip, _} = Else -> Else end. - -rtnode(C) -> - rtnode(C, []). - -rtnode(C, N) -> - rtnode(C, N, []). - -rtnode(Commands, Nodename, ErlPrefix) -> - rtnode(Commands, Nodename, ErlPrefix, []). - -rtnode(Commands, Nodename, ErlPrefix, Args) -> - case rtstart(Nodename, ErlPrefix, Args) of - {ok, _SPid, CPid, RTState} -> - Res = catch send_commands(CPid, Commands, 1), - Logs = rtstop(RTState), - case Res of - ok -> - rtnode_dump_logs(Logs), - ok; - _ -> - rtnode_dump_logs(Logs), - ok = Res - end, - {ok, Logs}; - Skip -> - Skip - end. - -rtstart(Args) -> - rtstart([], [], Args). - -rtstart(Nodename, ErlPrefix, Args) -> - case get_progs() of - {error,_Reason} -> - {skip,"No runerl present"}; - {RunErl,ToErl,Erl} -> - case create_tempdir() of - {error, Reason2} -> - {skip, Reason2}; - Tempdir -> - SPid = - start_runerl_node(RunErl,ErlPrefix++"\\\""++Erl++"\\\"", - Tempdir,Nodename,Args), - CPid = start_toerl_server(ToErl,Tempdir), - {ok, SPid, CPid, {CPid, SPid, ToErl, Tempdir}} - end - end. - -rtstop({CPid, SPid, ToErl, Tempdir}) -> - case stop_runerl_node(CPid) of - {error,_} -> - catch rtstop_try_harder(ToErl, Tempdir); - _ -> - ok - end, - wait_for_runerl_server(SPid), - Logs = rtnode_read_logs(Tempdir), - file:del_dir_r(Tempdir), - Logs. - -rtstop_try_harder(ToErl, Tempdir) -> - CPid = start_toerl_server(ToErl, Tempdir), - ok = send_commands(CPid, - [{putline,[7]}, - {expect, " --> $"}, - {putline, "s"}, - {putline, "c"}, - {putline, ""}], 1), - stop_runerl_node(CPid). - -timeout(longest) -> - timeout(long) + timeout(normal); -timeout(long) -> - 2 * timeout(normal); -timeout(short) -> - timeout(normal) div 10; -timeout(normal) -> - 10000 * test_server:timetrap_scale_factor(). - -start_node(Name, Args0) -> - PaDir = filename:dirname(code:which(?MODULE)), - Args1 = ["-pa",PaDir|Args0], - Args = lists:append(lists:join(" ", Args1)), - {ok, Node} = test_server:start_node(Name, slave, [{args,Args}]), - Node. - -send_commands(CPid, [{sleep, X}|T], N) -> - ?dbg({sleep, X}), - receive - after X -> - send_commands(CPid, T, N+1) - end; -send_commands(CPid, [{expect, Expect}|T], N) when is_list(Expect) -> - ?dbg(Exp), - case command(CPid, {expect, [Expect], timeout(normal)}) of - ok -> - send_commands(CPid, T, N + 1); - {expect_timeout, Got} -> - ct:pal("expect timed out waiting for ~p\ngot: ~p\n", [Expect,Got]), - {error, timeout}; - Other -> - Other - end; -send_commands(CPid, [{putline, Line}|T], N) -> - send_commands(CPid, [{putdata, Line ++ "\n"}|T], N); -send_commands(CPid, [{putdata, Data}|T], N) -> - ?dbg({putdata, Data}), - case command(CPid, {send_data, Data}) of - ok -> - send_commands(CPid, T, N+1); - Error -> - Error - end; -send_commands(_CPid, [], _) -> - ok. - -command(Pid, Req) -> - Timeout = timeout(longest), - Ref = erlang:monitor(process, Pid), - Pid ! {self(), Ref, Req}, - receive - {Ref, Reply} -> - erlang:demonitor(Ref, [flush]), - Reply; - {'DOWN', Ref, _, _, Reason} -> - {error, Reason} - after Timeout -> - io:format("timeout while executing ~p\n", [Req]), - {error, timeout} - end. - -wait_for_runerl_server(SPid) -> - Ref = erlang:monitor(process, SPid), - Timeout = timeout(long), - receive - {'DOWN', Ref, process, SPid, _Reason} -> - ok - after Timeout -> - {error, runerl_server_timeout} - end. - -stop_runerl_node(CPid) -> - Ref = erlang:monitor(process, CPid), - CPid ! {self(), kill_emulator}, - Timeout = timeout(longest), - receive - {'DOWN', Ref, process, CPid, noproc} -> - ok; - {'DOWN', Ref, process, CPid, normal} -> - ok; - {'DOWN', Ref, process, CPid, {error, Reason}} -> - {error, Reason} - after Timeout -> - {error, toerl_server_timeout} - end. - -get_progs() -> - try - do_get_progs() - catch - throw:Thrown -> - {error, Thrown} - end. - -do_get_progs() -> - case os:type() of - {unix,freebsd} -> - throw("Can't use run_erl on FreeBSD"); - {unix,openbsd} -> - throw("Can't use run_erl on OpenBSD"); - {unix,_} -> - RunErl = find_executable("run_erl"), - ToErl = find_executable("to_erl"), - Erl = find_executable("erl"), - {RunErl, ToErl, Erl}; - _ -> - throw("Not a Unix OS") - end. - -find_executable(Name) -> - case os:find_executable(Name) of - Prog when is_list(Prog) -> - Prog; - false -> - throw("Could not find " ++ Name) - end. - -create_tempdir() -> - create_tempdir(filename:join(["/tmp","rtnode"++os:getpid()]),$A). - -create_tempdir(Dir,X) when X > $Z, X < $a -> - create_tempdir(Dir,$a); -create_tempdir(Dir,X) when X > $z -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason eexist", - [Dir++[$z]])), - {error, Estr}; -create_tempdir(Dir0, Ch) -> - %% Expect fairly standard unix. - Dir = Dir0++[Ch], - case file:make_dir(Dir) of - {error, eexist} -> - create_tempdir(Dir0, Ch+1); - {error, Reason} -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason ~p", - [Dir,Reason])), - {error,Estr}; - ok -> - Dir - end. - -create_nodename() -> - create_nodename($A). - -create_nodename(X) when X > $Z, X < $a -> - create_nodename($a); -create_nodename(X) when X > $z -> - {error,out_of_nodenames}; -create_nodename(X) -> - NN = "rtnode"++os:getpid()++[X], - case file:read_file_info(filename:join(["/tmp",NN])) of - {error,enoent} -> - Host = lists:nth(2,string:tokens(atom_to_list(node()),"@")), - NN++"@"++Host; - _ -> - create_nodename(X+1) - end. - - -start_runerl_node(RunErl,Erl,Tempdir,Nodename,Args) -> - XArg = case Nodename of - [] -> - []; - _ -> - " -sname "++(if is_atom(Nodename) -> atom_to_list(Nodename); - true -> Nodename - end)++ - " -setcookie "++atom_to_list(erlang:get_cookie()) - end ++ " " ++ Args, - spawn(fun() -> start_runerl_command(RunErl, Tempdir, Erl++XArg) end). - -start_runerl_command(RunErl, Tempdir, Cmd) -> - FullCmd = "\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++" \""++Cmd++"\"", - ct:pal("~ts",[FullCmd]), - os:cmd(FullCmd). - -start_toerl_server(ToErl,Tempdir) -> - Pid = spawn(?MODULE,toerl_server,[self(),ToErl,Tempdir]), - receive - {Pid,started} -> - Pid; - {Pid,error,Reason} -> - {error,Reason} - end. - -try_to_erl(_Command, 0) -> - {error, cannot_to_erl}; -try_to_erl(Command, N) -> - ?dbg({?LINE,N}), - Port = open_port({spawn, Command},[eof]), - Timeout = timeout(short) div 2, - receive - {Port, eof} -> - timer:sleep(Timeout), - try_to_erl(Command, N-1) - after Timeout -> - ?dbg(Port), - Port - end. - -toerl_server(Parent, ToErl, TempDir) -> - Port = try_to_erl("\""++ToErl++"\" "++TempDir++"/ 2>/dev/null", 8), - case Port of - P when is_port(P) -> - Parent ! {self(),started}; - {error,Other} -> - Parent ! {self(),error,Other}, - exit(Other) - end, - - State = #{port => Port, acc => [], kill_emulator_command => init_stop}, - case toerl_loop(State) of - normal -> - ok; - {error, Reason} -> - error_logger:error_msg("toerl_server exit with reason ~p~n", - [Reason]), - exit(Reason) - end. - -toerl_loop(#{port := Port} = State0) -> - ?dbg({toerl_loop, Port, map_get(acc, State0), - maps:get(match, State0, nomatch)}), - - State = handle_expect(State0), - - receive - {Port,{data,Data}} when is_port(Port) -> - ?dbg({?LINE,Port,{data,Data}}), - toerl_loop(State#{acc => map_get(acc, State) ++ Data}); - {Pid, Ref, {expect, Expect, Timeout}} -> - toerl_loop(init_expect(Pid, Ref, Expect, Timeout, State)); - {Pid, Ref, {send_data, Data}} -> - Port ! {self(), {command, Data}}, - Pid ! {Ref, ok}, - toerl_loop(State); - {_Pid, kill_emulator} -> - kill_emulator(State); - {timeout,Timer,expect_timeout} -> - toerl_loop(handle_expect_timeout(Timer, State)); - {Port, eof} -> - {error, unexpected_eof}; - Other -> - {error, {unexpected, Other}} - end. - -kill_emulator(#{port := Port}) -> - %% If the line happens to end in a ".", issuing "init:stop()." - %% will result in a syntax error. To avoid that, issue a "\n" - %% before "init:stop().". - Port ! {self(),{command, "\ninit:stop().\n"}}, - wait_for_eof(Port). - -wait_for_eof(Port) -> - receive - {Port,eof} -> - normal; - _Other -> - wait_for_eof(Port) - after - timeout(long) -> - {error, kill_timeout} - end. - -init_expect(Pid, Ref, ExpectList, Timeout, State) -> - try compile_expect(ExpectList) of - Expect -> - Exp = #{expect => Expect, - ref => Ref, - source => ExpectList, - timer => erlang:start_timer(Timeout, self(), expect_timeout), - from => Pid}, - State#{expect => Exp} - catch - Class:Reason:Stk -> - io:put_chars("Compilation of expect pattern failed:"), - io:format("~p\n", [ExpectList]), - io:put_chars(erl_error:format_exception(Class, Reason, Stk)), - exit(expect_pattern_error) - end. - -handle_expect(#{acc := Acc, expect := Exp} = State) -> - #{expect := Expect, from := Pid, ref := Ref} = Exp, - case Expect(Acc) of - nomatch -> - State; - {matched, Eaten, Result} -> - Pid ! {Ref, Result}, - finish_expect(Eaten, State) - end; -handle_expect(State) -> - State. - -handle_expect_timeout(Timer, State) -> - #{acc := Acc, expect := Exp} = State, - #{expect := Expect, timer := Timer, from := Pid, ref := Ref} = Exp, - case Expect({timeout, Acc}) of - nomatch -> - Result = {expect_timeout, Acc}, - Pid ! {Ref, Result}, - finish_expect(0, State); - {matched, Eaten, Result} -> - Pid ! {Ref, Result}, - finish_expect(Eaten, State) +remsh_expand_compatibility_25(Config) when is_list(Config) -> + {ok, _Peer, TargetNode} = ?CT_PEER(#{}), %% Create a vsn 26 node + NodeName = atom_to_list(TargetNode), %% compatibility + %% Start a node on vsn 25 but run the shell on vsn 26 + case rtnode:start(peer:random_name(), "stty columns 200; ERL_AFLAGS= ", "-remsh "++NodeName, [{release, "25"}|Config]) of + {ok, _SRPid, STPid, _, SState} -> + try + ok = rtnode:send_commands(undefined, + STPid, + [{putdata, "erlang:is_atom\t"}, + {expect, "\\Qerlang:is_atom(\\E"}| + quit_hosting_node()], 1) + after + Logs = rtnode:stop(SState), + rtnode:dump_logs(Logs) + end; + Else when element(1, Else) =/= ok -> Else end. - -finish_expect(Eaten, #{acc := Acc0, - expect := #{timer := Timer}}=State) -> - erlang:cancel_timer(Timer), - receive - {timeout,Timer,timeout} -> - ok - after 0 -> - ok - end, - Acc = lists:nthtail(Eaten, Acc0), - maps:remove(expect, State#{acc := Acc}). - -compile_expect([{timeout,Action}|T]) when is_function(Action, 1) -> - Next = compile_expect(T), - fun({timeout, _}=Tm) -> - {matched, 0, Action(Tm)}; - (Subject) -> - Next(Subject) - end; -compile_expect([{{re,RE0},Action}|T]) when is_binary(RE0), is_function(Action, 1) -> - {ok, RE} = re:compile(RE0), - Next = compile_expect(T), - fun({timeout, _}=Subject) -> - Next(Subject); - (Subject) -> - case re:run(Subject, RE, [{capture,first,index}]) of - nomatch -> - Next(Subject); - {match, [{Pos,Len}]} -> - Matched = binary:part(list_to_binary(Subject), Pos, Len), - {matched, Pos+Len, Action(Matched)} +remsh_expand_compatibility_later_version(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + case ?CT_PEER([], "25", PrivDir) of + not_available -> {skip, "25 not available"}; + {ok, _Peer, TargetNode} -> + NodeName = atom_to_list(TargetNode), + %% Start a node on later version but run the shell on vsn 25 + case rtnode:start(peer:random_name(), "stty columns 200; ERL_AFLAGS= ", "-remsh " ++ NodeName, Config) of + {ok, _SRPid, STPid, _, SState} -> + try + ok = rtnode:send_commands(undefined, + STPid, + [{putdata, "erlang:is_atom\t"}, + {expect, "\\Qerlang:is_atom(\\E"}| + quit_hosting_node()], 1) + after + Logs = rtnode:stop(SState), + rtnode:dump_logs(Logs) + end; + Else when element(1, Else) =/= ok -> Else end - end; -compile_expect([RE|T]) when is_list(RE) -> - Ok = fun(_) -> ok end, - compile_expect([{{re,list_to_binary(RE)},Ok}|T]); -compile_expect([]) -> - fun(_) -> - nomatch - end. - -rtnode_check_logs(Logname, Pattern, Logs) -> -rtnode_check_logs(Logname, Pattern, true, Logs). -rtnode_check_logs(Logname, Pattern, Match, Logs) -> - case re:run(maps:get(Logname, Logs), Pattern) of - {match, [_]} when Match -> - ok; - nomatch when not Match -> - ok; - _ -> - rtnode_dump_logs(Logs), - ct:fail("~p not found in log ~ts",[Pattern, Logname]) - end. - -rtnode_dump_logs(Logs) -> - maps:foreach( - fun(File, Data) -> - ct:pal("~ts: ~ts",[File, Data]) - end, Logs). - -rtnode_read_logs(Tempdir) -> - {ok, LogFiles0} = file:list_dir(Tempdir), - - %% Make sure that we only read log files and not any named pipes. - LogFiles = [F || F <- LogFiles0, - case F of - "erlang.log" ++ _ -> true; - _ -> false - end], - - lists:foldl( - fun(File, Acc) -> - case file:read_file(filename:join(Tempdir, File)) of - {ok, Data} -> - Acc#{ File => Data }; - _ -> - Acc - end - end, #{}, LogFiles). - -get_default_shell() -> - try - rtnode([{putline,""}, - {putline, "is_pid(whereis(user_drv))."}, - {expect, "true\r\n"}], []), - new - catch _E:_R -> - ?dbg({_E,_R}), - old end. printed_atom(A) -> diff --git a/lib/kernel/test/logger_formatter_SUITE.erl b/lib/kernel/test/logger_formatter_SUITE.erl index f1c64110a1..d160dce32f 100644 --- a/lib/kernel/test/logger_formatter_SUITE.erl +++ b/lib/kernel/test/logger_formatter_SUITE.erl @@ -272,7 +272,7 @@ template(_Config) -> [[nested,subkey]]), String8 = format(info,{"~p",[term]},Meta8,#{template=>Template8, single_line=>true}), - ct:log(String6), + ct:log(String8), SelfStr = pid_to_list(self()), RefStr8 = ref_to_list(Ref8), ListStr = "[list,\"string\",4321,#{},{tuple}]", @@ -345,6 +345,18 @@ template(_Config) -> _ -> ct:fail({full_nested_map_unexpected,MultipleKeysStr10}) end, + Meta11A = #{time=>Time,be_short=>ok}, + Meta11B = #{time=>Time}, + Template11 = + [{be_short, + ["short:",msg], + ["long:[",level,"]",msg]}], + String11A = format(info,{"~p",[term]},Meta11A,#{template=>Template11,single_line=>true}), + String11B = format(info,{"~p",[term]},Meta11B,#{template=>Template11,single_line=>true}), + ct:log(String11A), + ct:log(String11B), + {"short:term","long:[info]term"} = {String11A,String11B}, + ok. format_msg(_Config) -> diff --git a/lib/kernel/test/logger_simple_h_SUITE.erl b/lib/kernel/test/logger_simple_h_SUITE.erl index 8ac52e54c7..9bc910a04b 100644 --- a/lib/kernel/test/logger_simple_h_SUITE.erl +++ b/lib/kernel/test/logger_simple_h_SUITE.erl @@ -80,6 +80,7 @@ groups() -> all() -> [start_stop, + start_crash, replace_default, replace_file, replace_disk_log @@ -101,6 +102,21 @@ start_stop(_Config) -> start_stop(cleanup,_Config) -> logger:remove_handler(simple). +%% Test that the simple logger works during startup crash +start_crash(_Config) -> + + Output = os:cmd(ct:get_progname() ++ " -user baduser"), + ErrorOutput = re:replace(unicode:characters_to_binary(Output),"\r\n","\n",[global]), + ct:log("~ts",[ErrorOutput]), + {match,[_]} = re:run(ErrorOutput,"^(=SUPERVISOR REPORT====| supervisor_report *\n)",[global]), + {match,[_, _]} = re:run(ErrorOutput," crash_report *\n",[global]), + {match,[_]} = re:run(ErrorOutput," std_info *\n",[global]), + {match,[[CD]]} = re:run(ErrorOutput,"\nCrash dump is being written to: (.*)\\.\\.\\.done", + [{capture, all_but_first, binary}, global]), + ok = file:delete(CD), + ok. + + %% This testcase just tests that it does not crash, the default handler prints %% to stdout which we cannot read from in a detached slave. replace_default(Config) -> diff --git a/lib/kernel/test/rtnode.erl b/lib/kernel/test/rtnode.erl new file mode 100644 index 0000000000..ffa0613ca7 --- /dev/null +++ b/lib/kernel/test/rtnode.erl @@ -0,0 +1,575 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2022. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(rtnode). + +-export([run/1, run/2, run/3, run/4, start/1, start/3, start/4, send_commands/4, stop/1, + start_runerl_command/3, + check_logs/3, check_logs/4, read_logs/1, dump_logs/1, + get_default_shell/0, get_progs/0, create_tempdir/0, timeout/1]). + +-include_lib("common_test/include/ct.hrl"). + +%% -define(debug, true). + +-ifdef(debug). +-define(dbg(Data),io:format(standard_error, "DBG: ~p\r\n",[Data])). +-else. +-define(dbg(Data),noop). +-endif. + +-export([toerl_server/4]). + +%% +%% Tool for running interactive shell, used by interactive_shell and io_proto SUITE +%% +run(C) -> + run(C, [], [], []). + +run(C, N) -> + run(C, N, [], []). + +run(Commands, Nodename, ErlPrefix) -> + run(Commands, Nodename, ErlPrefix, []). + +run(Commands, Nodename, ErlPrefix, Args) -> + case start(Nodename, ErlPrefix, Args) of + {ok, _SPid, CPid, Node, RTState} -> + Res = catch send_commands(Node, CPid, Commands, 1), + Logs = stop(RTState), + case Res of + ok -> + dump_logs(Logs), + ok; + _ -> + dump_logs(Logs), + ok = Res + end, + {ok, Logs}; + Skip -> + Skip + end. + +start(Args) -> + start([], " ", Args, []). +start(Nodename, ErlPrefix, Args) -> + start(Nodename, ErlPrefix, Args, []). +start(Nodename, ErlPrefix, Args, Options) -> + case get_progs(Options) of + {error,Reason} -> + {skip,Reason}; + {RunErl,ToErl,[Erl|ErlArgs] = ErlWArgs} -> + case create_tempdir() of + {error, Reason2} -> + {skip, Reason2}; + Tempdir when ErlPrefix =/= [] -> + {SPid, Node} = + start_runerl_node(RunErl, + ErlPrefix++"\\\""++Erl++"\\\" "++ + lists:join($\s, ErlArgs), + Tempdir,Nodename,Args), + CPid = start_toerl_server(ToErl,Tempdir,undefined), + {ok, SPid, CPid, Node, {CPid, SPid, ToErl, Tempdir}}; + Tempdir -> + {SPid, Node} = start_peer_runerl_node(RunErl,ErlWArgs,Tempdir,Nodename,Args), + CPid = start_toerl_server(ToErl,Tempdir,SPid), + {ok, SPid, CPid, Node, {CPid, SPid, ToErl, Tempdir}} + end + end. + +stop({CPid, SPid, ToErl, Tempdir}) -> + %% Unlink from peer so that we don't crash when peer quits + unlink(SPid), + case stop_runerl_node(CPid) of + {error,_} -> + catch stop_try_harder(ToErl, Tempdir, SPid); + _ -> + ok + end, + wait_for_runerl_server(SPid), + Logs = read_logs(Tempdir), + file:del_dir_r(Tempdir), + Logs. + +stop_try_harder(ToErl, Tempdir, SPid) -> + CPid = start_toerl_server(ToErl, Tempdir, SPid), + ok = send_commands(undefined, CPid, + [{putline,[7]}, + {expect, " --> $"}, + {putline, "s"}, + {putline, "c"}, + {putline, ""}], 1), + stop_runerl_node(CPid). + +timeout(longest) -> + timeout(long) + timeout(normal); +timeout(long) -> + 2 * timeout(normal); +timeout(short) -> + timeout(normal) div 10; +timeout(normal) -> + 10000 * test_server:timetrap_scale_factor(). + +send_commands(Node, CPid, [{sleep, X}|T], N) -> + ?dbg({sleep, X}), + receive + after X -> + send_commands(Node, CPid, T, N+1) + end; +send_commands(Node, CPid, [{expect, Expect}|T], N) when is_list(Expect) -> + send_commands(Node, CPid, [{expect, unicode, Expect}|T], N); +send_commands(Node, CPid, [{expect, Encoding, Expect}|T], N) when is_list(Expect) -> + ?dbg({expect, Expect}), + case command(CPid, {expect, Encoding, [Expect], timeout(normal)}) of + ok -> + send_commands(Node, CPid, T, N + 1); + {expect_timeout, Got} -> + ct:pal("expect timed out waiting for ~p\ngot: ~p\n", [Expect,Got]), + {error, timeout}; + Other -> + Other + end; +send_commands(Node, CPid, [{putline, Line}|T], N) -> + send_commands(Node, CPid, [{putdata, Line ++ "\n"}|T], N); +send_commands(Node, CPid, [{putdata, Data}|T], N) -> + ?dbg({putdata, Data}), + case command(CPid, {send_data, Data}) of + ok -> + send_commands(Node, CPid, T, N+1); + Error -> + Error + end; +send_commands(Node, CPid, [{eval, Fun}|T], N) -> + ?dbg({eval, Node, Fun}), + case erpc:call(Node, Fun) of + ok -> + ?dbg({eval, ok}), + send_commands(Node, CPid, T, N+1); + Error -> + ?dbg({eval, Error}), + Error + end; +send_commands(_Node, _CPid, [], _) -> + ok. + +command(Pid, Req) -> + Timeout = timeout(longest), + Ref = erlang:monitor(process, Pid), + Pid ! {self(), Ref, Req}, + receive + {Ref, Reply} -> + erlang:demonitor(Ref, [flush]), + Reply; + {'DOWN', Ref, _, _, Reason} -> + {error, Reason} + after Timeout -> + io:format("timeout while executing ~p\n", [Req]), + {error, timeout} + end. + +wait_for_runerl_server(SPid) -> + Ref = erlang:monitor(process, SPid), + Timeout = timeout(long), + receive + {'DOWN', Ref, process, SPid, _Reason} -> + ok + after Timeout -> + {error, runerl_server_timeout} + end. + +stop_runerl_node(CPid) -> + Ref = erlang:monitor(process, CPid), + CPid ! {self(), kill_emulator}, + Timeout = timeout(longest), + receive + {'DOWN', Ref, process, CPid, noproc} -> + ok; + {'DOWN', Ref, process, CPid, normal} -> + ok; + {'DOWN', Ref, process, CPid, {error, Reason}} -> + {error, Reason} + after Timeout -> + {error, toerl_server_timeout} + end. + +get_progs() -> + case os:type() of + {unix,freebsd} -> + {error,"Can't use run_erl on FreeBSD"}; + {unix,openbsd} -> + {error,"Can't use run_erl on OpenBSD"}; + {unix,_} -> + RunErl = find_executable("run_erl"), + ToErl = find_executable("to_erl"), + Erl = string:split(ct:get_progname()," ",all), + {RunErl, ToErl, Erl}; + _ -> + {error,"Not a Unix OS"} + end. +get_progs(Opts) -> + case get_progs() of + {RunErl, ToErl, Erl} -> + case proplists:get_value(release, Opts) of + undefined -> {RunErl, ToErl, Erl}; + Release -> + case test_server_node:find_release(Release) of + none -> {error, "Could not find release "++Release}; + R -> {RunErl, ToErl, [R]} + end + end; + E -> E + end. + + +find_executable(Name) -> + case os:find_executable(Name) of + Prog when is_list(Prog) -> + Prog; + false -> + throw("Could not find " ++ Name) + end. + +create_tempdir() -> + create_tempdir(filename:join(["/tmp","rtnode"++os:getpid()]),$A). + +create_tempdir(Dir,X) when X > $Z, X < $a -> + create_tempdir(Dir,$a); +create_tempdir(Dir,X) when X > $z -> + Estr = lists:flatten( + io_lib:format("Unable to create ~s, reason eexist", + [Dir++[$z]])), + {error, Estr}; +create_tempdir(Dir0, Ch) -> + %% Expect fairly standard unix. + Dir = Dir0++[Ch], + case file:make_dir(Dir) of + {error, eexist} -> + create_tempdir(Dir0, Ch+1); + {error, Reason} -> + Estr = lists:flatten( + io_lib:format("Unable to create ~s, reason ~p", + [Dir,Reason])), + {error,Estr}; + ok -> + Dir + end. + +start_runerl_node(RunErl,Erl,Tempdir,Nodename,Args) -> + {XArg, Node} = + case Nodename of + [] -> + {[], undefined}; + _ -> + NodenameStr = if is_atom(Nodename) -> atom_to_list(Nodename); + true -> Nodename + end, + [_Name,Host] = string:split(atom_to_list(node()), "@"), + {" -sname "++ NodenameStr ++ + " -setcookie "++atom_to_list(erlang:get_cookie()), + list_to_atom(NodenameStr ++ "@" ++ Host)} + end, + {spawn(fun() -> start_runerl_command(RunErl, Tempdir, Erl ++ XArg ++ " " ++ Args) end), + Node}. + +start_runerl_command(RunErl, Tempdir, Cmd) -> + FullCmd = "\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++" \""++Cmd++"\"", + ct:pal("~ts",[FullCmd]), + os:cmd(FullCmd). + +start_peer_runerl_node(RunErl,Erl,Tempdir,[],Args) -> + start_peer_runerl_node(RunErl,Erl,Tempdir,peer:random_name(),Args); +start_peer_runerl_node(RunErl,Erl,Tempdir,Nodename,Args) -> + {ok, Peer, Node} = + ?CT_PEER(#{ name => Nodename, + exec => {RunErl,Erl}, + detached => false, + shutdown => 10000, + post_process_args => + fun(As) -> + [Tempdir++"/",Tempdir, + lists:flatten( + lists:join( + " ",[[$',A,$'] || A <- As]))] + end, + args => ["-connect_all","false"|Args] }), + Self = self(), + TraceLog = filename:join(Tempdir,Nodename++".trace"), + ct:pal("Link to trace: file://~ts",[TraceLog]), + + spawn(Node, + fun() -> + try + %% {ok, _} = dbg:tracer(file, TraceLog), + %% dbg:p(whereis(user_drv),[c,m,timestamp]), + %% dbg:p(whereis(user_drv_reader),[c,m,timestamp]), + %% dbg:p(whereis(user_drv_writer),[c,m,timestamp]), + %% dbg:p(whereis(user),[c,m,timestamp]), + %% dbg:tp(user_drv,x), + %% dbg:tp(prim_tty,x), + %% dbg:tpl(prim_tty,read_nif,x), + Ref = monitor(process, Self), + receive {'DOWN',Ref,_,_,_} -> ok end + catch E:R:ST -> + io:format(user,"~p:~p:~p",[E,R,ST]), + erlang:raise(E,R,ST) + end + end), + {Peer, Node}. + +start_toerl_server(ToErl,Tempdir,SPid) -> + Pid = spawn(?MODULE,toerl_server,[self(),ToErl,Tempdir,SPid]), + receive + {Pid,started} -> + Pid; + {Pid,error,Reason} -> + {error,Reason} + end. + +try_to_erl(_Command, 0) -> + {error, cannot_to_erl}; +try_to_erl(Command, N) -> + ?dbg({?LINE,N}), + Port = open_port({spawn, Command},[eof]), + Timeout = timeout(short) div 2, + receive + {Port, eof} -> + timer:sleep(Timeout), + try_to_erl(Command, N-1) + after Timeout -> + ?dbg(Port), + Port + end. + +toerl_server(Parent, ToErl, TempDir, SPid) -> + Port = try_to_erl("\""++ToErl++"\" "++TempDir++"/ 2>/dev/null", 8), + case Port of + P when is_port(P) -> + Parent ! {self(),started}; + {error,Other} -> + Parent ! {self(),error,Other}, + exit(Other) + end, + + {ok, InitialData} = file:read_file(filename:join(TempDir,"erlang.log.1")), + + State = #{port => Port, acc => unicode:characters_to_list(InitialData), spid => SPid}, + case toerl_loop(State) of + normal -> + ok; + {error, Reason} -> + error_logger:error_msg("toerl_server exit with reason ~p~n", + [Reason]), + exit(Reason) + end. + +toerl_loop(#{port := Port} = State0) -> + ?dbg({toerl_loop, Port, map_get(acc, State0), + maps:get(match, State0, nomatch)}), + + State = handle_expect(State0), + + receive + {Port,{data,Data}} when is_port(Port) -> + ?dbg({?LINE,Port,{data,Data}}), + toerl_loop(State#{acc => map_get(acc, State) ++ Data}); + {Pid, Ref, {expect, Encoding, Expect, Timeout}} -> + toerl_loop(init_expect(Pid, Ref, Encoding, Expect, Timeout, State)); + {Pid, Ref, {send_data, Data}} -> + ?dbg({?LINE,Port,{send_data,Data}}), + Port ! {self(), {command, Data}}, + Pid ! {Ref, ok}, + toerl_loop(State); + {_Pid, kill_emulator} -> + kill_emulator(State); + {timeout,Timer,expect_timeout} -> + toerl_loop(handle_expect_timeout(Timer, State)); + {Port, eof} -> + {error, unexpected_eof}; + Other -> + {error, {unexpected, Other}} + end. + +kill_emulator(#{spid := SPid, port := Port}) when is_pid(SPid) -> + catch peer:stop(SPid), + wait_for_eof(Port); +kill_emulator(#{port := Port}) -> + %% If the line happens to end in a ".", issuing "init:stop()." + %% will result in a syntax error. To avoid that, issue a "\n" + %% before "init:stop().". + Port ! {self(),{command, "\ninit:stop().\n"}}, + wait_for_eof(Port). + +wait_for_eof(Port) -> + receive + {Port,eof} -> + normal; + _Other -> + wait_for_eof(Port) + after + timeout(long) -> + {error, kill_timeout} + end. + +init_expect(Pid, Ref, Encoding, ExpectList, Timeout, State) -> + try compile_expect(ExpectList, Encoding) of + Expect -> + Exp = #{expect => Expect, + ref => Ref, + source => ExpectList, + timer => erlang:start_timer(Timeout, self(), expect_timeout), + from => Pid}, + State#{expect => Exp} + catch + Class:Reason:Stk -> + io:put_chars("Compilation of expect pattern failed:"), + io:format("~p\n", [ExpectList]), + io:put_chars(erl_error:format_exception(Class, Reason, Stk)), + exit(expect_pattern_error) + end. + +handle_expect(#{acc := Acc, expect := Exp} = State) -> + #{expect := Expect, from := Pid, ref := Ref} = Exp, + case Expect(Acc) of + nomatch -> + State; + {matched, Eaten, Result} -> + ?dbg({matched, Eaten, Result}), + Pid ! {Ref, Result}, + finish_expect(Eaten, State) + end; +handle_expect(State) -> + State. + +handle_expect_timeout(Timer, State) -> + #{acc := Acc, expect := Exp} = State, + #{expect := Expect, timer := Timer, from := Pid, ref := Ref} = Exp, + case Expect({timeout, Acc}) of + nomatch -> + Result = {expect_timeout, Acc}, + Pid ! {Ref, Result}, + finish_expect(0, State); + {matched, Eaten, Result} -> + Pid ! {Ref, Result}, + finish_expect(Eaten, State) + end. + +finish_expect(Eaten, #{acc := Acc0, + expect := #{timer := Timer}}=State) -> + erlang:cancel_timer(Timer), + receive + {timeout,Timer,timeout} -> + ok + after 0 -> + ok + end, + Acc = lists:nthtail(Eaten, Acc0), + maps:remove(expect, State#{acc := Acc}). + +compile_expect([{timeout,Action}|T], E) when is_function(Action, 1) -> + Next = compile_expect(T, E), + fun({timeout, _}=Tm) -> + {matched, 0, Action(Tm)}; + (Subject) -> + Next(Subject) + end; +compile_expect([{{re,RE0},Action}|T], E) when is_binary(RE0), is_function(Action, 1) -> + {ok, RE} = re:compile(RE0, [unicode || E =:= unicode]), + Next = compile_expect(T, E), + fun({timeout, _}=Subject) -> + Next(Subject); + (Subject) -> + BinarySubject = if + E =:= unicode -> + unicode:characters_to_binary(list_to_binary(Subject)); + E =:= latin1 -> + list_to_binary(Subject) + end, + case re:run(BinarySubject, RE, [{capture,first,index}]) of + nomatch -> + Next(Subject); + {match, [{Pos,Len}]} -> + Matched = binary:part(BinarySubject, Pos, Len), + {matched, Pos+Len, Action(Matched)} + end + end; +compile_expect([RE|T], E) when is_list(RE) -> + Ok = fun(_) -> ok end, + compile_expect([{{re,unicode:characters_to_binary(RE, unicode, E)},Ok}|T], E); +compile_expect([], _E) -> + fun(_) -> + nomatch + end. + +check_logs(Logname, Pattern, Logs) -> + check_logs(Logname, Pattern, true, Logs). +check_logs(Logname, Pattern, Match, Logs) -> + case re:run(maps:get(Logname, Logs), Pattern) of + {match, [_]} when Match -> + ok; + nomatch when not Match -> + ok; + _ -> + dump_logs(Logs), + ct:fail("~p not found in log ~ts",[Pattern, Logname]) + end. + +dump_logs(Logs) -> + maps:foreach( + fun(File, Data) -> + try re:replace(Data,"\e","\\\\e",[unicode,global]) of + D -> ct:pal("~ts: ~ts",[File, D]) + catch error:badarg -> + ct:pal("~ts: ~s",[File, re:replace(Data,"\e","\\\\e",[global])]) + end + end, Logs). + +read_logs(Tempdir) -> + {ok, LogFiles0} = file:list_dir(Tempdir), + + %% Make sure that we only read log files and not any named pipes. + LogFiles = [F || F <- LogFiles0, + case F of + "erlang.log" ++ _ -> true; + _ -> false + end], + + lists:foldl( + fun(File, Acc) -> + case file:read_file(filename:join(Tempdir, File)) of + {ok, Data} -> + Acc#{ File => Data }; + _ -> + Acc + end + end, #{}, LogFiles). + +get_default_shell() -> + case get_progs() of + {error,_} -> + noshell; + _ -> + try + run([{putline,""}, + {putline, "is_pid(whereis(user_drv))."}, + {expect, "true\r\n"}]), + new + catch _E:_R -> + old + end + end. diff --git a/lib/kernel/test/standard_error_SUITE.erl b/lib/kernel/test/standard_error_SUITE.erl index 1d9026dc58..34bb880db8 100644 --- a/lib/kernel/test/standard_error_SUITE.erl +++ b/lib/kernel/test/standard_error_SUITE.erl @@ -34,8 +34,10 @@ badarg(Config) when is_list(Config) -> true = erlang:is_process_alive(whereis(standard_error)), ok. +%% Check that standard_out and standard_error have the same encoding getopts(Config) when is_list(Config) -> - [{encoding,latin1}] = io:getopts(standard_error), + Encoding = proplists:get_value(encoding, io:getopts(user)), + Encoding = proplists:get_value(encoding, io:getopts(standard_error)), ok. %% Test that writing a lot of output to standard_error does not cause the diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index 847cd0074b..e13738d535 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -32,6 +32,7 @@ do_update_op/3, get_info/1, get_transactions/0, + get_transactions_count/0, info/1, mnesia_down/1, prepare_checkpoint/2, @@ -406,6 +407,10 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= reply(From, {info, gb_trees:values(Participants), gb_trees:to_list(Coordinators)}, State); + {From, transactions_count} -> + reply(From, {transactions_count, gb_trees:size(Participants), + gb_trees:size(Coordinators)}, State); + {mnesia_down, N} -> verbose("Got mnesia_down from ~p, reconfiguring...~n", [N]), reconfigure_coordinators(N, gb_trees:to_list(Coordinators)), @@ -2121,6 +2126,14 @@ tr_status(Tid,Participant) -> false -> coordinator end. +get_transactions_count() -> + case req(transactions_count) of + {transactions_count, ParticipantsCount, CoordinatorsCount} -> + {ParticipantsCount, CoordinatorsCount}; + Error -> + Error + end. + get_info(Timeout) -> case whereis(?MODULE) of undefined -> diff --git a/lib/mnesia/test/mnesia_trans_access_test.erl b/lib/mnesia/test/mnesia_trans_access_test.erl index b33b807bfc..d5bbaf9729 100644 --- a/lib/mnesia/test/mnesia_trans_access_test.erl +++ b/lib/mnesia/test/mnesia_trans_access_test.erl @@ -28,7 +28,7 @@ -export([write/1, read/1, wread/1, delete/1, delete_object_bag/1, delete_object_set/1, - match_object/1, select/1, select14/1, all_keys/1, transaction/1, + match_object/1, select/1, select14/1, all_keys/1, transaction/1, transaction_counters/1, basic_nested/1, mix_of_nested_activities/1, nested_trans_both_ok/1, nested_trans_child_dies/1, nested_trans_parent_dies/1, nested_trans_both_dies/1, @@ -65,7 +65,7 @@ end_per_testcase(Func, Conf) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% all() -> [write, read, wread, delete, delete_object_bag, delete_object_set, - match_object, select, select14, all_keys, transaction, + match_object, select, select14, all_keys, transaction, transaction_counters, {group, nested_activities}, {group, index_tabs}, {group, index_lifecycle}]. @@ -546,6 +546,28 @@ transaction(Config) when is_list(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +transaction_counters(suite) -> []; +transaction_counters(Config) -> + Nodes = ?acquire_nodes(1, Config), + + {atomic, {{Participants1, Coordinators1}, {Participants2, Coordinators2}}} = + mnesia:transaction(fun get_transactions_counters/0), + + ?match(Coordinators1, Coordinators2), + ?match(Coordinators1, 1), + ?match(Participants1, Participants2), + ?match(Participants1, 0), + + ?verify_mnesia(Nodes, []). + +get_transactions_counters() -> + {count_sides(mnesia_tm:get_transactions()), mnesia_tm:get_transactions_count()}. + +count_sides(TransactionsList) -> + lists:foldl( + fun({_Tid, _Pid, participant}, {Participants, Coordinators}) -> {Participants + 1, Coordinators}; + ({_Tid, _Pid, coordinator}, {Participants, Coordinators}) -> {Participants, Coordinators + 1} + end, {0, 0}, TransactionsList). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/os_mon/c_src/win32sysinfo.c b/lib/os_mon/c_src/win32sysinfo.c index 1c9590fb8b..85622e0076 100644 --- a/lib/os_mon/c_src/win32sysinfo.c +++ b/lib/os_mon/c_src/win32sysinfo.c @@ -265,31 +265,30 @@ message_loop() print_error("Erlang has closed"); return; } + if ((res = read(0, &cmd, cmdLen)) == cmdLen){ if (cmdLen == 1) { - switch (cmd[0]) { - case MEM_INFO: - get_avail_mem_ext(); - return_answer(OK); - break; - case DISK_INFO: - get_disk_info_all(); - return_answer(OK); - break; - default: /* ignore all other messages */ - break; - } /* switch */ + switch (cmd[0]) { + case MEM_INFO: + get_avail_mem_ext(); + return_answer(OK); + break; + case DISK_INFO: + get_disk_info_all(); + return_answer(OK); + break; + default: /* ignore all other messages */ + break; + } /* switch */ } - else - if ((res > 0) && (cmd[0]==DISK_INFO)) { - cmd[cmdLen] = 0; - output_drive_info(&cmd[1]); - return_answer("OK"); - return; - } - else - return_answer("xEND"); - } + else { + if ((res > 0) && (cmd[0]==DISK_INFO)) { + cmd[cmdLen] = 0; + output_drive_info(&cmd[1]); + } + return_answer(OK); + } + } else if (res == 0) { print_error("Erlang has closed"); return; @@ -297,8 +296,8 @@ message_loop() else { print_error("Error reading from Erlang"); return; - } - } + } + } } int main(int argc, char ** argv){ diff --git a/lib/os_mon/src/os_mon_sysinfo.erl b/lib/os_mon/src/os_mon_sysinfo.erl index 4f5a682a1f..662d984dfd 100644 --- a/lib/os_mon/src/os_mon_sysinfo.erl +++ b/lib/os_mon/src/os_mon_sysinfo.erl @@ -45,7 +45,7 @@ get_disk_info() -> gen_server:call(os_mon_sysinfo, get_disk_info). get_disk_info(DriveRoot) -> - gen_server:call(os_mon_sysinfo, {get_disk_info,DriveRoot}). + gen_server:call(os_mon_sysinfo, {get_disk_info, DriveRoot}). get_mem_info() -> gen_server:call(os_mon_sysinfo, get_mem_info). @@ -65,8 +65,8 @@ init([]) -> handle_call(get_disk_info, _From, State) -> {reply, get_disk_info1(State#state.port), State}; -handle_call({get_disk_info,RootList}, _From, State) -> - {reply, get_disk_info1(State#state.port,RootList), State}; +handle_call({get_disk_info, DriveRoot}, _From, State) -> + {reply, get_disk_info1(State#state.port, DriveRoot), State}; handle_call(get_mem_info, _From, State) -> {reply, get_mem_info1(State#state.port), State}. @@ -108,8 +108,8 @@ get_disk_info1(Port) -> Port ! {self(),{command,[?DISK_INFO]}}, get_data(Port,[]). -get_disk_info1(Port,PathList) -> - Port ! {self(),{command,[?DISK_INFO|[P++[0]||P <- PathList]]}}, +get_disk_info1(Port, DriveRoot) -> + Port ! {self(), {command,[?DISK_INFO|DriveRoot++[0]]}}, get_data(Port,[]). get_mem_info1(Port) -> diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index ae7a907a60..8d7e44629c 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -1313,7 +1313,7 @@ extract(File, Ts) -> search_for_file_attr(PartialFilePathRegex, Forms) -> lists:search(fun ({attribute, _, file, {FileAttr, _}}) -> - case re:run(FileAttr, PartialFilePathRegex) of + case re:run(FileAttr, PartialFilePathRegex, [unicode]) of nomatch -> false; _ -> true end; diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index e76b98f0f5..383969e7a8 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -2326,7 +2326,7 @@ safe_second_element(Other) -> Other. search_for_file_attr(PartialFilePathRegex, Forms) -> lists:search(fun ({attribute, _, file, {FileAttr, _}}) -> - case re:run(FileAttr, PartialFilePathRegex) of + case re:run(FileAttr, PartialFilePathRegex, [unicode]) of nomatch -> false; _ -> true end; diff --git a/lib/public_key/.gitignore b/lib/public_key/.gitignore index 8d0ebe018f..bd43ec2abd 100644 --- a/lib/public_key/.gitignore +++ b/lib/public_key/.gitignore @@ -7,3 +7,4 @@ src/PKCS-FRAME.erl src/PKCS-FRAME.hrl include/OTP-PUB-KEY.hrl include/PKCS-FRAME.hrl +priv/lib/ diff --git a/lib/public_key/c_src/Makefile b/lib/public_key/c_src/Makefile index 535d6c9218..064868af42 100644 --- a/lib/public_key/c_src/Makefile +++ b/lib/public_key/c_src/Makefile @@ -91,7 +91,7 @@ endif _create_dirs := $(shell mkdir -p $(OBJDIR) $(LIBDIR)) -debug opt valgrind: $(DIRS) $(PUBKEY_LIB) +debug opt valgrind lcnt asan: $(DIRS) $(PUBKEY_LIB) $(OBJDIR): -@mkdir -p $(OBJDIR) diff --git a/lib/reltool/doc/src/reltool.xml b/lib/reltool/doc/src/reltool.xml index 49c5424969..ab27a03ff8 100644 --- a/lib/reltool/doc/src/reltool.xml +++ b/lib/reltool/doc/src/reltool.xml @@ -226,10 +226,10 @@ <tag><c>debug_info</c></tag> <item> - <p>The <c>debug_info</c> parameter controls whether the debug - information in the beam file should be kept (<c>keep</c>) or - stripped <c>strip</c> when the file is copied to the target - system.</p> + <p>The <c>debug_info</c> parameter controls what debug + information in the beam file should be kept or stripped. + <c>keep</c> keeps all debug info, <c>strip</c> strips all debug + info, and a list of chunkids keeps only those chunks.</p> </item> <tag><c>excl_lib</c></tag> @@ -542,7 +542,7 @@ app_vsn() = string() archive_opt = zip_create_opt() boot_rel() = rel_name() app_file() = keep | strip | all -debug_info() = keep | strip +debug_info() = keep | strip | [beam_lib:chunkid()] dir() = string() escript() = {incl_cond, incl_cond()} escript_file() = file() diff --git a/lib/reltool/src/reltool.app.src b/lib/reltool/src/reltool.app.src index dc85464750..2d0994713b 100644 --- a/lib/reltool/src/reltool.app.src +++ b/lib/reltool/src/reltool.app.src @@ -36,6 +36,6 @@ {registered, []}, {applications, [stdlib, kernel]}, {env, []}, - {runtime_dependencies, ["wx-1.2","tools-2.6.14","stdlib-3.4","sasl-2.4", + {runtime_dependencies, ["wx-1.2","tools-2.6.14","stdlib-@OTP-15680@","sasl-2.4", "kernel-3.0","erts-7.0"]} ]}. diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl index 844a3a880a..6cbadc3ade 100644 --- a/lib/reltool/src/reltool.hrl +++ b/lib/reltool/src/reltool.hrl @@ -28,7 +28,7 @@ %% derived - Include only those modules that others are dependent on -type mod_cond() :: all | app | ebin | derived | none. -type incl_cond() :: include | exclude | derived. --type debug_info() :: keep | strip. +-type debug_info() :: keep | strip | [beam_lib:chunkid()]. -type app_file() :: keep | strip | all. -type re_regexp() :: string(). % re:regexp() -type regexps() :: [re_regexp()] | diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index de7d73bf37..6b7a79c62f 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -1587,7 +1587,7 @@ decode(#sys{} = Sys, [{Key, Val} | KeyVals]) -> Sys#sys{embedded_app_type = Val}; app_file when Val =:= keep; Val =:= strip; Val =:= all -> Sys#sys{app_file = Val}; - debug_info when Val =:= keep; Val =:= strip -> + debug_info when Val =:= keep; Val =:= strip; is_list(Val) -> Sys#sys{debug_info = Val}; _ -> reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) @@ -1608,7 +1608,8 @@ decode(#app{} = App, [{Key, Val} | KeyVals]) -> App#app{incl_cond = Val}; debug_info when Val =:= keep; - Val =:= strip -> + Val =:= strip; + is_list(Val) -> App#app{debug_info = Val}; app_file when Val =:= keep; Val =:= strip; @@ -1663,7 +1664,7 @@ decode(#mod{} = Mod, [{Key, Val} | KeyVals]) -> case Key of incl_cond when Val =:= include; Val =:= exclude; Val =:= derived -> Mod#mod{incl_cond = Val}; - debug_info when Val =:= keep; Val =:= strip -> + debug_info when Val =:= keep; Val =:= strip; is_list(Val) -> Mod#mod{debug_info = Val}; _ -> reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index 1a0f868be8..304167c44b 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -1172,7 +1172,9 @@ spec_mod(Mod, DebugInfo) -> keep -> {copy_file, File}; strip -> - {strip_beam, File} + {strip_beam, File, []}; + ChunkIds -> + {strip_beam, File, ChunkIds} end. spec_app_file(#app{name = Name, @@ -1315,11 +1317,14 @@ do_eval_spec({write_file, File, Bin}, TargetDir) -> TargetFile = filename:join([TargetDir, File]), reltool_utils:write_file(TargetFile, Bin); -do_eval_spec({strip_beam, File}, _OrigSourceDir, SourceDir, TargetDir) -> +do_eval_spec({strip_beam, File, ChunkIds}, + _OrigSourceDir, + SourceDir, + TargetDir) -> SourceFile = filename:join([SourceDir, File]), TargetFile = filename:join([TargetDir, File]), BeamBin = reltool_utils:read_file(SourceFile), - {ok, {_, BeamBin2}} = beam_lib:strip(BeamBin), + {ok, {_, BeamBin2}} = beam_lib:strip(BeamBin, ChunkIds), reltool_utils:write_file(TargetFile, BeamBin2). cleanup_spec(List, TargetDir) when is_list(List) -> @@ -1349,7 +1354,7 @@ cleanup_spec({copy_file, NewFile, _OldFile}, TargetDir) -> cleanup_spec({write_file, File, _}, TargetDir) -> TargetFile = filename:join([TargetDir, File]), file:delete(TargetFile); -cleanup_spec({strip_beam, File}, TargetDir) -> +cleanup_spec({strip_beam, File, _ChunkIds}, TargetDir) -> TargetFile = filename:join([TargetDir, File]), file:delete(TargetFile). @@ -1419,7 +1424,7 @@ do_filter_spec(Path, do_filter_spec(Path, {write_file, File, _}, InclRegexps, ExclRegexps) -> Path2 = opt_join(Path, File), match(Path2, InclRegexps, ExclRegexps); -do_filter_spec(Path, {strip_beam, File}, InclRegexps, ExclRegexps) -> +do_filter_spec(Path, {strip_beam, File, _ChunkIds}, InclRegexps, ExclRegexps) -> Path2 = opt_join(Path, File), match(Path2, InclRegexps, ExclRegexps). diff --git a/lib/runtime_tools/doc/src/dbg.xml b/lib/runtime_tools/doc/src/dbg.xml index ba7641ffa9..5d9d7486d4 100644 --- a/lib/runtime_tools/doc/src/dbg.xml +++ b/lib/runtime_tools/doc/src/dbg.xml @@ -821,7 +821,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ <name since="">tracer(Type, Data) -> {ok, pid()} | {error, Error}</name> <fsummary>Start a tracer server with additional parameters</fsummary> <type> - <v>Type = port | process | module</v> + <v>Type = port | process | module | file</v> <v>Data = PortGenerator | HandlerSpec | ModuleSpec</v> <v>PortGenerator = fun() (no arguments)</v> <v>Error = term()</v> @@ -862,6 +862,9 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ be either a tuple describing the <seeerl marker="erts:erl_tracer"><c>erl_tracer</c></seeerl> module to be used for tracing and the state to be used for that tracer module or a fun returning the same tuple.</p> + <p>if <c>Type</c> is <c>file</c>, then the second parameter + should be a filename specifying a file where all the traces + are printed.</p> <p>If an error is returned, it can either be due to a tracer server already running (<c>{error,already_started}</c>) or due to the <c>HandlerFun</c> throwing an exception. diff --git a/lib/runtime_tools/src/dbg.erl b/lib/runtime_tools/src/dbg.erl index d5be9ac6a9..ca1ce7948c 100644 --- a/lib/runtime_tools/src/dbg.erl +++ b/lib/runtime_tools/src/dbg.erl @@ -317,8 +317,17 @@ tracer(process, {Handler,HandlerData}) -> tracer(module, Fun) when is_function(Fun) -> start(Fun); tracer(module, {Module, State}) -> - start(fun() -> {Module, State} end). - + start(fun() -> {Module, State} end); + +tracer(file, Filename) -> + tracer(process, + {fun F(E, undefined) -> + {ok, D} = file:open(Filename, [write]), + F(E, D); + F(E, D) -> + dhandler(E, D), + D + end, undefined}). remote_tracer(port, Fun) when is_function(Fun) -> remote_start(Fun); @@ -1599,14 +1608,14 @@ new_pattern_table() -> term_to_binary(x)}), ets:insert(PT, {c, - term_to_binary([{'_',[],[{message,{caller}}]}])}), + term_to_binary([{'_',[],[{message,{caller_line}}]}])}), ets:insert(PT, {caller_trace, term_to_binary(c)}), ets:insert(PT, {cx, term_to_binary([{'_',[],[{exception_trace}, - {message,{caller}}]}])}), + {message,{caller_line}}]}])}), ets:insert(PT, {caller_exception_trace, term_to_binary(cx)}), diff --git a/lib/runtime_tools/test/dbg_SUITE.erl b/lib/runtime_tools/test/dbg_SUITE.erl index 209af7be19..ce4e122107 100644 --- a/lib/runtime_tools/test/dbg_SUITE.erl +++ b/lib/runtime_tools/test/dbg_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, big/1, tiny/1, simple/1, message/1, distributed/1, port/1, send/1, recv/1, - ip_port/1, file_port/1, file_port2/1, + ip_port/1, file_port/1, file_port2/1, file_tracer/1, ip_port_busy/1, wrap_port/1, wrap_port_time/1, with_seq_trace/1, dead_suspend/1, local_trace/1, saved_patterns/1, tracer_exit_on_stop/1, @@ -41,7 +41,7 @@ suite() -> all() -> [big, tiny, simple, message, distributed, port, ip_port, send, recv, - file_port, file_port2, ip_port_busy, + file_port, file_port2, file_tracer, ip_port_busy, wrap_port, wrap_port_time, with_seq_trace, dead_suspend, local_trace, saved_patterns, tracer_exit_on_stop, erl_tracer, distributed_erl_tracer]. @@ -621,6 +621,31 @@ file_port2(Config) when is_list(Config) -> end, ok. +%% Test tracing to file +file_tracer(Config) when is_list(Config) -> + stop(), + FName = make_temp_name(Config), + %% Ok, lets try with flush and follow_file. + {ok, _} = dbg:tracer(file, FName), + try + {ok, [{matched, _node, 1}]} = dbg:p(self(),call), + {ok, _} = dbg:tp(dbg, ltp,[{'_',[],[{message, {self}}]}]), + {ok, _} = dbg:tp(dbg, ln, [{'_',[],[{message, hej}]}]), + ok = dbg:ltp(), + timer:sleep(100), + {ok, LTP} = file:read_file(FName), + <<"dbg:ltp()",_/binary>> = string:find(LTP, "dbg:ltp() ("++pid_to_list(self())++")"), + ok = dbg:ln(), + timer:sleep(100), + {ok, LN} = file:read_file(FName), + <<"dbg:ln()",_/binary>> = string:find(LN, "dbg:ln() (hej)"), + stop() + after + dbg:stop_clear(), + file:delete(FName) + end, + ok. + %% Test tracing to wrapping file port wrap_port(Config) when is_list(Config) -> Self = self(), diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 8fc41ca506..9441cced60 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -42,6 +42,6 @@ {applications, [kernel, stdlib]}, {env, []}, {mod, {sasl, []}}, - {runtime_dependencies, ["tools-2.6.14","stdlib-3.4","kernel-6.0", + {runtime_dependencies, ["tools-2.6.14","stdlib-4.0","kernel-6.0", "erts-10.2"]}]}. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index b875c445c6..671eaf7745 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -696,7 +696,15 @@ specified([], _) -> []. get_items([H|T], Dict) -> - Item = check_item(keysearch(H, 1, Dict),H), + Item = case check_item(keysearch(H, 1, Dict),H) of + [Atom|_]=Atoms when is_atom(Atom), is_list(Atoms) -> + %% Check for duplicate entries in lists + case Atoms =/= lists:uniq(Atoms) of + true -> throw({dupl_entry, H, lists:subtract(Atoms, lists:uniq(Atoms))}); + false -> Atoms + end; + X -> X + end, [Item|get_items(T, Dict)]; get_items([], _Dict) -> []. @@ -2436,6 +2444,8 @@ form_reading({read,File}) -> io_lib:format("Cannot read ~tp~n",[File]); form_reading({{bad_param, P},_}) -> io_lib:format("Bad parameter in .app file: ~tp~n",[P]); +form_reading({{dupl_entry, P, DE},_}) -> + io_lib:format("~tp parameter contains duplicates of: ~tp~n", [P, DE]); form_reading({{missing_param,P},_}) -> io_lib:format("Missing parameter in .app file: ~p~n",[P]); form_reading({badly_formatted_application,_}) -> diff --git a/lib/sasl/test/Makefile b/lib/sasl/test/Makefile index adc5a927ac..98c6cfb5bb 100644 --- a/lib/sasl/test/Makefile +++ b/lib/sasl/test/Makefile @@ -95,6 +95,7 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) sasl.spec sasl.cover $(EMAKEFILE) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cfh - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + $(INSTALL_DIR) "$(RELSYSDIR)/sasl_SUITE_data" $(INSTALL_DATA) $(ERL_TOP)/make/otp_version_tickets "$(RELSYSDIR)/sasl_SUITE_data" release_docs_spec: diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 570e1bc1fe..734b55155d 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -411,7 +411,8 @@ move_system_unix(NodeA, PeerA, TestRootDir, ErtsBinDir, NewSystemPath) -> [{env,[{"PATH",TestRootDir ++ ":" ++ os:getenv("PATH")}]}]), %% Wait for node to start - receive _ -> ok end, + receive M1 -> ct:pal("~p",[M1]) end, + receive M2 -> ct:pal("~p",[M2]) end, hej = erpc:call(LinkNode, app_callback_module, get_response, []), diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 801660ddac..e57b559105 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -63,7 +63,7 @@ groups() -> src_tests_script, crazy_script, optional_apps_script, included_script, included_override_script, included_fail_script, included_bug_script, exref_script, - duplicate_modules_script, + duplicate_modules_script, duplicate_entries_script, otp_3065_circular_dependenies, included_and_used_sort_script]}, {tar, [], [tar_options, relname_tar, normal_tar, no_mod_vsn_tar, system_files_tar, @@ -467,6 +467,18 @@ variable_script(Config) when is_list(Config) -> ok = file:set_cwd(OldDir), ok. +%% make_script: Duplicate entries in app file +duplicate_entries_script(Config) when is_list(Config) -> + DataDir = ?datadir, + create_apps_duplicate_entry(DataDir), + {LatestDir, LatestName} = create_script(latest_t21,Config), + error = systools:make_script(LatestName, + [{path, [DataDir, LatestDir]}]), + {LatestDir2, LatestName2} = create_script(latest_t22,Config), + ok = systools:make_script(LatestName2, + [{path, [DataDir, LatestDir2]}]), + ok. + %% make_script: Abnormal cases. abnormal_script(Config) when is_list(Config) -> {ok, OldDir} = file:get_cwd(), @@ -1109,9 +1121,9 @@ erts_tar(Config) -> {win32, _} -> {["beam.smp.pdb","erl.exe", "erl.pdb","erl_log.exe","erlexec.dll","erlsrv.exe","heart.exe", - "start_erl.exe","werl.exe","beam.smp.dll", + "start_erl.exe","beam.smp.dll", "epmd.exe","erl.ini","erl_call.exe", - "erlexec.pdb","escript.exe","inet_gethost.exe","werl.pdb"], + "erlexec.pdb","escript.exe","inet_gethost.exe"], ["dialyzer.exe","erlc.exe","yielding_c_fun.exe","ct_run.exe","typer.exe"]} end, @@ -2575,6 +2587,12 @@ create_script(latest_app_start_type2,Config) -> {xmerl,current,none}], Apps = core_apps(current) ++ OtherApps, do_create_script(latest_app_start_type2,Config,current,Apps); +create_script(latest_t21, Config) -> + Apps = core_apps(current) ++ [{t21, "1.0"}], + do_create_script(latest_t21, Config, "4.4", Apps); +create_script(latest_t22, Config) -> + Apps = core_apps(current) ++ [{t22, "1.0"}], + do_create_script(latest_t22, Config, "4.4", Apps); create_script(current_all_no_sasl,Config) -> Apps = [{kernel,current},{stdlib,current},{db,"2.1"},{fe,"3.1"}], do_create_script(current_all_no_sasl,Config,current,Apps); @@ -2906,7 +2924,6 @@ create_include_files(sort_apps_rev, Config) -> file:write_file(Name ++ ".rel", list_to_binary(Rel)), {filename:dirname(Name), filename:basename(Name)}. - create_apps(Dir) -> T1 = "{application, t1,\n" " [{vsn, \"1.0\"},\n" @@ -3025,8 +3042,6 @@ create_apps2(Dir) -> " {registered, []}]}.\n", file:write_file(fname(Dir, 't13.app'), list_to_binary(T13)). - - create_apps_3065(Dir) -> T11 = "{application, chTraffic,\n" " [{vsn, \"1.0\"},\n" @@ -3119,6 +3134,24 @@ create_sort_apps(Dir) -> " {registered, []}]}.\n", file:write_file(fname(Dir, 't20.app'), list_to_binary(T20)). +create_apps_duplicate_entry(Dir) -> + T21 = "{application, t21,\n" + " [{vsn, \"1.0\"},\n" + " {description, \"test\"},\n" + " {modules, []},\n" + " {applications, []},\n" + " {included_applications, []},\n" + " {registered, [test, test]}]}.\n", + file:write_file(fname(Dir, 't21.app'), list_to_binary(T21)), + T22 = "{application, t22,\n" + " [{vsn, \"1.0\"},\n" + " {description, \"test\"},\n" + " {modules, []},\n" + " {applications, []},\n" + " {included_applications, []},\n" + " {registered, [test]}]}.\n", + file:write_file(fname(Dir, 't22.app'), list_to_binary(T22)). + fname(N) -> filename:join(N). diff --git a/lib/ssh/src/.gitignore b/lib/ssh/src/.gitignore new file mode 100644 index 0000000000..3f9cfafd33 --- /dev/null +++ b/lib/ssh/src/.gitignore @@ -0,0 +1 @@ +deps diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 2cb8d80488..aded3fc06e 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -59,9 +59,9 @@ {mod, {ssh_app, []}}, {runtime_dependencies, [ "crypto-5.0", - "erts-11.0", - "kernel-6.0", + "erts-@OTP-17932@", + "kernel-@OTP-17932@", "public_key-1.6.1", - "stdlib-3.15", + "stdlib-@OTP-17932@", "runtime_tools-1.15.1" ]}]}. diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 13a44beea3..5820c08c44 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -281,6 +281,10 @@ handle_msg({Group, get_unicode_state}, State) -> Group ! {self(), get_unicode_state, false}, {ok, State}; +handle_msg({Group, get_terminal_state}, State) -> + Group ! {self(), get_terminal_state, true}, + {ok, State}; + handle_msg({Group, tty_geometry}, #state{group = Group, pty = Pty } = State) -> @@ -428,6 +432,8 @@ io_request({move_rel, N}, Buf, Tty, _Group) -> move_rel(N, Buf, Tty); io_request({delete_chars,N}, Buf, Tty, _Group) -> delete_chars(N, Buf, Tty); +io_request(clear, Buf, _Tty, _Group) -> + {"\e[H\e[2J", Buf}; io_request(beep, Buf, _Tty, _Group) -> {[7], Buf}; @@ -447,7 +453,7 @@ io_request(tty_geometry, Buf, Tty, Group) -> io_request({put_chars_sync, Class, Cs, Reply}, Buf, Tty, Group) -> %% We handle these asynchronous for now, if we need output guarantees %% we have to handle these synchronously - Group ! {reply, Reply}, + Group ! {reply, Reply, ok}, io_request({put_chars, Class, Cs}, Buf, Tty, Group); io_request(_R, Buf, _Tty, _Group) -> @@ -666,7 +672,9 @@ start_shell(ConnectionHandler, State) -> {_,_,_} = Shell -> Shell end, - State#state{group = group:start(self(), ShellSpawner, [{echo, get_echo(State#state.pty)}]), + State#state{group = group:start(self(), ShellSpawner, + [{expand_below, false}, + {echo, get_echo(State#state.pty)}]), buf = empty_buf()}. %%-------------------------------------------------------------------- @@ -687,7 +695,8 @@ start_exec_shell(ConnectionHandler, Cmd, State) -> {M,F,A} -> {M, F, A++[Cmd]} end, - State#state{group = group:start(self(), ExecShellSpawner, [{echo,false}]), + State#state{group = group:start(self(), ExecShellSpawner, [{expand_below, false}, + {echo,false}]), buf = empty_buf()}. %%-------------------------------------------------------------------- @@ -771,7 +780,8 @@ exec_in_self_group(ConnectionHandler, ChannelId, WantReply, State, Fun) -> end end) end, - {ok, State#state{group = group:start(self(), Exec, [{echo,false}]), + {ok, State#state{group = group:start(self(), Exec, [{expand_below, false}, + {echo,false}]), buf = empty_buf()}}. diff --git a/lib/ssl/Makefile b/lib/ssl/Makefile index 5cc111ec14..46a98464f7 100644 --- a/lib/ssl/Makefile +++ b/lib/ssl/Makefile @@ -40,4 +40,6 @@ include $(ERL_TOP)/make/otp_subdir.mk DIA_PLT_APPS=crypto runtime_tools inets public_key +TEST_NEEDS_RELEASE=true + include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index ba2bcc15b2..5450462ea7 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -1478,6 +1478,26 @@ fun(srp, Username :: binary(), UserState :: term()) -> </datatype> <datatype> + <name name="stateless_tickets_seed"/> + <desc> + <p>Configures the seed used for the encryption of stateless session tickets. + Allowed values are any randomly generated <c>binary()</c>. If this option is not + configured, an encryption seed will be randomly generated.</p> + + <warning><p>Reusing the ticket encryption seed between multiple server + instances enables stateless session tickets to work across multiple server + instances, but it breaks anti-replay protection across instances.</p> + + <p>Inaccurate time synchronization between server instances can also + affect session ticket freshness checks, potentially causing false negatives as + well as false positives.</p></warning> + + <note><p>This option is supported by TLS 1.3 and above and only with stateless + session tickets.</p></note> + </desc> + </datatype> + + <datatype> <name name="anti_replay"/> <desc> <p>Configures the server's built-in anti replay feature based on Bloom filters.</p> @@ -1485,7 +1505,7 @@ fun(srp, Username :: binary(), UserState :: term()) -> <p>Allowed values are the pre-defined <c>'10k'</c>, <c>'100k'</c> or a custom 3-tuple that defines the properties of the bloom filters: <c>{WindowSize, HashFunctions, Bits}</c>. <c>WindowSize</c> is the number of seconds after the current Bloom filter is rotated - and also the window size used for freshness checks. <c>HashFunctions</c> is the number + and also the window size used for freshness checks of ClientHello. <c>HashFunctions</c> is the number hash functions and <c>Bits</c> is the number of bits in the bit vector. <c>'10k'</c> and <c>'100k'</c> are simple defaults with the following properties:</p> <list type="bulleted"> diff --git a/lib/ssl/doc/src/using_ssl.xml b/lib/ssl/doc/src/using_ssl.xml index 318e367245..148c7d8dfc 100644 --- a/lib/ssl/doc/src/using_ssl.xml +++ b/lib/ssl/doc/src/using_ssl.xml @@ -776,7 +776,7 @@ ssl:connect("localhost", 9999, [{verify, verify_peer}, less than ticket lifetime.</p></item> <item><p>Actual ticket age shall be less than the ticket lifetime (stateless session tickets contain the servers timestamp when the ticket was issued).</p></item> - <item><p>Ticket shall be used within specified time window (freshness checks).</p></item> + <item><p>ClientHello created with the ticket shall be sent relatively recently (freshness checks).</p></item> <item><p>If all above checks passed both <em>current</em> and <em>old</em> Bloom filters are checked to detect if binder was already seen. Being a probabilistic data structure, false positives can occur and they trigger a full handshake.</p></item> diff --git a/lib/ssl/src/.gitignore b/lib/ssl/src/.gitignore new file mode 100644 index 0000000000..3f9cfafd33 --- /dev/null +++ b/lib/ssl/src/.gitignore @@ -0,0 +1 @@ +deps diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 8f2eb7d82b..3b9dc5ffc5 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -642,7 +642,7 @@ format_status(Type, Data) -> %%% Internal functions %%-------------------------------------------------------------------- initial_state(Role, Host, Port, Socket, - {#{client_renegotiation := ClientRenegotiation} = SSLOptions, SocketOptions, Trackers}, User, + {SSLOptions, SocketOptions, Trackers}, User, {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> BeastMitigation = maps:get(beast_mitigation, SSLOptions, disabled), ConnectionStates = dtls_record:init_connection_states(Role, BeastMitigation), @@ -668,13 +668,11 @@ initial_state(Role, Host, Port, Socket, handshake_env = #handshake_env{ tls_handshake_history = ssl_handshake:init_handshake_history(), renegotiation = {false, first}, - allow_renegotiate = ClientRenegotiation + allow_renegotiate = maps:get(client_renegotiation, SSLOptions, undefined) }, connection_env = #connection_env{user_application = {Monitor, User}}, socket_options = SocketOptions, - %% We do not want to save the password in the state so that - %% could be written in the clear into error logs. - ssl_options = SSLOptions#{password => undefined}, + ssl_options = SSLOptions, session = #session{is_resumable = false}, connection_states = ConnectionStates, protocol_buffers = #protocol_buffers{}, diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index f0b5da43b3..da68c960ad 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -29,8 +29,6 @@ -export([gen_listen/3, gen_accept/2, gen_accept_connection/6, gen_setup/6, gen_close/2, gen_select/2, gen_address/1]). --export([nodelay/0]). - -export([verify_client/3, cert_nodes/1]). -export([dbg/0]). % Debug @@ -41,8 +39,12 @@ -include_lib("public_key/include/public_key.hrl"). -include("ssl_api.hrl"). +-include("ssl_cipher.hrl"). +-include("ssl_internal.hrl"). -include_lib("kernel/include/logger.hrl"). +-define(PROTOCOL, tls). + %% ------------------------------------------------------------------------- childspecs() -> @@ -70,8 +72,27 @@ is_node_name(Node) -> %% ------------------------------------------------------------------------- -hs_data_common(#sslsocket{pid = [_, DistCtrl|_]} = SslSocket) -> +hs_data_inet_tcp(Driver, Socket) -> + Family = Driver:family(), + {ok, Peername} = inet:peername(Socket), + (inet_tcp_dist:gen_hs_data(Driver, Socket)) #hs_data{ + f_address = + fun(_, Node) -> + {node, _, Host} = dist_util:split_node(Node), + #net_address{ + address = Peername, + host = Host, + protocol = ?PROTOCOL, + family = Family + } + end}. + +hs_data_ssl(Driver, #sslsocket{pid = [_, DistCtrl|_]} = SslSocket) -> + Family = Driver:family(), + {ok, Peername} = ssl:peername(SslSocket), + #hs_data{ + socket = DistCtrl, f_send = fun (_Ctrl, Packet) -> f_send(SslSocket, Packet) @@ -95,7 +116,7 @@ hs_data_common(#sslsocket{pid = [_, DistCtrl|_]} = SslSocket) -> end, f_address = fun (Ctrl, Node) when Ctrl == DistCtrl -> - f_address(SslSocket, Node) + f_address(Family, Peername, Node) end, mf_tick = fun (Ctrl) when Ctrl == DistCtrl -> @@ -133,22 +154,19 @@ f_setopts_pre_nodeup(_SslSocket) -> ok. f_setopts_post_nodeup(SslSocket) -> - ssl:setopts(SslSocket, [nodelay()]). + ssl:setopts(SslSocket, [inet_tcp_dist:nodelay()]). f_getll(DistCtrl) -> {ok, DistCtrl}. -f_address(SslSocket, Node) -> - case ssl:peername(SslSocket) of - {ok, Address} -> - case dist_util:split_node(Node) of - {node,_,Host} -> - #net_address{ - address=Address, host=Host, - protocol=tls, family=inet}; - _ -> - {error, no_node} - end +f_address(Family, Address, Node) -> + case dist_util:split_node(Node) of + {node,_,Host} -> + #net_address{ + address=Address, host=Host, + protocol=?PROTOCOL, family=Family}; + _ -> + {error, no_node} end. mf_tick(DistCtrl) -> @@ -200,7 +218,7 @@ gen_listen(Driver, Name, Host) -> case inet_tcp_dist:gen_listen(Driver, Name, Host) of {ok, {Socket, Address, Creation}} -> inet:setopts(Socket, [{packet, 4}, {nodelay, true}]), - {ok, {Socket, Address#net_address{protocol=tls}, Creation}}; + {ok, {Socket, Address#net_address{protocol=?PROTOCOL}, Creation}}; Other -> Other end. @@ -272,6 +290,7 @@ spawn_accept({Driver, Listen, Kernel}) -> accept_one(Driver, Kernel, Socket) -> Opts = setup_verify_client(Socket, get_ssl_options(server)), + KTLS = proplists:get_value(ktls, Opts, false), wait_for_code_server(), case ssl:handshake( @@ -279,24 +298,27 @@ accept_one(Driver, Kernel, Socket) -> trace([{active, false},{packet, 4}|Opts]), net_kernel:connecttime()) of - {ok, #sslsocket{pid = [_, DistCtrl| _]} = SslSocket} -> - trace( - Kernel ! - {accept, self(), DistCtrl, - Driver:family(), tls}), - receive - {Kernel, controller, Pid} -> - case ssl:controlling_process(SslSocket, Pid) of + {ok, #sslsocket{pid = [Receiver, Sender| _]} = SslSocket} -> + case KTLS of + true -> + {ok, KtlsInfo} = ssl_gen_statem:ktls_handover(Receiver), + case set_ktls(KtlsInfo) of ok -> - trace(Pid ! {self(), controller}); - Error -> - trace(Pid ! {self(), exit}), + accept_one( + Driver, Kernel, Socket, + fun inet_tcp:controlling_process/2, Socket); + {error, KtlsReason} -> ?LOG_ERROR( - "Cannot control TLS distribution connection: ~p~n", - [Error]) + [{slogan, set_ktls_failed}, + {reason, KtlsReason}, + {pid, self()}]), + gen_tcp:close(Socket), + trace({ktls_error, KtlsReason}) end; - {Kernel, unsupported_protocol} -> - trace(unsupported_protocol) + false -> + accept_one( + Driver, Kernel, Sender, + fun ssl:controlling_process/2, SslSocket) end; {error, {options, _}} = Error -> %% Bad options: that's probably our fault. @@ -310,6 +332,24 @@ accept_one(Driver, Kernel, Socket) -> gen_tcp:close(Socket), trace(Other) end. +%% +accept_one(Driver, Kernel, DistCtrl, ControllingProcessFun, DistSocket) -> + trace(Kernel ! {accept, self(), DistCtrl, Driver:family(), ?PROTOCOL}), + receive + {Kernel, controller, Pid} -> + case ControllingProcessFun(DistSocket, Pid) of + ok -> + trace(Pid ! {self(), controller}); + {error, Reason} -> + trace(Pid ! {self(), exit}), + ?LOG_ERROR( + [{slogan, controlling_process_failed}, + {reason, Reason}, + {pid, self()}]) + end; + {Kernel, unsupported_protocol} -> + trace(unsupported_protocol) + end. %% {verify_fun,{fun ?MODULE:verify_client/3,_}} is used @@ -421,6 +461,7 @@ wait_for_code_server() -> timer:sleep(10), wait_for_code_server(); Pid when is_pid(Pid) -> + init:run_on_load_handlers([crypto,asn1rt_nif]), ok end. @@ -443,24 +484,30 @@ gen_accept_connection( dist_util:net_ticker_spawn_options())). do_accept( - _Driver, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime, Kernel) -> + Driver, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime, Kernel) -> MRef = erlang:monitor(process, AcceptPid), receive {AcceptPid, controller} -> erlang:demonitor(MRef, [flush]), - {ok, SslSocket} = tls_sender:dist_tls_socket(DistCtrl), - Timer = dist_util:start_timer(SetupTime), - NewAllowed = allowed_nodes(SslSocket, Allowed), - HSData0 = hs_data_common(SslSocket), + Timer = dist_util:start_timer(SetupTime), + {HSData0, NewAllowed} = + case is_port(DistCtrl) of + true -> + {hs_data_inet_tcp(Driver, DistCtrl), + Allowed}; + false -> + {ok, SslSocket} = tls_sender:dist_tls_socket(DistCtrl), + link(DistCtrl), + {hs_data_ssl(Driver, SslSocket), + allowed_nodes(SslSocket, Allowed)} + end, HSData = HSData0#hs_data{ kernel_pid = Kernel, this_node = MyNode, - socket = DistCtrl, timer = Timer, this_flags = 0, allowed = NewAllowed}, - link(DistCtrl), dist_util:handshake_other_started(trace(HSData)); {AcceptPid, exit} -> %% this can happen when connection was initiated, but dropped @@ -552,6 +599,8 @@ setup_fun(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> -spec do_setup(_,_,_,_,_,_,_) -> no_return(). do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> + wait_for_code_server(), + {Name, Address} = split_node(Driver, Node, LongOrShortNames), ErlEpmd = net_kernel:epmd_module(), {ARMod, ARFun} = get_address_resolver(ErlEpmd, Driver), @@ -579,35 +628,51 @@ do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> do_setup_connect(Driver, Kernel, Node, Address, Ip, TcpPort, Version, Type, MyNode, Timer) -> Opts = trace(connect_options(get_ssl_options(client))), + KTLS = proplists:get_value(ktls, Opts, false), dist_util:reset_timer(Timer), case ssl:connect( Ip, TcpPort, - [binary, {active, false}, {packet, 4}, {server_name_indication, Address}, + [binary, {active, false}, {packet, 4}, + {server_name_indication, Address}, Driver:family(), {nodelay, true}] ++ Opts, - net_kernel:connecttime()) of - {ok, #sslsocket{pid = [_, DistCtrl| _]} = SslSocket} -> - _ = monitor_pid(DistCtrl), - ok = ssl:controlling_process(SslSocket, self()), - HSData0 = hs_data_common(SslSocket), - HSData = - HSData0#hs_data{ - kernel_pid = Kernel, - other_node = Node, - this_node = MyNode, - socket = DistCtrl, - timer = Timer, - this_flags = 0, - other_version = Version, - request_type = Type}, - link(DistCtrl), - dist_util:handshake_we_started(trace(HSData)); - Other -> - %% Other Node may have closed since - %% port_please ! - ?shutdown2( - Node, - trace( - {ssl_connect_failed, Ip, TcpPort, Other})) + net_kernel:connecttime() + ) of + {ok, #sslsocket{pid = [Receiver, Sender| _]} = SslSocket} -> + HSData = + case KTLS of + true -> + {ok, KtlsInfo} = + ssl_gen_statem:ktls_handover(Receiver), + case set_ktls(KtlsInfo) of + ok -> + #{socket := Socket} = KtlsInfo, + hs_data_inet_tcp(Driver, Socket); + {error, KtlsReason} -> + ?shutdown2( + Node, + trace({set_ktls_failed, KtlsReason})) + end; + false -> + _ = monitor_pid(Sender), + ok = ssl:controlling_process(SslSocket, self()), + link(Sender), + hs_data_ssl(Driver, SslSocket) + end + #hs_data{ + kernel_pid = Kernel, + other_node = Node, + this_node = MyNode, + timer = Timer, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(trace(HSData)); + Other -> + %% Other Node may have closed since + %% port_please ! + ?shutdown2( + Node, + trace({ssl_connect_failed, Ip, TcpPort, Other})) end. close(Socket) -> @@ -792,21 +857,6 @@ connect_options(Opts) -> Opts end. -%% we may not always want the nodelay behaviour -%% for performance reasons -nodelay() -> - case application:get_env(kernel, dist_nodelay) of - undefined -> - {nodelay, true}; - {ok, true} -> - {nodelay, true}; - {ok, false} -> - {nodelay, false}; - _ -> - {nodelay, true} - end. - - get_ssl_options(Type) -> try ets:lookup(ssl_dist_opts, Type) of [{Type, Opts0}] -> @@ -871,7 +921,13 @@ ssl_option(client, Opt) -> "secure_renegotiate" -> fun atomize/1; "depth" -> fun erlang:list_to_integer/1; "hibernate_after" -> fun erlang:list_to_integer/1; - "ciphers" -> fun listify/1; + "ciphers" -> + %% Allows just one cipher, for now (could be , separated) + fun (Val) -> [listify(Val)] end; + "versions" -> + %% Allows just one version, for now (could be , separated) + fun (Val) -> [atomize(Val)] end; + "ktls" -> fun atomize/1; _ -> error end. @@ -897,6 +953,124 @@ verify_fun(Value) -> error(malformed_ssl_dist_opt, [Value]) end. +set_ktls(KtlsInfo) -> + %% + %% Check OS type and version + %% + case {os:type(), os:version()} of + {{unix,linux}, {Major,Minor,_}} + when 5 == Major, 2 =< Minor; + 5 < Major -> + set_ktls_1(KtlsInfo); + OsTypeVersion -> + {error, {ktls_invalid_os, OsTypeVersion}} + end. + +%% Check TLS version and cipher suite +%% +set_ktls_1( + #{tls_version := {3,4}, % 'tlsv1.3' + cipher_suite := CipherSuite, + socket := Socket} = KtlsInfo) + when CipherSuite =:= ?TLS_AES_256_GCM_SHA384 -> + %% + %% See https://www.kernel.org/doc/html/latest/networking/tls.html + %% and include/netinet/tcp.h + %% + SOL_TCP = 6, + TCP_ULP = 31, + KtlsMod = <<"tls">>, % Linux kernel module name + KtlsModSize = byte_size(KtlsMod), + _ = inet:setopts(Socket, [{raw, SOL_TCP, TCP_ULP, KtlsMod}]), + %% + %% Check if kernel module loaded, + %% i.e if getopts SOL_TCP,TCP_ULP returns KtlsMod + %% + case + inet:getopts(Socket, [{raw, SOL_TCP, TCP_ULP, KtlsModSize + 1}]) + of + {ok, [{raw, SOL_TCP, TCP_ULP, <<KtlsMod:KtlsModSize/binary,0>>}]} -> + set_ktls_2(KtlsInfo, Socket); + Other -> + {error, {ktls_not_supported, Other}} + end; +set_ktls_1( + #{tls_version := TLSVersion, + cipher_suite := CipherSuite, + socket := _}) -> + {error, {ktls_invalid_cipher, TLSVersion, CipherSuite}}. + +%% Set kTLS cipher +%% +set_ktls_2( + #{write_state := + #cipher_state{ + key = <<WriteKey:32/bytes>>, + iv = <<WriteSalt:4/bytes, WriteIV:8/bytes>> + }, + write_seq := WriteSeq, + read_state := + #cipher_state{ + key = <<ReadKey:32/bytes>>, + iv = <<ReadSalt:4/bytes, ReadIV:8/bytes>> + }, + read_seq := ReadSeq, + socket_options := SocketOptions}, + Socket) -> + %% + %% See include/linux/tls.h + %% + TLS_1_3_VERSION_MAJOR = 3, + TLS_1_3_VERSION_MINOR = 4, + TLS_1_3_VERSION = + (TLS_1_3_VERSION_MAJOR bsl 8) bor TLS_1_3_VERSION_MINOR, + TLS_CIPHER_AES_GCM_256 = 52, + TLS_crypto_info_TX = + <<TLS_1_3_VERSION:16/native, + TLS_CIPHER_AES_GCM_256:16/native, + WriteIV/bytes, WriteKey/bytes, + WriteSalt/bytes, WriteSeq:64/native>>, + TLS_crypto_info_RX = + <<TLS_1_3_VERSION:16/native, + TLS_CIPHER_AES_GCM_256:16/native, + ReadIV/bytes, ReadKey/bytes, + ReadSalt/bytes, ReadSeq:64/native>>, + SOL_TLS = 282, + TLS_TX = 1, + TLS_RX = 2, + RawOptTX = {raw, SOL_TLS, TLS_TX, TLS_crypto_info_TX}, + RawOptRX = {raw, SOL_TLS, TLS_RX, TLS_crypto_info_RX}, + _ = inet:setopts(Socket, [RawOptTX]), + _ = inet:setopts(Socket, [RawOptRX]), + %% + %% Check if cipher could be set + %% + case + inet:getopts( + Socket, [{raw, SOL_TLS, TLS_TX, byte_size(TLS_crypto_info_TX)}]) + of + {ok, [RawOptTX]} -> + #socket_options{ + mode = _Mode, + packet = Packet, + packet_size = PacketSize, + header = Header, + active = Active + } = SocketOptions, + case + inet:setopts( + Socket, + [list, {packet, Packet}, {packet_size, PacketSize}, + {header, Header}, {active, Active}]) + of + ok -> ok; + {error, SetoptError} -> + {error, {ktls_setopt_failed, SetoptError}} + end; + Other -> + {error, {ktls_set_cipher_failed, Other}} + end. + %% ------------------------------------------------------------------------- %% Trace point diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index b5cb6b5d91..9801d0c319 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -85,6 +85,6 @@ {applications, [crypto, public_key, kernel, stdlib]}, {env, []}, {mod, {ssl_app, []}}, - {runtime_dependencies, ["stdlib-4.1","public_key-1.11.3","kernel-8.4", - "erts-10.0","crypto-5.0", "inets-5.10.7", + {runtime_dependencies, ["stdlib-4.1","public_key-1.11.3","kernel-@OTP-18235@", + "erts-@OTP-18248@","crypto-5.0", "inets-5.10.7", "runtime_tools-1.15.1"]}]}. diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 8050b354bd..f4eb51b5e2 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -94,9 +94,9 @@ connection_information/1, connection_information/2]). %% Misc --export([handle_options/2, - handle_options/3, - tls_version/1, +-export([handle_options/3, + update_options/3, + tls_version/1, suite_to_str/1, suite_to_openssl_str/1, str_to_suite/1]). @@ -470,6 +470,7 @@ {honor_ecc_order, honor_ecc_order()} | {client_renegotiation, client_renegotiation()}| {session_tickets, server_session_tickets()} | + {stateless_tickets_seed, stateless_tickets_seed()} | {anti_replay, anti_replay()} | {cookie, cookie()} | {early_data, server_early_data()}. @@ -490,6 +491,7 @@ -type honor_cipher_order() :: boolean(). -type honor_ecc_order() :: boolean(). -type client_renegotiation() :: boolean(). +-type stateless_tickets_seed() :: binary(). -type cookie() :: boolean(). -type server_certificate_authorities() :: boolean(). %% ------------------------------------------------------------------------------------------------------- @@ -593,11 +595,11 @@ connect(Socket, SslOptions) -> connect(Socket, SslOptions0, Timeout) when is_list(SslOptions0) andalso (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - CbInfo = handle_option_cb_info(SslOptions0, tls), - Transport = element(1, CbInfo), - try handle_options(Transport, Socket, SslOptions0, client, undefined) of - {ok, Config} -> - tls_socket:upgrade(Socket, Config, Timeout) + try + CbInfo = handle_option_cb_info(SslOptions0, tls), + Transport = element(1, CbInfo), + {ok, Config} = handle_options(Transport, Socket, SslOptions0, client, undefined), + tls_socket:upgrade(Socket, Config, Timeout) catch _:{error, Reason} -> {error, Reason} @@ -642,7 +644,7 @@ listen(_Port, []) -> {error, nooptions}; listen(Port, Options0) -> try - {ok, Config} = handle_options(Options0, server), + {ok, Config} = handle_options(Options0, server, undefined), do_listen(Port, Config, Config#config.connection_cb) catch Error = {error, _} -> @@ -729,7 +731,7 @@ handshake(ListenSocket, SslOptions) -> Reason :: closed | timeout | {options, any()} | error_alert(). handshake(#sslsocket{} = Socket, [], Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or - (Timeout == infinity)-> + (Timeout == infinity)-> handshake(Socket, Timeout); handshake(#sslsocket{fd = {_, _, _, Trackers}} = Socket, SslOpts, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> @@ -751,19 +753,20 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) Error = {error, _Reason} -> Error end; handshake(Socket, SslOptions, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - CbInfo = handle_option_cb_info(SslOptions, tls), - Transport = element(1, CbInfo), - ConnetionCb = connection_cb(SslOptions), - try handle_options(Transport, Socket, SslOptions, server, undefined) of - {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} -> - ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), - {ok, Port} = tls_socket:port(Transport, Socket), - {ok, SessionIdHandle} = tls_socket:session_id_tracker(ssl_unknown_listener, SslOpts), - ssl_gen_statem:handshake(ConnetionCb, Port, Socket, - {SslOpts, - tls_socket:emulated_socket_options(EmOpts, #socket_options{}), - [{session_id_tracker, SessionIdHandle}]}, - self(), CbInfo, Timeout) + try + CbInfo = handle_option_cb_info(SslOptions, tls), + Transport = element(1, CbInfo), + ConnetionCb = connection_cb(SslOptions), + {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} = + handle_options(Transport, Socket, SslOptions, server, undefined), + ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), + {ok, Port} = tls_socket:port(Transport, Socket), + {ok, SessionIdHandle} = tls_socket:session_id_tracker(ssl_unknown_listener, SslOpts), + ssl_gen_statem:handshake(ConnetionCb, Port, Socket, + {SslOpts, + tls_socket:emulated_socket_options(EmOpts, #socket_options{}), + [{session_id_tracker, SessionIdHandle}]}, + self(), CbInfo, Timeout) catch Error = {error, _Reason} -> Error end. @@ -1514,37 +1517,112 @@ do_listen(Port, #config{transport_info = {Transport, _, _, _,_}} = Config, tls_g do_listen(Port, Config, dtls_gen_connection) -> dtls_socket:listen(Port, Config). - --spec handle_options([any()], client | server) -> {ok, #config{}}; - ([any()], ssl_options()) -> ssl_options(). -handle_options(Opts, Role) -> - handle_options(undefined, undefined, Opts, Role, undefined). - -handle_options(Opts, Role, InheritedSslOpts) -> - handle_options(undefined, undefined, Opts, Role, InheritedSslOpts). +ssl_options() -> + [ + alpn_advertised_protocols, alpn_preferred_protocols, + anti_replay, + beast_mitigation, + cacertfile, cacerts, + cert, certs_keys,certfile, + certificate_authorities, + ciphers, + client_renegotiation, + cookie, + crl_cache, crl_check, + customize_hostname_check, + depth, + dh, dhfile, + + early_data, + eccs, + erl_dist, + fail_if_no_peer_cert, + fallback, + handshake, + hibernate_after, + honor_cipher_order, honor_ecc_order, + keep_secrets, + key, keyfile, + key_update_at, + ktls, + + log_level, + max_handshake_size, + middlebox_comp_mode, + max_fragment_length, + next_protocol_selector, next_protocols_advertised, + ocsp_stapling, ocsp_responder_certs, ocsp_nonce, + padding_check, + partial_chain, + password, + protocol, + psk_identity, + receiver_spawn_opts, + renegotiate_at, + reuse_session, reuse_sessions, + + secure_renegotiate, + sender_spawn_opts, + server_name_indication, + session_tickets, + stateless_tickets_seed, + signature_algs, signature_algs_cert, + sni_fun, + sni_hosts, + srp_identity, + supported_groups, + use_ticket, + user_lookup_fun, + verify, verify_fun, + versions + ]. %% Handle ssl options at handshake, handshake_continue -handle_options(_, _, Opts0, Role, InheritedSslOpts) when is_map(InheritedSslOpts) -> - {SslOpts, _} = expand_options(Opts0, ?RULES), - process_options(SslOpts, InheritedSslOpts, #{role => Role, - rules => ?RULES}); +-spec update_options([any()], client | server, map()) -> map(). +update_options(Opts, Role, InheritedSslOpts) when is_map(InheritedSslOpts) -> + {UserSslOpts, _} = split_options(Opts, ssl_options()), + process_options(UserSslOpts, InheritedSslOpts, #{role => Role}). + +process_options(UserSslOpts, SslOpts0, Env) -> + %% Reverse option list so we get the last set option if set twice, + %% users depend on it. + UserSslOptsMap = proplists:to_map(lists:reverse(UserSslOpts)), + SslOpts1 = opt_protocol_versions(UserSslOptsMap, SslOpts0, Env), + SslOpts2 = opt_verification(UserSslOptsMap, SslOpts1, Env), + SslOpts3 = opt_certs(UserSslOptsMap, SslOpts2, Env), + SslOpts4 = opt_tickets(UserSslOptsMap, SslOpts3, Env), + SslOpts5 = opt_ocsp(UserSslOptsMap, SslOpts4, Env), + SslOpts6 = opt_sni(UserSslOptsMap, SslOpts5, Env), + SslOpts7 = opt_signature_algs(UserSslOptsMap, SslOpts6, Env), + SslOpts8 = opt_alpn(UserSslOptsMap, SslOpts7, Env), + SslOpts9 = opt_mitigation(UserSslOptsMap, SslOpts8, Env), + SslOpts10 = opt_server(UserSslOptsMap, SslOpts9, Env), + SslOpts11 = opt_client(UserSslOptsMap, SslOpts10, Env), + SslOpts12 = opt_renegotiate(UserSslOptsMap, SslOpts11, Env), + SslOpts13 = opt_reuse_sessions(UserSslOptsMap, SslOpts12, Env), + SslOpts14 = opt_identity(UserSslOptsMap, SslOpts13, Env), + SslOpts15 = opt_supported_groups(UserSslOptsMap, SslOpts14, Env), + SslOpts16 = opt_crl(UserSslOptsMap, SslOpts15, Env), + SslOpts17 = opt_handshake(UserSslOptsMap, SslOpts16, Env), + SslOpts = opt_process(UserSslOptsMap, SslOpts17, Env), + SslOpts. + +-spec handle_options([any()], client | server, undefined|host()) -> {ok, #config{}}. +handle_options(Opts, Role, Host) -> + handle_options(undefined, undefined, Opts, Role, Host). + %% Handle all options in listen, connect and handshake handle_options(Transport, Socket, Opts0, Role, Host) -> - {SslOpts0, SockOpts0} = expand_options(Opts0, ?RULES), - - %% Ensure all options are evaluated at startup - SslOpts1 = add_missing_options(SslOpts0, ?RULES), - SslOpts2 = #{protocol := Protocol} - = process_options(SslOpts1, - #{}, - #{role => Role, - host => Host, - rules => ?RULES}), - - maybe_client_warn_no_verify(SslOpts2, Role), - SslOpts = maps:without([warn_verify_none], SslOpts2), + {UserSslOptsList, SockOpts0} = split_options(Opts0, ssl_options()), + + Env = #{role => Role, host => Host}, + SslOptsPost = process_options(UserSslOptsList, #{}, Env), + + SslOpts = maybe_client_warn_no_verify(SslOptsPost, Role), + %% Handle special options + #{protocol := Protocol} = SslOpts, {Sock, Emulated} = emulated_options(Transport, Socket, Protocol, SockOpts0), ConnetionCb = connection_cb(Protocol), CbInfo = handle_option_cb_info(Opts0, Protocol), @@ -1559,501 +1637,792 @@ handle_options(Transport, Socket, Opts0, Role, Host) -> }}. -%% process_options(SSLOptions, OptionsMap, Env) where -%% SSLOptions is the following tuple: -%% {InOptions, SkippedOptions, Counter} -%% -%% The list of options is processed in multiple passes. When -%% processing an option all dependencies must already be resolved. -%% If there are unresolved dependencies the option will be -%% skipped and processed in a subsequent pass. -%% Counter is equal to the number of unprocessed options at -%% the beginning of a pass. Its value must monotonically decrease -%% after each successful pass. -%% If the value of the counter is unchanged at the end of a pass, -%% the processing stops due to faulty input data. -process_options({[], [], _}, OptionsMap, _Env) -> - OptionsMap; -process_options({[], [_|_] = Skipped, Counter}, OptionsMap, Env) - when length(Skipped) < Counter -> - %% Continue handling options if current pass was successful - process_options({Skipped, [], length(Skipped)}, OptionsMap, Env); -process_options({[], [_|_], _Counter}, _OptionsMap, _Env) -> - throw({error, faulty_configuration}); -process_options({[{K0,V} = E|T], S, Counter}, OptionsMap0, Env) -> - K = maybe_map_key_internal(K0), - case check_dependencies(K, OptionsMap0, Env) of - true -> - OptionsMap = handle_option(K, V, OptionsMap0, Env), - process_options({T, S, Counter}, OptionsMap, Env); - false -> - %% Skip option for next pass - process_options({T, [E|S], Counter}, OptionsMap0, Env) +opt_protocol_versions(UserOpts, Opts, Env) -> + {_, PRC} = get_opt_of(protocol, [tls, dtls], tls, UserOpts, Opts), + + LogLevels = [none, all, emergency, alert, critical, error, + warning, notice, info, debug], + {_, LL} = get_opt_of(log_level, LogLevels, notice, UserOpts, Opts), + + {_, KS} = get_opt_bool(keep_secrets, false, UserOpts, Opts), + + {_, ED} = get_opt_bool(erl_dist, false, UserOpts, Opts), + {_, KTLS} = get_opt_bool(ktls, false, UserOpts, Opts), + + opt_versions(UserOpts, + Opts#{protocol => PRC, log_level => LL, keep_secrets => KS, + erl_dist => ED, ktls => KTLS}, + Env). + +opt_versions(UserOpts, #{protocol := Protocol} = Opts, _Env) -> + Versions = case get_opt(versions, unbound, UserOpts, Opts) of + {default, unbound} -> default_versions(Protocol); + {new, Vs} -> validate_versions(Protocol, Vs); + {old, Vs} -> Vs + end, + + {Where, MCM} = get_opt_bool(middlebox_comp_mode, true, UserOpts, Opts), + assert_version_dep(Where =:= new, middlebox_comp_mode, Versions, ['tlsv1.3']), + + Opts#{versions => Versions, middlebox_comp_mode => MCM}. + +default_versions(tls) -> + Vsns0 = tls_record:supported_protocol_versions(), + lists:sort(fun tls_record:is_higher/2, Vsns0); +default_versions(dtls) -> + Vsns0 = dtls_record:supported_protocol_versions(), + lists:sort(fun dtls_record:is_higher/2, Vsns0). + +validate_versions(tls, Vsns0) -> + Validate = + fun(Version) -> + try tls_record:sufficient_crypto_support(Version) of + true -> tls_record:protocol_version(Version); + false -> option_error(insufficient_crypto_support, + {Version, {versions, Vsns0}}) + catch error:function_clause -> + option_error(Version, {versions, Vsns0}) + end + end, + Vsns = [Validate(V) || V <- Vsns0], + tls_validate_version_gap(Vsns0), + option_error([] =:= Vsns, versions, Vsns0), + lists:sort(fun tls_record:is_higher/2, Vsns); +validate_versions(dtls, Vsns0) -> + Validate = + fun(Version) -> + try tls_record:sufficient_crypto_support( + dtls_v1:corresponding_tls_version( + dtls_record:protocol_version(Version))) of + true -> dtls_record:protocol_version(Version); + false-> option_error(insufficient_crypto_support, + {Version, {versions, Vsns0}}) + catch error:function_clause -> + option_error(Version, {versions, Vsns0}) + end + end, + Vsns = [Validate(V) || V <- Vsns0], + option_error([] =:= Vsns, versions, Vsns0), + lists:sort(fun dtls_record:is_higher/2, Vsns). + +opt_verification(UserOpts, Opts0, #{role := Role} = Env) -> + {Verify, Opts} = + case get_opt_of(verify, [verify_none, verify_peer], verify_none, UserOpts, Opts0) of + {default, verify_none} when Role =:= client -> + {verify_none, Opts0#{warn_verify_none => true, verify => verify_none}}; + {_, verify_none} -> + {verify_none, Opts0#{verify => verify_none}}; + {_, verify_peer} -> + %% If 'verify' is changed from verify_none to verify_peer, (via update_options/3) + %% the 'verify_fun' must also be changed to undefined. + %% i.e remove the default verify_none fun + {verify_peer, Opts0#{verify => verify_peer, verify_fun => undefined}} + end, + {_, PartialChain} = get_opt_fun(partial_chain, 1, fun(_) -> unknown_ca end, UserOpts, Opts), + + {_, FailNoPeerCert} = get_opt_bool(fail_if_no_peer_cert, false, UserOpts, Opts), + assert_server_only(Role, FailNoPeerCert, fail_if_no_peer_cert), + option_incompatible(FailNoPeerCert andalso Verify =:= verify_none, + [{verify, verify_none}, {fail_if_no_peer_cert, true}]), + + {_, Depth} = get_opt_int(depth, 0, 255, 10, UserOpts, Opts), + + opt_verify_fun(UserOpts, Opts#{partial_chain => PartialChain, + fail_if_no_peer_cert => FailNoPeerCert, + depth => Depth + }, + Env). + +opt_verify_fun(UserOpts, Opts, _Env) -> + DefVerifyNoneFun = {default_verify_fun(), []}, + VerifyFun = case get_opt(verify_fun, DefVerifyNoneFun, UserOpts, Opts) of + {_, {F,_} = FA} when is_function(F, 3) -> + FA; + {_, UserFun} when is_function(UserFun, 1) -> + {convert_verify_fun(), UserFun}; + {_, undefined} -> + undefined; + {_, Value} -> + option_error(verify_fun, Value) + end, + Opts#{verify_fun => VerifyFun}. + +default_verify_fun() -> + fun(_, {bad_cert, _}, UserState) -> + {valid, UserState}; + (_, {extension, #'Extension'{critical = true}}, UserState) -> + %% This extension is marked as critical, so + %% certificate verification should fail if we don't + %% understand the extension. However, this is + %% `verify_none', so let's accept it anyway. + {valid, UserState}; + (_, {extension, _}, UserState) -> + {unknown, UserState}; + (_, valid, UserState) -> + {valid, UserState}; + (_, valid_peer, UserState) -> + {valid, UserState} end. -handle_option(anti_replay = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(anti_replay = Option, Value0, - #{session_tickets := SessionTickets, - versions := Versions} = OptionsMap, #{rules := Rules}) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - assert_option_dependency(Option, session_tickets, [SessionTickets], [stateless]), - case SessionTickets of - stateless -> - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; - _ -> - OptionsMap#{Option => default_value(Option, Rules)} - end; -handle_option(beast_mitigation = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(beast_mitigation = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(cacertfile = Option, unbound, #{cacerts := CaCerts, - verify := Verify, - verify_fun := VerifyFun} = OptionsMap, _Env) - when Verify =:= verify_none orelse - Verify =:= 0 -> - Value = validate_option(Option, ca_cert_default(verify_none, VerifyFun, CaCerts)), - OptionsMap#{Option => Value}; -handle_option(cacertfile = Option, unbound, #{cacerts := CaCerts, - verify := Verify, - verify_fun := VerifyFun} = OptionsMap, _Env) - when Verify =:= verify_peer orelse - Verify =:= 1 orelse - Verify =:= 2 -> - Value = validate_option(Option, ca_cert_default(verify_peer, VerifyFun, CaCerts)), - OptionsMap#{Option => Value}; -handle_option(cacertfile = Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(ciphers = Option, unbound, #{versions := Versions} = OptionsMap, #{rules := Rules}) -> - Value = handle_cipher_option(default_value(Option, Rules), Versions), - OptionsMap#{Option => Value}; -handle_option(ciphers = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - Value = handle_cipher_option(Value0, Versions), - OptionsMap#{Option => Value}; -handle_option(client_renegotiation = Option, unbound, OptionsMap, #{role := Role}) -> - Value = default_option_role(server, true, Role), - OptionsMap#{Option => Value}; -handle_option(client_renegotiation = Option, Value0, - #{versions := Versions} = OptionsMap, #{role := Role}) -> - assert_role(server_only, Role, Option, Value0), - assert_option_dependency(Option, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(early_data = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(early_data = Option, Value0, #{session_tickets := SessionTickets, - versions := Versions} = OptionsMap, - #{role := server = Role}) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - assert_option_dependency(Option, session_tickets, [SessionTickets], - [stateful, stateless]), - Value = validate_option(Option, Value0, Role), - OptionsMap#{Option => Value}; -handle_option(early_data = Option, Value0, #{session_tickets := SessionTickets, - use_ticket := UseTicket, - versions := Versions} = OptionsMap, - #{role := client = Role}) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - assert_option_dependency(Option, session_tickets, [SessionTickets], - [manual, auto]), - case UseTicket of - undefined when SessionTickets =/= auto -> - throw({error, {options, dependency, {Option, use_ticket}}}); - _ -> - ok - end, - Value = validate_option(Option, Value0, Role), - OptionsMap#{Option => Value}; -handle_option(eccs = Option, unbound, #{versions := [HighestVersion|_]} = OptionsMap, #{rules := _Rules}) -> - Value = handle_eccs_option(eccs(), HighestVersion), - OptionsMap#{Option => Value}; -handle_option(eccs = Option, Value0, #{versions := [HighestVersion|_]} = OptionsMap, _Env) -> - Value = handle_eccs_option(Value0, HighestVersion), - OptionsMap#{Option => Value}; -handle_option(fallback = Option, unbound, OptionsMap, #{role := Role}) -> - Value = default_option_role(client, false, Role), - OptionsMap#{Option => Value}; -handle_option(fallback = Option, Value0, OptionsMap, #{role := Role}) -> - assert_role(client_only, Role, Option, Value0), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(certificate_authorities = Option, unbound, OptionsMap, #{role := server}) -> - OptionsMap#{Option => true}; -handle_option(certificate_authorities = Option, unbound, OptionsMap, #{role := client}) -> - OptionsMap#{Option => false}; -handle_option(certificate_authorities = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(cookie = Option, unbound, OptionsMap, #{role := Role}) -> - Value = default_option_role(server, true, Role), - OptionsMap#{Option => Value}; -handle_option(cookie = Option, Value0, #{versions := Versions} = OptionsMap, #{role := Role}) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - assert_role(server_only, Role, Option, Value0), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(honor_cipher_order = Option, unbound, OptionsMap, #{role := Role}) -> - Value = default_option_role(server, false, Role), - OptionsMap#{Option => Value}; -handle_option(honor_cipher_order = Option, Value0, OptionsMap, #{role := Role}) -> - assert_role(server_only, Role, Option, Value0), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(honor_ecc_order = Option, unbound, OptionsMap, #{role := Role}) -> - Value = default_option_role(server, false, Role), - OptionsMap#{Option => Value}; -handle_option(honor_ecc_order = Option, Value0, OptionsMap, #{role := Role}) -> - assert_role(server_only, Role, Option, Value0), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(keyfile = Option, unbound, #{certfile := CertFile} = OptionsMap, _Env) -> - Value = validate_option(Option, CertFile), - OptionsMap#{Option => Value}; -handle_option(key_update_at = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(key_update_at = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(next_protocols_advertised = Option, unbound, OptionsMap, - #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(next_protocols_advertised = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(next_protocols_advertised, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(next_protocol_selector = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = default_value(Option, Rules), - OptionsMap#{Option => Value}; -handle_option(next_protocol_selector = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(client_preferred_next_protocols, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = make_next_protocol_selector( - validate_option(client_preferred_next_protocols, Value0)), - OptionsMap#{Option => Value}; -handle_option(padding_check = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(padding_check = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(password = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{password => Value}; -handle_option(password = Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{password => Value}; -handle_option(certs_keys, unbound, OptionsMap, _Env) -> - OptionsMap; -handle_option(certs_keys = Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{certs_keys => Value}; -handle_option(psk_identity = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(psk_identity = Option, Value0, #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(secure_renegotiate = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(secure_renegotiate= Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(secure_renegotiate, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(reuse_session = Option, unbound, OptionsMap, #{role := Role}) -> - Value = - case Role of - client -> - undefined; - server -> - fun(_, _, _, _) -> true end - end, - OptionsMap#{Option => Value}; -handle_option(reuse_session = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(reuse_session, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -%% TODO: validate based on role -handle_option(reuse_sessions = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(reuse_sessions = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(reuse_sessions, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(server_name_indication = Option, unbound, OptionsMap, #{host := Host, - role := Role}) -> - Value = default_option_role(client, server_name_indication_default(Host), Role), - OptionsMap#{Option => Value}; -handle_option(server_name_indication = Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(session_tickets = Option, unbound, OptionsMap, #{role := Role, - rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules), Role), - OptionsMap#{Option => Value}; -handle_option(session_tickets = Option, Value0, #{versions := Versions} = OptionsMap, #{role := Role}) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - Value = validate_option(Option, Value0, Role), - OptionsMap#{Option => Value}; -handle_option(signature_algs = Option, unbound, #{versions := [HighestVersion | _] = Versions} = OptionsMap, #{role := Role}) -> - Value = - handle_hashsigns_option( - default_option_role_sign_algs( - server, - tls_v1:default_signature_algs(Versions), - Role, - HighestVersion), - tls_version(HighestVersion)), - OptionsMap#{Option => Value}; -handle_option(signature_algs = Option, Value0, #{versions := [HighestVersion|_]} = OptionsMap, _Env) -> - Value = handle_hashsigns_option(Value0, tls_version(HighestVersion)), - OptionsMap#{Option => Value}; -handle_option(signature_algs_cert = Option, unbound, #{versions := [HighestVersion|_]} = OptionsMap, _Env) -> - %% Do not send by default - Value = handle_signature_algorithms_option(undefined, tls_version(HighestVersion)), - OptionsMap#{Option => Value}; -handle_option(signature_algs_cert = Option, Value0, #{versions := [HighestVersion|_]} = OptionsMap, _Env) -> - Value = handle_signature_algorithms_option(Value0, tls_version(HighestVersion)), - OptionsMap#{Option => Value}; -handle_option(sni_fun = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = default_value(Option, Rules), - OptionsMap#{Option => Value}; -handle_option(sni_fun = Option, Value0, OptionsMap, _Env) -> - validate_option(Option, Value0), - OptHosts = maps:get(sni_hosts, OptionsMap, undefined), - Value = - case {Value0, OptHosts} of - {undefined, _} -> - Value0; - {_, []} -> - Value0; - _ -> - throw({error, {conflict_options, [sni_fun, sni_hosts]}}) +convert_verify_fun() -> + fun(_,{bad_cert, _} = Reason, OldFun) -> + case OldFun([Reason]) of + true -> {valid, OldFun}; + false -> {fail, Reason} + end; + (_,{extension, _}, UserState) -> + {unknown, UserState}; + (_, valid, UserState) -> + {valid, UserState}; + (_, valid_peer, UserState) -> + {valid, UserState} + end. + +opt_certs(UserOpts, #{log_level := LogLevel} = Opts0, Env) -> + Opts = case get_opt_list(certs_keys, unbound, UserOpts, Opts0) of + {_, unbound} -> + opt_old_certs(UserOpts, Opts0, Env); + {Where, CKs} when is_list(CKs) -> + warn_override(Where, UserOpts, certs_keys, [cert,certfile,key,keyfile,password], LogLevel), + Opts0#{certs_keys => CKs} + end, + opt_cacerts(UserOpts, Opts, Env). + +opt_old_certs(UserOpts, #{log_level := LogLevel}=Opts, _Env) -> + CertOpts = Opts, %% #{} FIXME remove and always make a cert_keys list here + + CertKeys0 = case get_opt(cert, undefined, UserOpts, Opts) of + {Where, Cert} when is_binary(Cert) -> + warn_override(Where, UserOpts, cert, [certfile], LogLevel), + CertOpts#{cert => [Cert]}; + {Where, [C0|_] = Certs} when is_binary(C0) -> + warn_override(Where, UserOpts, cert, [certfile], LogLevel), + CertOpts#{cert => Certs}; + {new, Err0} -> + option_error(cert, Err0); + {_, undefined} -> + case get_opt_file(certfile, unbound, UserOpts, Opts) of + {default, unbound} -> CertOpts#{cert => undefined}; + {_, CertFile} -> CertOpts#{certfile => CertFile} + end + end, + + CertKeys1 = case get_opt(key, undefined, UserOpts, Opts) of + {_, undefined} -> + case get_opt_file(keyfile, <<>>, UserOpts, Opts) of + {new, KeyFile} -> + CertKeys0#{keyfile => KeyFile}; + {_, <<>>} -> + case maps:get(certfile, CertKeys0, unbound) of + unbound -> CertKeys0#{key => undefined}; + CF -> CertKeys0#{keyfile => CF} + end; + {old, _} -> + CertKeys0 + end; + {_, {KF, K0} = Key} + when is_binary(K0), KF =:= rsa; KF =:= dsa; + KF == 'RSAPrivateKey'; KF == 'DSAPrivateKey'; + KF == 'ECPrivateKey'; KF == 'PrivateKeyInfo' -> + CertKeys0#{key => Key}; + {_, #{engine := _, key_id := _, algorithm := _} = Key} -> + CertKeys0#{key => Key}; + {new, Err1} -> + option_error(key, Err1) + end, + + CertKeys2 = case get_opt(password, unbound, UserOpts, Opts) of + {default, _} -> CertKeys1; + {_, Pwd} when is_binary(Pwd); is_list(Pwd); is_function(Pwd, 0) -> + CertKeys1#{password => Pwd}; + {_, Err2} -> + option_error(password, Err2) + end, + %% Compatibility FIXME remove these but check usage first + CertKeys3 = case maps:is_key(certfile, CertKeys2) of + true -> CertKeys2; + false -> CertKeys2#{certfile => <<>>} + end, + CertKeys4 = case maps:is_key(keyfile, CertKeys3) of + true -> CertKeys3; + false -> CertKeys3#{keyfile => <<>>} + end, + CertKeys5 = case maps:is_key(password, CertKeys4) of + true -> CertKeys4; + false -> CertKeys4#{password => ""} + end, + CertKeys6 = case maps:is_key(cert, CertKeys5) of + true -> CertKeys5; + false -> CertKeys5#{cert => undefined} + end, + CertKeys7 = case maps:is_key(key, CertKeys6) of + true -> CertKeys6; + false -> CertKeys6#{key => undefined} + end, + CertKeys7. + +opt_cacerts(UserOpts, #{verify := Verify, log_level := LogLevel, versions := Versions} = Opts, + #{role := Role}) -> + {_, CaCerts} = get_opt_list(cacerts, undefined, UserOpts, Opts), + + CaCertFile = case get_opt_file(cacertfile, <<>>, UserOpts, Opts) of + {Where1, _FileName} when CaCerts =/= undefined -> + warn_override(Where1, UserOpts, cacerts, [cacertfile], LogLevel), + <<>>; + {new, FileName} -> unambiguous_path(FileName); + {_, FileName} -> FileName + end, + option_incompatible(CaCertFile =:= <<>> andalso CaCerts =:= undefined andalso Verify =:= verify_peer, + [{verify, verify_peer}, {cacerts, undefined}]), + + {Where2, CA} = get_opt_bool(certificate_authorities, Role =:= server, UserOpts, Opts), + assert_version_dep(Where2 =:= new, certificate_authorities, Versions, ['tlsv1.3']), + + Opts#{cacerts => CaCerts, cacertfile => CaCertFile, certificate_authorities => CA}. + +opt_tickets(UserOpts, #{versions := Versions} = Opts, #{role := client}) -> + {_, SessionTickets} = get_opt_of(session_tickets, [disabled,manual,auto], disabled, UserOpts, Opts), + assert_version_dep(SessionTickets =/= disabled, session_tickets, Versions, ['tlsv1.3']), + + {_, UseTicket} = get_opt_list(use_ticket, undefined, UserOpts, Opts), + option_error(UseTicket =:= [], use_ticket, UseTicket), + option_incompatible(UseTicket =/= undefined andalso SessionTickets =/= manual, + [{use_ticket, UseTicket}, {session_tickets, SessionTickets}]), + + {_, EarlyData} = get_opt_bin(early_data, undefined, UserOpts, Opts), + option_incompatible(is_binary(EarlyData) andalso SessionTickets =:= disabled, + [early_data, {session_tickets, disabled}]), + option_incompatible(is_binary(EarlyData) andalso SessionTickets =:= manual andalso UseTicket =:= undefined, + [early_data, {session_tickets, manual}, {use_ticket, undefined}]), + + assert_server_only(anti_replay, UserOpts), + assert_server_only(stateless_tickets_seed, UserOpts), + Opts#{session_tickets => SessionTickets, use_ticket => UseTicket, early_data => EarlyData}; +opt_tickets(UserOpts, #{versions := Versions} = Opts, #{role := server}) -> + {_, SessionTickets} = get_opt_of(session_tickets, [disabled, stateful, stateless], disabled, UserOpts, Opts), + assert_version_dep(SessionTickets =/= disabled, session_tickets, Versions, ['tlsv1.3']), + + {_, EarlyData} = get_opt_of(early_data, [enabled, disabled], disabled, UserOpts, Opts), + option_incompatible(SessionTickets =:= disabled andalso EarlyData =:= enabled, + [early_data, {session_tickets, disabled}]), + + AntiReplay = + case get_opt(anti_replay, undefined, UserOpts, Opts) of + {_, undefined} -> undefined; + {_,AR} when SessionTickets =/= stateless -> + option_incompatible([{anti_replay, AR}, {session_tickets, SessionTickets}]); + {_,'10k'} -> {10, 5, 72985}; %% n = 10000 p = 0.030003564 (1 in 33) m = 72985 (8.91KiB) k = 5 + {_,'100k'} -> {10, 5, 729845}; %% n = 10000 p = 0.03000428 (1 in 33) m = 729845 (89.09KiB) k = 5 + {_, {_,_,_} = AR} -> AR; + {_, AR} -> option_error(anti_replay, AR) end, - OptionsMap#{Option => Value}; -handle_option(srp_identity = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(srp_identity = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(srp_identity, versions, Versions, - ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(supported_groups = Option, unbound, #{versions := [HighestVersion|_]} = OptionsMap, #{rules := _Rules}) -> - Value = handle_supported_groups_option(groups(default), HighestVersion), - OptionsMap#{Option => Value}; -handle_option(supported_groups = Option, Value0, - #{versions := [HighestVersion|_] = Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - Value = handle_supported_groups_option(Value0, HighestVersion), - OptionsMap#{Option => Value}; -handle_option(use_ticket = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(use_ticket = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1.3']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(user_lookup_fun = Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(user_lookup_fun = Option, Value0, - #{versions := Versions} = OptionsMap, _Env) -> - assert_option_dependency(Option, versions, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(verify = Option, unbound, OptionsMap, #{rules := Rules}) -> - handle_verify_option(default_value(Option, Rules), OptionsMap#{warn_verify_none => true}); -handle_option(verify = _Option, Value, OptionsMap, _Env) -> - handle_verify_option(Value, OptionsMap); -handle_option(verify_fun = Option, unbound, #{verify := Verify} = OptionsMap, #{rules := Rules}) - when Verify =:= verify_none -> - OptionsMap#{Option => default_value(Option, Rules)}; -handle_option(verify_fun = Option, unbound, #{verify := Verify} = OptionsMap, _Env) - when Verify =:= verify_peer -> - OptionsMap#{Option => undefined}; -handle_option(verify_fun = Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}; -handle_option(versions = Option, unbound, #{protocol := Protocol} = OptionsMap, _Env) -> - RecordCb = record_cb(Protocol), - Vsns0 = RecordCb:supported_protocol_versions(), - Value = lists:sort(fun RecordCb:is_higher/2, Vsns0), - OptionsMap#{Option => Value}; -handle_option(versions = Option, Vsns0, #{protocol := Protocol} = OptionsMap, _Env) -> - validate_option(versions, Vsns0), - RecordCb = record_cb(Protocol), - Vsns1 = [RecordCb:protocol_version(Vsn) || Vsn <- Vsns0], - Value = lists:sort(fun RecordCb:is_higher/2, Vsns1), - OptionsMap#{Option => Value}; -%% Special options -handle_option(cb_info = Option, unbound, #{protocol := Protocol} = OptionsMap, _Env) -> - Default = default_cb_info(Protocol), - validate_option(Option, Default), - Value = handle_cb_info(Default), - OptionsMap#{Option => Value}; -handle_option(cb_info = Option, Value0, OptionsMap, _Env) -> - validate_option(Option, Value0), - Value = handle_cb_info(Value0), - OptionsMap#{Option => Value}; -%% Generic case -handle_option(Option, unbound, OptionsMap, #{rules := Rules}) -> - Value = validate_option(Option, default_value(Option, Rules)), - OptionsMap#{Option => Value}; -handle_option(Option, Value0, OptionsMap, _Env) -> - Value = validate_option(Option, Value0), - OptionsMap#{Option => Value}. -handle_option_cb_info(Options, Protocol) -> - Value = proplists:get_value(cb_info, Options, default_cb_info(Protocol)), - #{cb_info := CbInfo} = handle_option(cb_info, Value, #{protocol => Protocol}, #{}), - CbInfo. + {_, STS} = get_opt_bin(stateless_tickets_seed, undefined, UserOpts, Opts), + option_incompatible(STS =/= undefined andalso SessionTickets =/= stateless, + [stateless_tickets_seed, {session_tickets, SessionTickets}]), + + assert_client_only(use_ticket, UserOpts), + Opts#{session_tickets => SessionTickets, early_data => EarlyData, + anti_replay => AntiReplay, stateless_tickets_seed => STS}. + +opt_ocsp(UserOpts, #{versions := _Versions} = Opts, #{role := Role}) -> + {_, Stapling} = get_opt_bool(ocsp_stapling, false, UserOpts, Opts), + assert_client_only(Role, Stapling, ocsp_stapling), + + {_, Nonce} = get_opt_bool(ocsp_nonce, true, UserOpts, Opts), + option_incompatible(Stapling =:= false andalso Nonce =:= false, [{ocsp_nonce, false}, {ocsp_stapling, false}]), + + {_, ORC} = get_opt_list(ocsp_responder_certs, [], UserOpts, Opts), + option_incompatible(Stapling =:= false andalso ORC =/= [], [ocsp_responder_certs, {ocsp_stapling, false}]), + %% FIXME should not be decoded in users process !! + Decode = fun(CertDer) -> + try public_key:pkix_decode_cert(CertDer, plain) + catch _:_ -> option_error(ocsp_responder_certs, ORC) + end + end, + Certs = [Decode(CertDer) || CertDer <- ORC], + %% FIXME make it one option? + Opts#{ocsp_stapling => Stapling, ocsp_nonce => Nonce, ocsp_responder_certs => Certs}. + +opt_sni(UserOpts, #{versions := _Versions} = Opts, #{role := server}) -> + {_, SniHosts} = get_opt_list(sni_hosts, [], UserOpts, Opts), + %% Postpone option checking until all other options are checked FIXME + Check = fun({[_|_], SO}) when is_list(SO) -> + case proplists:get_value(sni_hosts, SO, undefined) of + undefined -> ok; + Recursive -> option_error(sni_hosts, Recursive) + end; + (HostOpts) -> option_error(sni_hosts, HostOpts) + end, + [Check(E) || E <- SniHosts], + + {_, SniFun} = get_opt_fun(sni_fun, 1, undefined, UserOpts, Opts), + + option_incompatible(is_function(SniFun) andalso SniHosts =/= [], + [sni_fun, sni_hosts]), + assert_client_only(server_name_indication, UserOpts), + %% FIXME: remove sni_hosts and + %% sni_fun = fun(SNIHostName) -> proplists:get_value(SNIHostname, SNIHosts) end, + Opts#{sni_fun => SniFun, sni_hosts => SniHosts, + server_name_indication => undefined %% FIXME + }; +opt_sni(UserOpts, #{versions := _Versions} = Opts, #{role := client} = Env) -> + %% RFC 6066, Section 3: Currently, the only server names supported are + %% DNS hostnames + %% case inet_parse:domain(Value) of + %% false -> + %% throw({error, {options, {{Opt, Value}}}}); + %% true -> + %% Value + %% end; + %% + %% But the definition seems very diffuse, so let all strings through + %% and leave it up to public_key to decide... + SNI = case get_opt(server_name_indication, unbound, UserOpts, Opts) of + {_, unbound} -> server_name_indication_default(maps:get(host, Env, undefined)); + {_, [_|_] = SN} -> SN; + {_, disable} -> disable; + {_, SN} -> option_error(server_name_indication, SN) + end, + assert_server_only(sni_fun, UserOpts), + assert_server_only(sni_hosts, UserOpts), + Opts#{server_name_indication => SNI}. +server_name_indication_default(Host) when is_list(Host) -> + %% SNI should not contain a trailing dot that a hostname may + string:strip(Host, right, $.); +server_name_indication_default(_) -> + undefined. -maybe_map_key_internal(client_preferred_next_protocols) -> - next_protocol_selector; -maybe_map_key_internal(K) -> - K. +opt_signature_algs(UserOpts, #{versions := Versions} = Opts, _Env) -> + [TlsVersion|_] = TlsVsns = [tls_version(V) || V <- Versions], + SA = case get_opt_list(signature_algs, undefined, UserOpts, Opts) of + {default, undefined} when TlsVersion >= {3,3} -> + DefAlgs = tls_v1:default_signature_algs(TlsVsns), + handle_hashsigns_option(DefAlgs, TlsVersion); + {new, Algs} -> + assert_version_dep(signature_algs, Versions, ['tlsv1.2', 'tlsv1.3']), + SA0 = handle_hashsigns_option(Algs, TlsVersion), + option_error(SA0 =:= [], no_supported_algorithms, {signature_algs, Algs}), + SA0; + {_, Algs} -> + Algs + end, + SAC = case get_opt_list(signature_algs_cert, undefined, UserOpts, Opts) of + {new, Schemes} -> + %% Do not send by default + assert_version_dep(signature_algs_cert, Versions, ['tlsv1.2', 'tlsv1.3']), + SAC0 = handle_signature_algorithms_option(Schemes, TlsVersion), + option_error(SAC0 =:= [], no_supported_signature_schemes, {signature_algs_cert, Schemes}), + SAC0; + {_, Schemes} -> + Schemes + end, + Opts#{signature_algs => SA, signature_algs_cert => SAC}. + +opt_alpn(UserOpts, #{versions := Versions} = Opts, #{role := server}) -> + {_, APP} = get_opt_list(alpn_preferred_protocols, undefined, UserOpts, Opts), + validate_protocols(is_list(APP), alpn_preferred_protocols, APP), + + {_, NPA} = get_opt_list(next_protocols_advertised, undefined, UserOpts, Opts), + validate_protocols(is_list(NPA), next_protocols_advertised, NPA), + assert_version_dep(is_list(NPA), next_protocols_advertised, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + + assert_client_only(alpn_advertised_protocols, UserOpts), + assert_client_only(client_preferred_next_protocols, UserOpts), + + Opts#{alpn_preferred_protocols => APP, next_protocols_advertised => NPA, + alpn_advertised_protocols => undefined, next_protocol_selector => undefined %% FIXME remove + }; +opt_alpn(UserOpts, #{versions := Versions} = Opts, #{role := client}) -> + {_, AAP} = get_opt_list(alpn_advertised_protocols, undefined, UserOpts, Opts), + validate_protocols(is_list(AAP), alpn_advertised_protocols, AAP), + + NPS = case get_opt(client_preferred_next_protocols, undefined, UserOpts, Opts) of + {new, CPNP} -> + assert_version_dep(client_preferred_next_protocols, + Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + make_next_protocol_selector(CPNP); + {_, CPNP} -> CPNP + end, -maybe_map_key_external(next_protocol_selector) -> - client_preferred_next_protocols; -maybe_map_key_external(K) -> - K. + validate_protocols(is_list(NPS), client_preferred_next_protocols, NPS), -check_dependencies(K, OptionsMap, Env) -> - Rules = maps:get(rules, Env), - Deps = get_dependencies(K, Rules), - case Deps of - [] -> - true; - L -> - option_already_defined(K,OptionsMap) orelse - dependecies_already_defined(L, OptionsMap) + assert_server_only(alpn_preferred_protocols, UserOpts), + assert_server_only(next_protocols_advertised, UserOpts), + + Opts#{alpn_preferred_protocols => undefined, next_protocols_advertised => undefined, %% FIXME remove + alpn_advertised_protocols => AAP, next_protocol_selector => NPS + }. + +validate_protocols(false, _Opt, _List) -> ok; +validate_protocols(true, Opt, List) -> + Check = fun(Bin) -> + IsOK = is_binary(Bin) andalso byte_size(Bin) > 0 andalso byte_size(Bin) < 256, + option_error(not IsOK, Opt, {invalid_protocol, Bin}) + end, + lists:foreach(Check, List). + +opt_mitigation(UserOpts, #{versions := Versions} = Opts, _Env) -> + {Where1, BM} = get_opt_of(beast_mitigation, [disabled, one_n_minus_one, zero_n], one_n_minus_one, UserOpts, Opts), + assert_version_dep(Where1 =:= new, beast_mitigation, Versions, ['tlsv1']), + + {Where2, PC} = get_opt_bool(padding_check, true, UserOpts, Opts), + assert_version_dep(Where2 =:= new, padding_check, Versions, ['tlsv1']), + + Opts#{beast_mitigation => BM, padding_check => PC}. + +opt_server(UserOpts, #{versions := Versions} = Opts, #{role := server}) -> + {_, ECC} = get_opt_bool(honor_ecc_order, false, UserOpts, Opts), + + {_, Cipher} = get_opt_bool(honor_cipher_order, false, UserOpts, Opts), + + {Where1, Cookie} = get_opt_bool(cookie, true, UserOpts, Opts), + assert_version_dep(Where1 =:= new, cookie, Versions, ['tlsv1.3']), + + {Where2, ReNeg} = get_opt_bool(client_renegotiation, true, UserOpts, Opts), + assert_version_dep(Where2 =:= new, client_renegotiation, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + + Opts#{honor_ecc_order => ECC, honor_cipher_order => Cipher, + cookie => Cookie, client_renegotiation => ReNeg}; +opt_server(UserOpts, Opts, #{role := client}) -> + assert_server_only(honor_ecc_order, UserOpts), + assert_server_only(honor_cipher_order, UserOpts), + assert_server_only(cookie, UserOpts), + assert_server_only(client_renegotiation, UserOpts), + Opts. + +opt_client(UserOpts, #{versions := Versions} = Opts, #{role := client}) -> + {Where, FB} = get_opt_bool(fallback, false, UserOpts, Opts), + assert_version_dep(Where =:= new, fallback, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + + {_, CHC} = get_opt_list(customize_hostname_check, [], UserOpts, Opts), + + ValidMFL = [undefined, ?MAX_FRAGMENT_LENGTH_BYTES_1, ?MAX_FRAGMENT_LENGTH_BYTES_2, %% RFC 6066, Section 4 + ?MAX_FRAGMENT_LENGTH_BYTES_3, ?MAX_FRAGMENT_LENGTH_BYTES_4], + {_, MFL} = get_opt_of(max_fragment_length, ValidMFL, undefined, UserOpts, Opts), + + Opts#{fallback => FB, customize_hostname_check => CHC, max_fragment_length => MFL}; +opt_client(UserOpts, Opts, #{role := server}) -> + assert_client_only(fallback, UserOpts), + assert_client_only(customize_hostname_check, UserOpts), + assert_client_only(max_fragment_length, UserOpts), + Opts#{customize_hostname_check => []}. + +opt_renegotiate(UserOpts, #{versions := Versions} = Opts, _Env) -> + {Where1, KUA} = get_opt_pos_int(key_update_at, ?KEY_USAGE_LIMIT_AES_GCM, UserOpts, Opts), + assert_version_dep(Where1 =:= new, key_update_at, Versions, ['tlsv1.3']), + + %% Undocumented, old ? + {_, RA0} = get_opt_pos_int(renegotiate_at, ?DEFAULT_RENEGOTIATE_AT, UserOpts, Opts), + RA = min(RA0, ?DEFAULT_RENEGOTIATE_AT), %% Override users choice without notifying ?? + + {Where3, SR} = get_opt_bool(secure_renegotiate, true, UserOpts, Opts), + assert_version_dep(Where3 =:= new, secure_renegotiate, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + + Opts#{secure_renegotiate => SR, key_update_at => KUA, renegotiate_at => RA}. + +opt_reuse_sessions(UserOpts, #{versions := Versions} = Opts, #{role := client}) -> + {Where1, RUSS} = get_opt_of(reuse_sessions, [true, false, save], true, UserOpts, Opts), + + {Where2, RS} = RST = get_opt(reuse_session, undefined, UserOpts, Opts), + case RST of + {new, Bin} when is_binary(Bin) -> ok; + {new, {B1,B2}} when is_binary(B1), is_binary(B2) -> ok; + {new, Bad} -> option_error(reuse_session, Bad); + {_, _} -> ok + end, + + assert_version_dep(Where1 =:= new, reuse_sessions, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + assert_version_dep(Where2 =:= new, reuse_session, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + Opts#{reuse_sessions => RUSS, reuse_session => RS}; +opt_reuse_sessions(UserOpts, #{versions := Versions} = Opts, #{role := server}) -> + {Where1, RUSS} = get_opt_bool(reuse_sessions, true, UserOpts, Opts), + + DefRS = fun(_, _, _, _) -> true end, + {Where2, RS} = get_opt_fun(reuse_session, 4, DefRS, UserOpts, Opts), + + assert_version_dep(Where1 =:= new, reuse_sessions, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + assert_version_dep(Where2 =:= new, reuse_session, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + Opts#{reuse_sessions => RUSS, reuse_session => RS}. + +opt_identity(UserOpts, #{versions := Versions} = Opts, _Env) -> + PSK = case get_opt_list(psk_identity, undefined, UserOpts, Opts) of + {new, PSK0} -> + PSK1 = unicode:characters_to_binary(PSK0), + PSKSize = byte_size(PSK1), + assert_version_dep(psk_identity, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + option_error(not (0 < PSKSize andalso PSKSize < 65536), + psk_identity, {psk_identity, PSK0}), + PSK1; + {_, PSK0} -> + PSK0 + end, + + SRP = case get_opt(srp_identity, undefined, UserOpts, Opts) of + {new, {S1, S2}} when is_list(S1), is_list(S2) -> + User = unicode:characters_to_binary(S1), + UserSize = byte_size(User), + assert_version_dep(srp_identity, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + option_error(not (0 < UserSize andalso UserSize < 65536), + srp_identity, {srp_identity, PSK0}), + {User, unicode:characters_to_binary(S2)}; + {new, Err} -> + option_error(srp_identity, Err); + {_, SRP0} -> + SRP0 + end, + + ULF = case get_opt(user_lookup_fun, undefined, UserOpts, Opts) of + {new, {Fun, _} = ULF0} when is_function(Fun, 3) -> + assert_version_dep(user_lookup_fun, Versions, ['tlsv1','tlsv1.1','tlsv1.2']), + ULF0; + {new, ULF0} -> + option_error(user_lookup_fun, ULF0); + {_, ULF0} -> + ULF0 + end, + + Opts#{psk_identity => PSK, srp_identity => SRP, user_lookup_fun => ULF}. + +opt_supported_groups(UserOpts, #{versions := Versions, log_level := LogLevel} = Opts0, _Env) -> + [TlsVersion|_] = TlsVsns = [tls_version(V) || V <- Versions], + SG = case get_opt_list(supported_groups, undefined, UserOpts, Opts0) of + {default, undefined} -> + handle_supported_groups_option(groups(default), TlsVersion); + {new, SG0} -> + assert_version_dep(supported_groups, TlsVsns, ['tlsv1.3']), + handle_supported_groups_option(SG0, TlsVersion); + {old, SG0} -> + SG0 + end, + + Opts = case get_opt(dh, undefined, UserOpts, Opts0) of + {Where, DH} when is_binary(DH) -> + warn_override(Where, UserOpts, dh, [dhfile], LogLevel), + Opts0#{dh => DH, dhfile => undefined}; + {new, DH} -> + option_error(dh, DH); + {_, undefined} -> + case get_opt_file(dhfile, unbound, UserOpts, Opts0) of + {default, unbound} -> Opts0#{dh => undefined, dhfile => undefined}; + {_, DHFile} -> Opts0#{dh => undefined, dhfile => DHFile} + end + end, + + CPHS = case get_opt_list(ciphers, [], UserOpts, Opts) of + {old, CPS0} -> CPS0; + {_, CPS0} -> handle_cipher_option(CPS0, Versions) + end, + + ECCS = case get_opt_list(eccs, undefined, UserOpts, Opts) of + {old, ECCS0} -> ECCS0; + {default, _} -> handle_eccs_option(tls_v1:ecc_curves(all), TlsVersion); + {new, ECCS0} -> handle_eccs_option(ECCS0, TlsVersion) + end, + + Opts#{ciphers => CPHS, eccs => ECCS, supported_groups => SG}. + +opt_crl(UserOpts, Opts, _Env) -> + {_, Check} = get_opt_of(crl_check, [best_effort, peer, true, false], false, UserOpts, Opts), + Cache = case get_opt(crl_cache, {ssl_crl_cache, {internal, []}}, UserOpts, Opts) of + {_, {Cb, {_Handle, Options}} = Value} when is_atom(Cb), is_list(Options) -> + Value; + {_, Err} -> + option_error(crl_cache, Err) + end, + Opts#{crl_check => Check, crl_cache => Cache}. + +opt_handshake(UserOpts, Opts, _Env) -> + {_, HS} = get_opt_of(handshake, [hello, full], full, UserOpts, Opts), + + {_, MHSS} = get_opt_int(max_handshake_size, 1, ?MAX_UNIT24, ?DEFAULT_MAX_HANDSHAKE_SIZE, + UserOpts, Opts), + + Opts#{handshake => HS, max_handshake_size => MHSS}. + +opt_process(UserOpts, Opts, _Env) -> + {_, RSO} = get_opt_list(receiver_spawn_opts, [], UserOpts, Opts), + {_, SSO} = get_opt_list(sender_spawn_opts, [], UserOpts, Opts), + {_, HA} = get_opt_int(hibernate_after, 0, infinity, infinity, UserOpts, Opts), + Opts#{receiver_spawn_opts => RSO, sender_spawn_opts => SSO, hibernate_after => HA}. + +%%%% + +get_opt(Opt, Default, UserOpts, Opts) -> + case maps:get(Opt, UserOpts, unbound) of + unbound -> + case maps:get(maybe_map_key_internal(Opt), Opts, unbound) of + unbound -> %% Uses default value + {default, Default}; + Value -> %% Uses already set value (merge) + {old, Value} + end; + Value -> %% Uses new user option + {new, Value} + end. + +get_opt_of(Opt, Valid, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {new, Value} = Res -> + case lists:member(Value, Valid) of + true -> Res; + false -> option_error(Opt, Value) + end; + Res -> + Res + end. + +get_opt_bool(Opt, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {_, Value} = Res when is_boolean(Value) -> Res; + {_, Value} -> option_error(Opt, Value) + end. + +get_opt_pos_int(Opt, Default, UserOpts, Opts) -> + get_opt_int(Opt, 1, infinity, Default, UserOpts, Opts). + +get_opt_int(Opt, Min, Max, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {_, Value} = Res when is_integer(Value), Min =< Value, Value =< Max -> + Res; + {_, Value} = Res when Value =:= infinity, Max =:= infinity -> + Res; + {_, Value} -> + option_error(Opt, Value) + end. + +get_opt_fun(Opt, Arity, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {_, Fun} = Res when is_function(Fun, Arity) -> Res; + {new, Err} -> option_error(Opt, Err); + Res -> Res end. +get_opt_list(Opt, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {new, Err} when not is_list(Err) -> option_error(Opt, Err); + Res -> Res + end. -%% Handle options that are not present in the map -get_dependencies(K, _) when K =:= cb_info orelse K =:= log_alert-> - []; -get_dependencies(K, Rules) -> - {_, Deps} = maps:get(K, Rules), - Deps. +get_opt_bin(Opt, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {new, Err} when not is_binary(Err) -> option_error(Opt, Err); + Res -> Res + end. +get_opt_file(Opt, Default, UserOpts, Opts) -> + case get_opt(Opt, Default, UserOpts, Opts) of + {new, File} -> {new, validate_filename(File, Opt)}; + Res -> Res + end. -option_already_defined(K, Map) -> - maps:get(K, Map, unbound) =/= unbound. +%%%% -dependecies_already_defined(L, OptionsMap) -> - Fun = fun (E) -> option_already_defined(E, OptionsMap) end, - lists:all(Fun, L). +default_cb_info(tls) -> + {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; +default_cb_info(dtls) -> + {gen_udp, udp, udp_closed, udp_error, udp_passive}. +handle_cb_info({V1, V2, V3, V4}) -> + {V1,V2,V3,V4, list_to_atom(atom_to_list(V2) ++ "_passive")}; +handle_cb_info(CbInfo) when tuple_size(CbInfo) =:= 5 -> + CbInfo; +handle_cb_info(CbInfo) -> + option_error(cb_info, CbInfo). + +handle_option_cb_info(Options, Protocol) -> + CbInfo = proplists:get_value(cb_info, Options, default_cb_info(Protocol)), + handle_cb_info(CbInfo). -expand_options(Opts0, Rules) -> +maybe_map_key_internal(client_preferred_next_protocols) -> + next_protocol_selector; +maybe_map_key_internal(K) -> + K. + +split_options(Opts0, AllOptions) -> Opts1 = proplists:expand([{binary, [{mode, binary}]}, - {list, [{mode, list}]}], Opts0), + {list, [{mode, list}]}], Opts0), Opts2 = handle_option_format(Opts1, []), - %% Remove deprecated ssl_imp option Opts = proplists:delete(ssl_imp, Opts2), - AllOpts = maps:keys(Rules), - SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) end, - Opts, - AllOpts ++ - [ssl_imp, %% TODO: remove ssl_imp - cb_info, - client_preferred_next_protocols, %% next_protocol_selector - log_alert]), %% obsoleted by log_level - - SslOpts0 = Opts -- SockOpts, - SslOpts = {SslOpts0, [], length(SslOpts0)}, - {SslOpts, SockOpts}. - - -add_missing_options({L0, S, _C}, Rules) -> - Fun = fun(K0, Acc) -> - K = maybe_map_key_external(K0), - case proplists:is_defined(K, Acc) of - true -> - Acc; - false -> - Default = unbound, - [{K, Default}|Acc] - end - end, - AllOpts = maps:keys(Rules), - L = lists:foldl(Fun, L0, AllOpts), - {L, S, length(L)}. - -default_value(Key, Rules) -> - {Default, _} = maps:get(Key, Rules, {undefined, []}), - Default. + DeleteUserOpts = fun(Key, PropList) -> proplists:delete(Key, PropList) end, + AllOpts = [cb_info, client_preferred_next_protocols] ++ AllOptions, + SockOpts = lists:foldl(DeleteUserOpts, Opts, AllOpts), + {Opts -- SockOpts, SockOpts}. + +assert_server_only(Option, Opts) -> + Value = maps:get(Option, Opts, undefined), + role_error(Value =/= undefined, server_only, Option). +assert_client_only(Option, Opts) -> + Value = maps:get(Option, Opts, undefined), + role_error(Value =/= undefined, client_only, Option). + +assert_server_only(client, Bool, Option) -> + role_error(Bool, server_only, Option); +assert_server_only(_, _, _) -> + ok. +assert_client_only(server, Bool, Option) -> + role_error(Bool, client_only, Option); +assert_client_only(_, _, _) -> + ok. -assert_role(client_only, client, _, _) -> +role_error(false, _ErrorDesc, _Option) -> ok; -assert_role(server_only, server, _, _) -> - ok; -assert_role(client_only, _, _, undefined) -> - ok; -assert_role(server_only, _, _, undefined) -> - ok; -assert_role(Type, _, Key, _) -> - throw({error, {option, Type, Key}}). +role_error(true, ErrorDesc, Option) + when ErrorDesc =:= client_only; ErrorDesc =:= server_only -> + throw_error({option, ErrorDesc, Option}). -assert_option_dependency(Option, OptionDep, Values0, AllowedValues) -> - case is_dtls_configured(Values0) of - true -> - %% TODO: Check option dependency for DTLS - ok; +option_incompatible(false, _Options) -> ok; +option_incompatible(true, Options) -> option_incompatible(Options). + +-spec option_incompatible(_) -> no_return(). +option_incompatible(Options) -> + throw_error({options, incompatible, Options}). + +option_error(false, _, _What) -> true; +option_error(true, Tag, What) -> option_error(Tag,What). + +-spec option_error(_,_) -> no_return(). +option_error(Tag, What) -> + throw_error({options, {Tag, What}}). + +-spec throw_error(_) -> no_return(). +throw_error(Err) -> + throw({error, Err}). + +assert_version_dep(Option, Vsns, AllowedVsn) -> + assert_version_dep(true, Option, Vsns, AllowedVsn). + +assert_version_dep(false, _, _, _) -> true; +assert_version_dep(true, Option, SSLVsns, AllowedVsn) -> + case is_dtls_configured(SSLVsns) of + true -> %% TODO: Check option dependency for DTLS + true; false -> - %% special handling for version - Values = - case OptionDep of - versions -> - lists:map(fun tls_record:protocol_version/1, Values0); - _ -> - Values0 - end, - Set1 = sets:from_list(Values), - Set2 = sets:from_list(AllowedValues), + APIVsns = lists:map(fun tls_record:protocol_version/1, SSLVsns), + Set1 = sets:from_list(APIVsns), + Set2 = sets:from_list(AllowedVsn), case sets:size(sets:intersection(Set1, Set2)) > 0 of - true -> - ok; - false -> - throw({error, {options, dependency, - {Option, {OptionDep, AllowedValues}}}}) + true -> ok; + false -> option_incompatible([Option, {versions, APIVsns}]) end end. +warn_override(new, UserOpts, NewOpt, OldOpts, LogLevel) -> + Check = fun(Key) -> maps:is_key(Key,UserOpts) end, + case lists:filter(Check, OldOpts) of + [] -> ok; + Ignored -> + Desc = lists:flatten(io_lib:format("Options ~w are ignored", [Ignored])), + Reas = lists:flatten(io_lib:format("Option ~w is set", [NewOpt])), + ssl_logger:log(notice, LogLevel, #{description => Desc, reason => Reas}, ?LOCATION) + end; +warn_override(_, _UserOpts, _NewOpt, _OldOpts, _LogLevel) -> + ok. + is_dtls_configured(Versions) -> Fun = fun (Version) when Version =:= {254, 253} orelse Version =:= {254, 255} -> @@ -2063,500 +2432,33 @@ is_dtls_configured(Versions) -> end, lists:any(Fun, Versions). -validate_option(Option, Value) -> - validate_option(Option, Value, undefined). -%% -validate_option(Opt, Value, _) - when Opt =:= alpn_advertised_protocols orelse - Opt =:= alpn_preferred_protocols, - is_list(Value) -> - validate_binary_list(Opt, Value), - Value; -validate_option(Opt, Value, _) - when Opt =:= alpn_advertised_protocols orelse - Opt =:= alpn_preferred_protocols, - Value =:= undefined -> - undefined; -validate_option(anti_replay, '10k', _) -> - %% n = 10000 - %% p = 0.030003564 (1 in 33) - %% m = 72985 (8.91KiB) - %% k = 5 - {10, 5, 72985}; -validate_option(anti_replay, '100k', _) -> - %% n = 100000 - %% p = 0.03000428 (1 in 33) - %% m = 729845 (89.09KiB) - %% k = 5 - {10, 5, 729845}; -validate_option(anti_replay, Value, _) - when (is_tuple(Value) andalso - tuple_size(Value) =:= 3) -> - Value; -validate_option(beast_mitigation, Value, _) - when Value == one_n_minus_one orelse - Value == zero_n orelse - Value == disabled -> - Value; -%% certfile must be present in some cases otherwise it can be set -%% to the empty string. -validate_option(cacertfile, undefined, _) -> - <<>>; -validate_option(cacertfile, Value, _) - when is_binary(Value) -> - unambiguous_path(Value); -validate_option(cacertfile, Value, _) - when is_list(Value), Value =/= ""-> - binary_filename(unambiguous_path(Value)); -validate_option(cacerts, Value, _) - when Value == undefined; - is_list(Value) -> - Value; -validate_option(cb_info, {V1, V2, V3, V4} = Value, _) - when is_atom(V1), - is_atom(V2), - is_atom(V3), - is_atom(V4) -> - Value; -validate_option(cb_info, {V1, V2, V3, V4, V5} = Value, _) - when is_atom(V1), - is_atom(V2), - is_atom(V3), - is_atom(V4), - is_atom(V5) -> - Value; -validate_option(cert, Value, _) when Value == undefined; - is_list(Value)-> - Value; -validate_option(cert, Value, _) when Value == undefined; - is_binary(Value)-> - [Value]; -validate_option(certificate_authorities, Value, _) when is_boolean(Value)-> - Value; -validate_option(certfile, undefined = Value, _) -> - Value; -validate_option(certfile, Value, _) - when is_binary(Value) -> - Value; -validate_option(certfile, Value, _) - when is_list(Value) -> - binary_filename(Value); -validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols}, _) - when is_list(PreferredProtocols) -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), - {Precedence, PreferredProtocols, ?NO_PROTOCOL}; -validate_option(client_preferred_next_protocols, - {Precedence, PreferredProtocols, Default} = Value, _) - when is_list(PreferredProtocols), is_binary(Default), - byte_size(Default) > 0, byte_size(Default) < 256 -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), - Value; -validate_option(client_preferred_next_protocols, undefined, _) -> - undefined; -validate_option(client_renegotiation, Value, _) - when is_boolean(Value) -> - Value; -validate_option(cookie, Value, _) - when is_boolean(Value) -> - Value; -validate_option(crl_cache, {Cb, {_Handle, Options}} = Value, _) - when is_atom(Cb) and is_list(Options) -> - Value; -validate_option(crl_check, Value, _) - when is_boolean(Value) -> - Value; -validate_option(crl_check, Value, _) - when (Value == best_effort) or - (Value == peer) -> - Value; -validate_option(customize_hostname_check, Value, _) - when is_list(Value) -> - Value; -validate_option(depth, Value, _) - when is_integer(Value), - Value >= 0, Value =< 255-> - Value; -validate_option(dh, Value, _) - when Value == undefined; - is_binary(Value) -> - Value; -validate_option(dhfile, undefined = Value, _) -> - Value; -validate_option(dhfile, Value, _) - when is_binary(Value) -> - Value; -validate_option(dhfile, Value, _) - when is_list(Value), Value =/= "" -> - binary_filename(Value); -validate_option(early_data, Value, server) - when Value =:= disabled orelse - Value =:= enabled -> - Value; -validate_option(early_data = Option, Value, server) -> - throw({error, - {options, role, {Option, {Value, {server, [disabled, enabled]}}}}}); -validate_option(early_data, Value, client) - when is_binary(Value) -> - Value; -validate_option(early_data = Option, Value, client) -> - throw({error, - {options, type, {Option, {Value, not_binary}}}}); -validate_option(erl_dist, Value, _) - when is_boolean(Value) -> - Value; -validate_option(fail_if_no_peer_cert, Value, _) - when is_boolean(Value) -> - Value; -validate_option(fallback, Value, _) - when is_boolean(Value) -> - Value; -validate_option(handshake, hello = Value, _) -> - Value; -validate_option(handshake, full = Value, _) -> - Value; -validate_option(hibernate_after, undefined, _) -> %% Backwards compatibility - infinity; -validate_option(hibernate_after, infinity, _) -> - infinity; -validate_option(hibernate_after, Value, _) - when is_integer(Value), Value >= 0 -> - Value; -validate_option(honor_cipher_order, Value, _) - when is_boolean(Value) -> - Value; -validate_option(honor_ecc_order, Value, _) - when is_boolean(Value) -> - Value; -validate_option(keep_secrets, Value, _) when is_boolean(Value) -> - Value; -validate_option(key, undefined, _) -> - undefined; -validate_option(key, {KeyType, Value}, _) - when is_binary(Value), - KeyType == rsa; %% Backwards compatibility - KeyType == dsa; %% Backwards compatibility - KeyType == 'RSAPrivateKey'; - KeyType == 'DSAPrivateKey'; - KeyType == 'ECPrivateKey'; - KeyType == 'PrivateKeyInfo' -> - {KeyType, Value}; -validate_option(key, #{algorithm := _} = Value, _) -> - Value; -validate_option(keyfile, undefined, _) -> - <<>>; -validate_option(keyfile, Value, _) - when is_binary(Value) -> - Value; -validate_option(keyfile, Value, _) - when is_list(Value), Value =/= "" -> - binary_filename(Value); -validate_option(key_update_at, Value, _) - when is_integer(Value) andalso - Value > 0 -> - Value; -validate_option(log_level, Value, _) when - is_atom(Value) andalso - (Value =:= none orelse - Value =:= all orelse - Value =:= emergency orelse - Value =:= alert orelse - Value =:= critical orelse - Value =:= error orelse - Value =:= warning orelse - Value =:= notice orelse - Value =:= info orelse - Value =:= debug) -> - Value; -%% RFC 6066, Section 4 -validate_option(max_fragment_length, I, _) - when I == ?MAX_FRAGMENT_LENGTH_BYTES_1; - I == ?MAX_FRAGMENT_LENGTH_BYTES_2; - I == ?MAX_FRAGMENT_LENGTH_BYTES_3; - I == ?MAX_FRAGMENT_LENGTH_BYTES_4 -> - I; -validate_option(max_fragment_length, undefined, _) -> - undefined; -validate_option(max_handshake_size, Value, _) - when is_integer(Value) andalso - Value =< ?MAX_UNIT24 -> - Value; -validate_option(middlebox_comp_mode, Value, _) - when is_boolean(Value) -> - Value; -validate_option(next_protocols_advertised, Value, _) when is_list(Value) -> - validate_binary_list(next_protocols_advertised, Value), - Value; -validate_option(next_protocols_advertised, undefined, _) -> - undefined; -validate_option(ocsp_nonce, Value, _) - when Value =:= true orelse - Value =:= false -> - Value; -%% The OCSP responders' certificates can be given as a suggestion and -%% will be used to verify the OCSP response. -validate_option(ocsp_responder_certs, Value, _) - when is_list(Value) -> - [public_key:pkix_decode_cert(CertDer, plain) || CertDer <- Value, - is_binary(CertDer)]; -validate_option(ocsp_stapling, Value, _) - when Value =:= true orelse - Value =:= false -> - Value; -validate_option(padding_check, Value, _) - when is_boolean(Value) -> - Value; -validate_option(partial_chain, Value, _) - when is_function(Value) -> - Value; -validate_option(password, Value, _) - when is_list(Value); is_binary(Value) -> - Value; -validate_option(password, Value, _) - when is_function(Value, 0) -> - Value; -validate_option(certs_keys, Value, _) when is_list(Value) -> - Value; -validate_option(protocol, Value = tls, _) -> - Value; -validate_option(protocol, Value = dtls, _) -> - Value; -validate_option(psk_identity, undefined, _) -> - undefined; -validate_option(psk_identity, Identity, _) - when is_list(Identity), Identity =/= "", length(Identity) =< 65535 -> - binary_filename(Identity); -validate_option(receiver_spawn_opts, Value, _) - when is_list(Value) -> - Value; -validate_option(renegotiate_at, Value, _) when is_integer(Value) -> - erlang:min(Value, ?DEFAULT_RENEGOTIATE_AT); -validate_option(reuse_session, undefined, _) -> - undefined; -validate_option(reuse_session, Value, _) - when is_function(Value) -> - Value; -validate_option(reuse_session, Value, _) - when is_binary(Value) -> - Value; -validate_option(reuse_session, {Id, Data} = Value, _) - when is_binary(Id) andalso - is_binary(Data) -> - Value; -validate_option(reuse_sessions, Value, _) - when is_boolean(Value) -> - Value; -validate_option(reuse_sessions, save = Value, _) -> - Value; -validate_option(secure_renegotiate, Value, _) - when is_boolean(Value) -> - Value; -validate_option(sender_spawn_opts, Value, _) - when is_list(Value) -> - Value; -validate_option(server_name_indication, Value, _) - when is_list(Value) -> - %% RFC 6066, Section 3: Currently, the only server names supported are - %% DNS hostnames - %% case inet_parse:domain(Value) of - %% false -> - %% throw({error, {options, {{Opt, Value}}}}); - %% true -> - %% Value - %% end; - %% - %% But the definition seems very diffuse, so let all strings through - %% and leave it up to public_key to decide... - Value; -validate_option(server_name_indication, undefined, _) -> - undefined; -validate_option(server_name_indication, disable, _) -> - disable; -validate_option(session_tickets, Value, server) - when Value =:= disabled orelse - Value =:= stateful orelse - Value =:= stateless -> - Value; -validate_option(session_tickets, Value, server) -> - throw({error, - {options, role, - {session_tickets, - {Value, {server, [disabled, stateful, stateless]}}}}}); -validate_option(session_tickets, Value, client) - when Value =:= disabled orelse - Value =:= manual orelse - Value =:= auto -> - Value; -validate_option(session_tickets, Value, client) -> - throw({error, - {options, role, - {session_tickets, - {Value, {client, [disabled, manual, auto]}}}}}); -validate_option(sni_fun, undefined, _) -> - undefined; -validate_option(sni_fun, Fun, _) - when is_function(Fun) -> - Fun; -validate_option(sni_hosts, [], _) -> - []; -validate_option(sni_hosts, [{Hostname, SSLOptions} | Tail], _) - when is_list(Hostname) -> - RecursiveSNIOptions = proplists:get_value(sni_hosts, SSLOptions, undefined), - case RecursiveSNIOptions of - undefined -> - [{Hostname, validate_options(SSLOptions)} | - validate_option(sni_hosts, Tail)]; - _ -> - throw({error, {options, {sni_hosts, RecursiveSNIOptions}}}) - end; -validate_option(srp_identity, undefined, _) -> - undefined; -validate_option(srp_identity, {Username, Password}, _) - when is_list(Username), - is_list(Password), Username =/= "", - length(Username) =< 255 -> - {unicode:characters_to_binary(Username), - unicode:characters_to_binary(Password)}; -validate_option(user_lookup_fun, undefined, _) -> - undefined; -validate_option(user_lookup_fun, {Fun, _} = Value, _) - when is_function(Fun, 3) -> - Value; -validate_option(use_ticket, Value, _) - when is_list(Value) -> - Value; -validate_option(verify, Value, _) - when Value == verify_none; Value == verify_peer -> - Value; -validate_option(verify_fun, undefined, _) -> - undefined; -%% Backwards compatibility -validate_option(verify_fun, Fun, _) when is_function(Fun) -> - {fun(_,{bad_cert, _} = Reason, OldFun) -> - case OldFun([Reason]) of - true -> - {valid, OldFun}; - false -> - {fail, Reason} - end; - (_,{extension, _}, UserState) -> - {unknown, UserState}; - (_, valid, UserState) -> - {valid, UserState}; - (_, valid_peer, UserState) -> - {valid, UserState} - end, Fun}; -validate_option(verify_fun, {Fun, _} = Value, _) when is_function(Fun) -> - Value; -validate_option(versions, Versions, _) -> - validate_versions(Versions, Versions); -validate_option(Opt, undefined = Value, _) -> - AllOpts = maps:keys(?RULES), - case lists:member(Opt, AllOpts) of - true -> - Value; - false -> - throw({error, {options, {Opt, Value}}}) - end; -validate_option(Opt, Value, _) -> - throw({error, {options, {Opt, Value}}}). - -handle_cb_info({V1, V2, V3, V4}) -> - {V1,V2,V3,V4, list_to_atom(atom_to_list(V2) ++ "_passive")}; -handle_cb_info(CbInfo) -> - CbInfo. - -handle_hashsigns_option(Value, Version) when is_list(Value) - andalso Version >= {3, 4} -> - case tls_v1:signature_schemes(Version, Value) of - [] -> - throw({error, {options, - no_supported_signature_schemes, - {signature_algs, Value}}}); - _ -> - Value - end; -handle_hashsigns_option(Value, Version) when is_list(Value) - andalso Version =:= {3, 3} -> - case tls_v1:signature_algs(Version, Value) of - [] -> - throw({error, {options, no_supported_algorithms, {signature_algs, Value}}}); - _ -> - Value - end; -handle_hashsigns_option(_, Version) when Version =:= {3, 3} -> - handle_hashsigns_option(tls_v1:default_signature_algs([Version]), Version); -handle_hashsigns_option(_, _Version) -> - undefined. -handle_signature_algorithms_option(Value, Version) when is_list(Value) - andalso Version >= {3, 3} -> - case tls_v1:signature_schemes(Version, Value) of - [] -> - throw({error, {options, - no_supported_signature_schemes, - {signature_algs_cert, Value}}}); - _ -> - Value - end; -handle_signature_algorithms_option(_, _Version) -> - undefined. +handle_hashsigns_option(Value, Version) -> + try + if Version >= {3,4} -> + tls_v1:signature_schemes(Version, Value); + Version =:= {3,3} -> + tls_v1:signature_algs(Version, Value); + true -> + undefined + end + catch error:function_clause -> + option_error(signature_algs, Value) + end. -validate_options([]) -> - []; -validate_options([{Opt, Value} | Tail]) -> - [{Opt, validate_option(Opt, Value)} | validate_options(Tail)]. +handle_signature_algorithms_option(Value, Version) -> + try tls_v1:signature_schemes(Version, Value) + catch error:function_clause -> + option_error(signature_algs_cert, Value) + end. -validate_npn_ordering(client) -> - ok; -validate_npn_ordering(server) -> - ok; -validate_npn_ordering(Value) -> - throw({error, {options, {client_preferred_next_protocols, {invalid_precedence, Value}}}}). - -validate_binary_list(Opt, List) -> - lists:foreach( - fun(Bin) when is_binary(Bin), - byte_size(Bin) > 0, - byte_size(Bin) < 256 -> - ok; - (Bin) -> - throw({error, {options, {Opt, {invalid_protocol, Bin}}}}) - end, List). -validate_versions([], Versions) -> - Versions; -validate_versions([Version | Rest], Versions) when Version == 'tlsv1.3'; - Version == 'tlsv1.2'; - Version == 'tlsv1.1'; - Version == tlsv1 -> - case tls_record:sufficient_crypto_support(Version) of - true -> - tls_validate_versions(Rest, Versions); - false -> - throw({error, {options, {insufficient_crypto_support, {Version, {versions, Versions}}}}}) - end; -validate_versions([Version | Rest], Versions) when Version == 'dtlsv1'; - Version == 'dtlsv1.2'-> - DTLSVer = dtls_record:protocol_version(Version), - case tls_record:sufficient_crypto_support(dtls_v1:corresponding_tls_version(DTLSVer)) of - true -> - dtls_validate_versions(Rest, Versions); - false -> - throw({error, {options, {insufficient_crypto_support, {Version, {versions, Versions}}}}}) - end; -validate_versions([Version| _], Versions) -> - throw({error, {options, {Version, {versions, Versions}}}}). - -tls_validate_versions([], Versions) -> - tls_validate_version_gap(Versions); -tls_validate_versions([Version | Rest], Versions) when Version == 'tlsv1.3'; - Version == 'tlsv1.2'; - Version == 'tlsv1.1'; - Version == tlsv1 -> - tls_validate_versions(Rest, Versions); -tls_validate_versions([Version| _], Versions) -> - throw({error, {options, {Version, {versions, Versions}}}}). +validate_filename(FN, _Option) when is_binary(FN), FN =/= <<>> -> + FN; +validate_filename([_|_] = FN, _Option) -> + Enc = file:native_name_encoding(), + unicode:characters_to_binary(FN, unicode, Enc); +validate_filename(FN, Option) -> + option_error(Option, FN). %% Do not allow configuration of TLS 1.3 with a gap where TLS 1.2 is not supported %% as that configuration can trigger the built in version downgrade protection @@ -2573,25 +2475,7 @@ tls_validate_version_gap(Versions) -> _ -> Versions end. -dtls_validate_versions([], Versions) -> - Versions; -dtls_validate_versions([Version | Rest], Versions) when Version == 'dtlsv1'; - Version == 'dtlsv1.2'-> - dtls_validate_versions(Rest, Versions); -dtls_validate_versions([Ver| _], Versions) -> - throw({error, {options, {Ver, {versions, Versions}}}}). - -%% The option cacerts overrides cacertfile -ca_cert_default(_,_, [_|_]) -> - undefined; -ca_cert_default(verify_none, _, _) -> - undefined; -ca_cert_default(verify_peer, {Fun,_}, _) when is_function(Fun) -> - undefined; -%% Server that wants to verify_peer and has no verify_fun must have -%% some trusted certs. -ca_cert_default(verify_peer, undefined, _) -> - "". + emulated_options(undefined, undefined, Protocol, Opts) -> case Protocol of tls -> @@ -2611,9 +2495,9 @@ handle_cipher_option(Value, Versions) when is_list(Value) -> Suites catch exit:_ -> - throw({error, {options, {ciphers, Value}}}); + option_error(ciphers, Value); error:_-> - throw({error, {options, {ciphers, Value}}}) + option_error(ciphers, Value) end. binary_cipher_suites([{3,4} = Version], []) -> @@ -2686,22 +2570,25 @@ tuple_to_map_mac(chacha20_poly1305, _) -> tuple_to_map_mac(_, MAC) -> MAC. -handle_eccs_option(Value, Version) when is_list(Value) -> - {_Major, Minor} = tls_version(Version), +handle_eccs_option(Value, {_Major, Minor}) when is_list(Value) -> try tls_v1:ecc_curves(Minor, Value) of - Curves -> #elliptic_curves{elliptic_curve_list = Curves} + Curves -> + option_error(Curves =:= [], eccs, none_valid), + #elliptic_curves{elliptic_curve_list = Curves} catch - exit:_ -> throw({error, {options, {eccs, Value}}}); - error:_ -> throw({error, {options, {eccs, Value}}}) + exit:_ -> option_error(eccs, Value); + error:_ -> option_error(eccs, Value) end. handle_supported_groups_option(Value, Version) when is_list(Value) -> {_Major, Minor} = tls_version(Version), try tls_v1:groups(Minor, Value) of - Groups -> #supported_groups{supported_groups = Groups} + Groups -> + option_error(Groups =:= [], supported_groups, none_valid), + #supported_groups{supported_groups = Groups} catch - exit:_ -> throw({error, {options, {supported_groups, Value}}}); - error:_ -> throw({error, {options, {supported_groups, Value}}}) + exit:_ -> option_error(supported_groups, Value); + error:_ -> option_error(supported_groups, Value) end. @@ -2727,42 +2614,37 @@ file_desc(keyfile) -> file_desc(dhfile) -> "Invalid DH params file ". -detect(_Pred, []) -> - undefined; -detect(Pred, [H|T]) -> - case Pred(H) of - true -> - H; - _ -> - detect(Pred, T) - end. - make_next_protocol_selector(undefined) -> undefined; -make_next_protocol_selector({client, AllProtocols, DefaultProtocol}) -> - fun(AdvertisedProtocols) -> - case detect(fun(PreferredProtocol) -> - lists:member(PreferredProtocol, AdvertisedProtocols) - end, AllProtocols) of - undefined -> - DefaultProtocol; - PreferredProtocol -> - PreferredProtocol - end +make_next_protocol_selector({Precedence, PrefProtcol} = V) -> + option_error(not is_list(PrefProtcol), client_preferred_next_protocols, V), + make_next_protocol_selector({Precedence, PrefProtcol, ?NO_PROTOCOL}); +make_next_protocol_selector({Precedence, AllProtocols, DefP} = V) -> + option_error(not is_list(AllProtocols), client_preferred_next_protocols, V), + option_error(not (is_binary(DefP) andalso byte_size(DefP) < 256), client_preferred_next_protocols, V), + validate_protocols(true, client_preferred_next_protocols, AllProtocols), + case Precedence of + client -> + fun(Advertised) -> + Search = fun(P) -> lists:member(P, Advertised) end, + case lists:search(Search, AllProtocols) of + false -> DefP; + {value, Preferred} -> Preferred + end + end; + server -> + fun(Advertised) -> + Search = fun(P) -> lists:member(P, AllProtocols) end, + case lists:search(Search, Advertised) of + false -> DefP; + {value, Preferred} -> Preferred + end + end; + Value -> + option_error(client_preferred_next_protocols, {invalid_precedence, Value}) end; - -make_next_protocol_selector({server, AllProtocols, DefaultProtocol}) -> - fun(AdvertisedProtocols) -> - case detect(fun(PreferredProtocol) -> - lists:member(PreferredProtocol, AllProtocols) - end, - AdvertisedProtocols) of - undefined -> - DefaultProtocol; - PreferredProtocol -> - PreferredProtocol - end - end. +make_next_protocol_selector(What) -> + option_error(client_preferred_next_protocols, What). connection_cb(tls) -> tls_gen_connection; @@ -2771,16 +2653,6 @@ connection_cb(dtls) -> connection_cb(Opts) -> connection_cb(proplists:get_value(protocol, Opts, tls)). -record_cb(tls) -> - tls_record; -record_cb(dtls) -> - dtls_record; -record_cb(Opts) -> - record_cb(proplists:get_value(protocol, Opts, tls)). - -binary_filename(FileName) -> - Enc = file:native_name_encoding(), - unicode:characters_to_binary(FileName, unicode, Enc). %% Assert that basic options are on the format {Key, Value} %% with a few exceptions and phase out log_alert @@ -2788,12 +2660,12 @@ handle_option_format([], Acc) -> lists:reverse(Acc); handle_option_format([{log_alert, Bool} | Rest], Acc) when is_boolean(Bool) -> case proplists:get_value(log_level, Acc ++ Rest, undefined) of - undefined -> + undefined -> handle_option_format(Rest, [{log_level, map_log_level(Bool)} | Acc]); - _ -> + _ -> handle_option_format(Rest, Acc) - end; + end; handle_option_format([{Key,_} = Opt | Rest], Acc) when is_atom(Key) -> handle_option_format(Rest, [Opt | Acc]); %% Handle exceptions @@ -2804,52 +2676,13 @@ handle_option_format([inet = Opt | Rest], Acc) -> handle_option_format([inet6 = Opt | Rest], Acc) -> handle_option_format(Rest, [Opt | Acc]); handle_option_format([Value | _], _) -> - throw({option_not_a_key_value_tuple, Value}). + option_error(option_not_a_key_value_tuple, Value). map_log_level(true) -> notice; map_log_level(false) -> none. -handle_verify_option(verify_none, #{fail_if_no_peer_cert := false} = OptionsMap) -> - OptionsMap#{verify => verify_none}; -handle_verify_option(verify_none, #{fail_if_no_peer_cert := true}) -> - throw({error, {options, incompatible, - {verify, verify_none}, - {fail_if_no_peer_cert, true}}}); -%% The option 'verify' is simulated by the configured 'verify_fun' that is mostly -%% hidden from the end user. When 'verify' is set to verify_none, the option -%% 'verify_fun' is also set to a default verify-none-verify_fun when processing -%% the configuration. If 'verify' is later changed from verify_none to verify_peer, -%% the 'verify_fun' must also be changed to undefined. When 'verify_fun' is set to -%% undefined, public_key's default verify_fun will be used that performs a full -%% verification. -handle_verify_option(verify_peer, #{verify := verify_none} = OptionsMap) -> - OptionsMap#{verify => verify_peer, - verify_fun => undefined}; -handle_verify_option(verify_peer, OptionsMap) -> - OptionsMap#{verify => verify_peer}; -handle_verify_option(Value, _) -> - throw({error, {options, {verify, Value}}}). - -%% Added to handle default values for signature_algs in TLS 1.3 -default_option_role_sign_algs(_, Value, _, Version) when Version >= {3,4} -> - Value; -default_option_role_sign_algs(Role, Value, Role, _) -> - Value; -default_option_role_sign_algs(_, _, _, _) -> - undefined. - -default_option_role(Role, Value, Role) -> - Value; -default_option_role(_,_,_) -> - undefined. - - -default_cb_info(tls) -> - {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; -default_cb_info(dtls) -> - {gen_udp, udp, udp_closed, udp_error, udp_passive}. include_security_info([]) -> false; @@ -2861,11 +2694,6 @@ include_security_info([Item | Items]) -> include_security_info(Items) end. -server_name_indication_default(Host) when is_list(Host) -> - %% SNI should not contain a trailing dot that a hostname may - string:strip(Host, right, $.); -server_name_indication_default(_) -> - undefined. add_filter(undefined, Filters) -> Filters; @@ -2874,25 +2702,27 @@ add_filter(Filter, Filters) -> maybe_client_warn_no_verify(#{verify := verify_none, warn_verify_none := true, - log_level := LogLevel}, client) -> + log_level := LogLevel} = Opts, client) -> ssl_logger:log(warning, LogLevel, #{description => "Server authenticity is not verified since certificate path validation is not enabled", reason => "The option {verify, verify_peer} and one of the options 'cacertfile' or " - "'cacerts' are required to enable this."}, ?LOCATION); -maybe_client_warn_no_verify(_,_) -> + "'cacerts' are required to enable this."}, ?LOCATION), + maps:without([warn_verify_none], Opts); +maybe_client_warn_no_verify(Opts,_) -> %% Warning not needed. Note client certificate validation is optional in TLS - ok. + Opts. unambiguous_path(Value) -> AbsName = filename:absname(Value), - case file:read_link(AbsName) of - {ok, PathWithNoLink} -> - case filename:pathtype(PathWithNoLink) of - relative -> - Dirname = filename:dirname(AbsName), - filename:join([Dirname, PathWithNoLink]); - _ -> - PathWithNoLink - end; - _ -> - AbsName - end. + UP = case file:read_link(AbsName) of + {ok, PathWithNoLink} -> + case filename:pathtype(PathWithNoLink) of + relative -> + Dirname = filename:dirname(AbsName), + filename:join([Dirname, PathWithNoLink]); + _ -> + PathWithNoLink + end; + _ -> + AbsName + end, + validate_filename(UP, cacertfile). diff --git a/lib/ssl/src/ssl_gen_statem.erl b/lib/ssl/src/ssl_gen_statem.erl index 2f59b150ca..e470103074 100644 --- a/lib/ssl/src/ssl_gen_statem.erl +++ b/lib/ssl/src/ssl_gen_statem.erl @@ -62,7 +62,8 @@ set_opts/2, peer_certificate/1, negotiated_protocol/1, - connection_information/2 + connection_information/2, + ktls_handover/1 ]). %% Erlang Distribution export @@ -422,6 +423,14 @@ peer_certificate(ConnectionPid) -> negotiated_protocol(ConnectionPid) -> call(ConnectionPid, negotiated_protocol). +%%-------------------------------------------------------------------- +-spec ktls_handover(pid()) -> {ok, map()} | {error, reason()}. +%% +%% Description: Returns the negotiated protocol +%%-------------------------------------------------------------------- +ktls_handover(ConnectionPid) -> + call(ConnectionPid, ktls_handover). + dist_handshake_complete(ConnectionPid, DHandle) -> gen_statem:cast(ConnectionPid, {dist_handshake_complete, DHandle}). @@ -546,7 +555,7 @@ initial_hello({call, From}, {start, {Opts, EmOpts}, Timeout}, ssl_options = OrigSSLOptions, socket_options = SockOpts} = State0) -> try - SslOpts = ssl:handle_options(Opts, Role, OrigSSLOptions), + SslOpts = ssl:update_options(Opts, Role, OrigSSLOptions), State = ssl_config(SslOpts, Role, State0), initial_hello({call, From}, {start, Timeout}, State#state{ssl_options = SslOpts, @@ -648,6 +657,45 @@ connection({call, From}, {error, timeout} -> {stop_and_reply, {shutdown, downgrade_fail}, [{reply, From, {error, timeout}}]} end; +connection({call, From}, ktls_handover, #state{ + static_env = #static_env{ + transport_cb = Transport, + socket = Socket + }, + connection_env = #connection_env{ + user_application = {_Mon, Pid}, + negotiated_version = TlsVersion + }, + ssl_options = #{ktls := true}, + socket_options = SocketOpts, + connection_states = #{ + current_write := #{ + security_parameters := #security_parameters{cipher_suite = CipherSuite}, + cipher_state := WriteState, + sequence_number := WriteSeq + }, + current_read := #{ + cipher_state := ReadState, + sequence_number := ReadSeq + } + } +}) -> + Reply = case Transport:controlling_process(Socket, Pid) of + ok -> + {ok, #{ + socket => Socket, + tls_version => TlsVersion, + cipher_suite => CipherSuite, + socket_options => SocketOpts, + write_state => WriteState, + write_seq => WriteSeq, + read_state => ReadState, + read_seq => ReadSeq + }}; + {error, Reason} -> + {error, Reason} + end, + {stop_and_reply, {shutdown, ktls}, [{reply, From, Reply}]}; connection({call, From}, Msg, State) -> handle_call(Msg, From, ?FUNCTION_NAME, State); connection(cast, {dist_handshake_complete, DHandle}, @@ -1145,6 +1193,9 @@ maybe_invalidate_session({false, first}, server = Role, Host, Port, Session) -> maybe_invalidate_session(_, _, _, _, _) -> ok. +terminate({shutdown, ktls}, connection, State) -> + %% Socket shall not be closed as it should be returned to user + handle_trusted_certs_db(State); terminate({shutdown, downgrade}, downgrade, State) -> %% Socket shall not be closed as it should be returned to user handle_trusted_certs_db(State); @@ -1341,7 +1392,7 @@ update_ssl_options_from_sni(#{sni_fun := SNIFun, _ -> VersionsOpt = proplists:get_value(versions, SSLOptions, []), FallBackOptions = filter_for_versions(VersionsOpt, OrigSSLOptions), - ssl:handle_options(SSLOptions, server, FallBackOptions) + ssl:update_options(SSLOptions, server, FallBackOptions) end. filter_for_versions([], OrigSSLOptions) -> diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 6b1bd360f8..828847a495 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -117,109 +117,6 @@ -define(DEFAULT_MAX_EARLY_DATA_SIZE, 16384). -%% This map stores all supported options with default values and -%% list of dependencies: -%% #{<option> => {<default_value>, [<option>]}, -%% ...} --define(RULES, - #{ - alpn_advertised_protocols => {undefined, [versions]}, - alpn_preferred_protocols => {undefined, [versions]}, - anti_replay => {undefined, [versions, session_tickets]}, - beast_mitigation => {one_n_minus_one, [versions]}, - cacertfile => {undefined, [versions, - verify_fun, - cacerts]}, - cacerts => {undefined, [versions]}, - cert => {undefined, [versions]}, - certs_keys => {undefined, [versions]}, - certfile => {<<>>, [versions]}, - certificate_authorities => {false, [versions]}, - ciphers => {[], [versions]}, - client_renegotiation => {undefined, [versions]}, - cookie => {true, [versions]}, - crl_cache => {{ssl_crl_cache, {internal, []}}, [versions]}, - crl_check => {false, [versions]}, - customize_hostname_check => {[], [versions]}, - depth => {10, [versions]}, - dh => {undefined, [versions]}, - dhfile => {undefined, [versions]}, - early_data => {undefined, [versions, - session_tickets, - use_ticket]}, - eccs => {undefined, [versions]}, - erl_dist => {false, [versions]}, - fail_if_no_peer_cert => {false, [versions]}, - fallback => {false, [versions]}, - handshake => {full, [versions]}, - hibernate_after => {infinity, [versions]}, - honor_cipher_order => {false, [versions]}, - honor_ecc_order => {undefined, [versions]}, - keep_secrets => {false, [versions]}, - key => {undefined, [versions]}, - keyfile => {undefined, [versions, - certfile]}, - key_update_at => {?KEY_USAGE_LIMIT_AES_GCM, [versions]}, - log_level => {notice, [versions]}, - max_handshake_size => {?DEFAULT_MAX_HANDSHAKE_SIZE, [versions]}, - middlebox_comp_mode => {true, [versions]}, - max_fragment_length => {undefined, [versions]}, - next_protocol_selector => {undefined, [versions]}, - next_protocols_advertised => {undefined, [versions]}, - %% If enable OCSP stapling - ocsp_stapling => {false, [versions]}, - %% Optional arg, if give suggestion of OCSP responders - ocsp_responder_certs => {[], [versions, - ocsp_stapling]}, - %% Optional arg, if add nonce extension in request - ocsp_nonce => {true, [versions, - ocsp_stapling]}, - padding_check => {true, [versions]}, - partial_chain => {fun(_) -> unknown_ca end, [versions]}, - password => {"", [versions]}, - protocol => {tls, []}, - psk_identity => {undefined, [versions]}, - receiver_spawn_opts => {[], [versions]}, - renegotiate_at => {?DEFAULT_RENEGOTIATE_AT, [versions]}, - reuse_session => {undefined, [versions]}, - reuse_sessions => {true, [versions]}, - secure_renegotiate => {true, [versions]}, - sender_spawn_opts => {[], [versions]}, - server_name_indication => {undefined, [versions]}, - session_tickets => {disabled, [versions]}, - signature_algs => {undefined, [versions]}, - signature_algs_cert => {undefined, [versions]}, - sni_fun => {undefined, [versions, - sni_hosts]}, - sni_hosts => {[], [versions]}, - srp_identity => {undefined, [versions]}, - supported_groups => {undefined, [versions]}, - use_ticket => {undefined, [versions]}, - user_lookup_fun => {undefined, [versions]}, - verify => {verify_none, [versions, - fail_if_no_peer_cert, - partial_chain]}, - verify_fun => - { - {fun(_, {bad_cert, _}, UserState) -> - {valid, UserState}; - (_, {extension, #'Extension'{critical = true}}, UserState) -> - %% This extension is marked as critical, so - %% certificate verification should fail if we don't - %% understand the extension. However, this is - %% `verify_none', so let's accept it anyway. - {valid, UserState}; - (_, {extension, _}, UserState) -> - {unknown, UserState}; - (_, valid, UserState) -> - {valid, UserState}; - (_, valid_peer, UserState) -> - {valid, UserState} - end, []}, - [versions, verify]}, - versions => {[], [protocol]} - }). - -define('TLS-1_3_ONLY_OPTIONS', [anti_replay, certificate_authorities, cookie, diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 5c5b22d909..629650e802 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -474,10 +474,8 @@ code_change(_OldVsn, StateName, State, _) -> %%-------------------------------------------------------------------- initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trackers}, User, {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> - #{erl_dist := IsErlDist, - %% Use highest supported version for client/server random nonce generation - versions := [Version|_], - client_renegotiation := ClientRenegotiation} = SSLOptions, + %% Use highest supported version for client/server random nonce generation + #{erl_dist := IsErlDist, versions := [Version|_]} = SSLOptions, BeastMitigation = maps:get(beast_mitigation, SSLOptions, disabled), ConnectionStates = tls_record:init_connection_states(Role, Version, @@ -503,7 +501,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac handshake_env = #handshake_env{ tls_handshake_history = ssl_handshake:init_handshake_history(), renegotiation = {false, first}, - allow_renegotiate = ClientRenegotiation + allow_renegotiate = maps:get(client_renegotiation, SSLOptions, undefined) }, connection_env = #connection_env{user_application = {UserMonitor, User}}, socket_options = SocketOptions, diff --git a/lib/ssl/src/tls_connection_1_3.erl b/lib/ssl/src/tls_connection_1_3.erl index b1fe0c69f5..d0a86c08b5 100644 --- a/lib/ssl/src/tls_connection_1_3.erl +++ b/lib/ssl/src/tls_connection_1_3.erl @@ -247,7 +247,7 @@ user_hello({call, From}, {handshake_continue, NewOptions, Timeout}, #state{static_env = #static_env{role = client = Role}, handshake_env = HSEnv, ssl_options = Options0} = State0) -> - Options = ssl:handle_options(NewOptions, Role, Options0), + Options = ssl:update_options(NewOptions, Role, Options0), State = ssl_gen_statem:ssl_config(Options, Role, State0), {next_state, wait_sh, State#state{start_or_recv_from = From, handshake_env = HSEnv#handshake_env{continue_status = continue}}, @@ -256,7 +256,7 @@ user_hello({call, From}, {handshake_continue, NewOptions, Timeout}, #state{static_env = #static_env{role = server = Role}, handshake_env = #handshake_env{continue_status = {pause, ClientVersions}} = HSEnv, ssl_options = Options0} = State0) -> - Options = #{versions := Versions} = ssl:handle_options(NewOptions, Role, Options0), + Options = #{versions := Versions} = ssl:update_options(NewOptions, Role, Options0), State = ssl_gen_statem:ssl_config(Options, Role, State0), case ssl_handshake:select_supported_version(ClientVersions, Versions) of {3,4} -> @@ -604,10 +604,8 @@ do_client_start(ServerHello, State0) -> initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trackers}, User, {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> - #{erl_dist := IsErlDist, - %% Use highest supported version for client/server random nonce generation - versions := [Version|_], - client_renegotiation := ClientRenegotiation} = SSLOptions, + %% Use highest supported version for client/server random nonce generation + #{versions := [Version|_]} = SSLOptions, MaxEarlyDataSize = init_max_early_data_size(Role), ConnectionStates = tls_record:init_connection_states(Role, Version, @@ -631,8 +629,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac static_env = InitStatEnv, handshake_env = #handshake_env{ tls_handshake_history = ssl_handshake:init_handshake_history(), - renegotiation = {false, first}, - allow_renegotiate = ClientRenegotiation + renegotiation = {false, first} }, connection_env = #connection_env{user_application = {UserMonitor, User}}, socket_options = SocketOptions, @@ -645,12 +642,15 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac start_or_recv_from = undefined, flight_buffer = [], protocol_specific = #{sender => Sender, - active_n => internal_active_n(IsErlDist), + active_n => internal_active_n(SSLOptions, Socket), active_n_toggle => true } }. -internal_active_n(true) -> +internal_active_n(#{ktls := true}, Socket) -> + inet:setopts(Socket, [{packet, ssl_tls}]), + 1; +internal_active_n(#{erl_dist := true}, _) -> %% Start with a random number between 1 and ?INTERNAL_ACTIVE_N %% In most cases distribution connections are established all at %% the same time, and flow control engages with ?INTERNAL_ACTIVE_N for @@ -659,7 +659,7 @@ internal_active_n(true) -> %% a random number between 1 and ?INTERNAL_ACTIVE_N helps to spread the %% spike. erlang:system_time() rem ?INTERNAL_ACTIVE_N + 1; -internal_active_n(false) -> +internal_active_n(#{erl_dist := false}, _) -> case application:get_env(ssl, internal_active_n) of {ok, N} when is_integer(N) -> N; diff --git a/lib/ssl/src/tls_dtls_connection.erl b/lib/ssl/src/tls_dtls_connection.erl index 1b1c724a1f..d9695e79f4 100644 --- a/lib/ssl/src/tls_dtls_connection.erl +++ b/lib/ssl/src/tls_dtls_connection.erl @@ -171,7 +171,7 @@ user_hello({call, From}, {handshake_continue, NewOptions, Timeout}, #state{static_env = #static_env{role = Role}, handshake_env = HSEnv, ssl_options = Options0} = State0) -> - Options = ssl:handle_options(NewOptions, Role, Options0), + Options = ssl:update_options(NewOptions, Role, Options0), State = ssl_gen_statem:ssl_config(Options, Role, State0), {next_state, hello, State#state{start_or_recv_from = From, handshake_env = HSEnv#handshake_env{continue_status = continue} diff --git a/lib/ssl/src/tls_gen_connection.erl b/lib/ssl/src/tls_gen_connection.erl index d58489bb55..d75ab02551 100644 --- a/lib/ssl/src/tls_gen_connection.erl +++ b/lib/ssl/src/tls_gen_connection.erl @@ -352,6 +352,11 @@ handle_info({CloseTag, Socket}, StateName, %% is called after all data has been deliver. {next_state, StateName, State#state{protocol_specific = PS#{active_n_toggle => true}}, []} end; +handle_info({ssl_tls, Port, Type, {Major, Minor}, Data}, StateName, + #state{static_env = #static_env{data_tag = Protocol}, + ssl_options = #{ktls := true}} = State0) -> + Len = size(Data), + handle_info({Protocol, Port, <<Type, Major, Minor, Len:16, Data/binary>>}, StateName, State0); handle_info(Msg, StateName, State) -> ssl_gen_statem:handle_info(Msg, StateName, State). @@ -669,6 +674,8 @@ next_record(_, #state{protocol_buffers = #protocol_buffers{tls_cipher_texts = [] next_record(_, State) -> {no_record, State}. +flow_ctrl(#state{ssl_options = #{ktls := true}} = State) -> + {no_record, State}; %%% bytes_to_read equals the integer Length arg of ssl:recv %%% the actual value is only relevant for packet = raw | 0 %%% bytes_to_read = undefined means no recv call is ongoing diff --git a/lib/ssl/src/tls_server_session_ticket.erl b/lib/ssl/src/tls_server_session_ticket.erl index f6b91404fb..609df77f1f 100644 --- a/lib/ssl/src/tls_server_session_ticket.erl +++ b/lib/ssl/src/tls_server_session_ticket.erl @@ -31,7 +31,7 @@ -include("ssl_cipher.hrl"). %% API --export([start_link/6, +-export([start_link/7, new/3, use/4 ]). @@ -54,12 +54,16 @@ %%%=================================================================== %%% API %%%=================================================================== --spec start_link(term(), atom(), integer(), integer(), integer(), tuple()) -> {ok, Pid :: pid()} | +-spec start_link(term(), Mode, integer(), integer(), integer(), tuple(), Seed) -> + {ok, Pid :: pid()} | {error, Error :: {already_started, pid()}} | {error, Error :: term()} | - ignore. -start_link(Listener, Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay) -> - gen_server:start_link(?MODULE, [Listener, Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay], []). + ignore + when Mode :: stateless | stateful, + Seed :: undefined | binary(). +start_link(Listener, Mode, Lifetime, TicketStoreSize, MaxEarlyDataSize, AntiReplay, Seed) -> + gen_server:start_link(?MODULE, [Listener, Mode, Lifetime, TicketStoreSize, + MaxEarlyDataSize, AntiReplay, Seed], []). new(Pid, Prf, MasterSecret) -> gen_server:call(Pid, {new_session_ticket, Prf, MasterSecret}, infinity). @@ -118,10 +122,13 @@ handle_cast(_Request, State) -> {noreply, NewState :: term()}. handle_info(rotate_bloom_filters, #state{stateless = #{bloom_filter := BloomFilter0, + warm_up_windows_remaining := WarmUp0, window := Window} = Stateless} = State) -> BloomFilter = tls_bloom_filter:rotate(BloomFilter0), erlang:send_after(Window * 1000, self(), rotate_bloom_filters), - {noreply, State#state{stateless = Stateless#{bloom_filter => BloomFilter}}}; + WarmUp = max(WarmUp0 - 1, 0), + {noreply, State#state{stateless = Stateless#{bloom_filter => BloomFilter, + warm_up_windows_remaining => WarmUp}}}; handle_info({'DOWN', Monitor, _, _, _}, #state{listen_monitor = Monitor} = State) -> {stop, normal, State}; handle_info(_Info, State) -> @@ -148,20 +155,19 @@ format_status(_Opt, Status) -> %%% Internal functions %%%=================================================================== -inital_state([stateless, Lifetime, _, MaxEarlyDataSize, undefined]) -> +inital_state([stateless, Lifetime, _, MaxEarlyDataSize, undefined, Seed]) -> #state{nonce = 0, - stateless = #{seed => {crypto:strong_rand_bytes(16), - crypto:strong_rand_bytes(32)}, + stateless = #{seed => stateless_seed(Seed), window => undefined}, lifetime = Lifetime, max_early_data_size = MaxEarlyDataSize }; -inital_state([stateless, Lifetime, _, MaxEarlyDataSize, {Window, K, M}]) -> +inital_state([stateless, Lifetime, _, MaxEarlyDataSize, {Window, K, M}, Seed]) -> erlang:send_after(Window * 1000, self(), rotate_bloom_filters), #state{nonce = 0, stateless = #{bloom_filter => tls_bloom_filter:new(K, M), - seed => {crypto:strong_rand_bytes(16), - crypto:strong_rand_bytes(32)}, + warm_up_windows_remaining => warm_up_windows(Seed), + seed => stateless_seed(Seed), window => Window}, lifetime = Lifetime, max_early_data_size = MaxEarlyDataSize @@ -382,19 +388,62 @@ stateless_usable_ticket(#stateless_ticket{hash = Prf, stateless_living_ticket(0, _, _, _, _) -> true; +%% If `anti_replay` is not enabled, then a ticket is considered to be living +%% if it has not exceeded its lifetime. +%% +%% If `anti_replay` is enabled, we must additionally perform a freshness check +%% as is outlined in section 8.3 Freshness Checks - RFC 8446 stateless_living_ticket(ObfAge, TicketAgeAdd, Lifetime, Timestamp, Window) -> - ReportedAge = ObfAge - TicketAgeAdd, + %% RealAge is the server's view of the age of the ticket in seconds. RealAge = erlang:system_time(second) - Timestamp, + + %% ReportedAge is the client's view of the age of the ticket in milliseconds. + ReportedAge = ObfAge - TicketAgeAdd, + + %% DeltaAge is the difference of the client's view of the age of the ticket + %% and the server's view of the age of the ticket in seconds. + DeltaAge = abs(RealAge - (ReportedAge / 1000)), + + %% We ensure that both the client's view of the age of the ticket and the + %% server's view of the age of the ticket do not exceed the lifetime specified. (ReportedAge =< Lifetime * 1000) andalso (RealAge =< Lifetime) - andalso (in_window(RealAge, Window)). + andalso (in_window(DeltaAge, Window)). in_window(_, undefined) -> true; +%% RFC 8446 - section 8.2 Client Hello Recording +%% describes an anti-replay implementation that can use bounded memory +%% by storing a unique value from a ClientHello (in our case the PSK binder) +%% withing a given time window. +%% +%% In order implement this, when a ClientHello is received, the server +%% must ensure that a ClientHello has been sent relatively recently. +%% We do this by ensuring that the client and server view of the age +%% of the ticket is not larger than our recording window. +%% +%% In the case of an attempted replay attack, there are 2 possible +%% outcomes: +%% - A ClientHello is replayed within the recording window +%% * The ticket looks valid, `in_window` returns true +%% so we proceed to check the unique value +%% * The unique value (PSK Binder) is stored in the bloom filter +%% and we reject the ticket. +%% +%% - A ClientHello is replayed outside the recording window +%% * We reject the ticket as `in_window` returns false. in_window(Age, Window) when is_integer(Window) -> Age =< Window. -stateless_anti_replay(Index, PSK, Binder, +stateless_anti_replay(_Index, _PSK, _Binder, + #state{stateless = #{warm_up_windows_remaining := WarmUpRemaining} + } = State) when WarmUpRemaining > 0 -> + %% Reject all tickets during the warm-up period: + %% RFC 8446 8.2 Client Hello Recording + %% "When implementations are freshly started, they SHOULD reject 0-RTT as + %% long as any portion of their recording window overlaps the startup time." + {{ok, undefined}, State}; +stateless_anti_replay(Index, PSK, Binder, #state{stateless = #{bloom_filter := BloomFilter0} = Stateless} = State) -> case tls_bloom_filter:contains(BloomFilter0, Binder) of @@ -408,3 +457,20 @@ stateless_anti_replay(Index, PSK, Binder, end; stateless_anti_replay(Index, PSK, _, State) -> {{ok, {Index, PSK}}, State}. + +-spec stateless_seed(Seed :: undefined | binary()) -> + {IV :: binary(), Shard :: binary()}. +stateless_seed(undefined) -> + {crypto:strong_rand_bytes(16), crypto:strong_rand_bytes(32)}; +stateless_seed(Seed) -> + <<IV:16/binary, Shard:32/binary, _/binary>> = crypto:hash(sha512, Seed), + {IV, Shard}. + +-spec warm_up_windows(Seed :: undefined | binary()) -> 0 | 2. +warm_up_windows(undefined) -> + 0; +warm_up_windows(_) -> + %% When the encryption seed is specified, "warm up" the bloom filter for + %% 2*WindowSize to ensure tickets from a previous instance of the server + %% (before a restart) cannot be reused, if the ticket encryption seed is reused. + 2. diff --git a/lib/ssl/src/tls_socket.erl b/lib/ssl/src/tls_socket.erl index 779043f1de..d9d90ac426 100644 --- a/lib/ssl/src/tls_socket.erl +++ b/lib/ssl/src/tls_socket.erl @@ -261,20 +261,24 @@ session_tickets_tracker(_,_, _, _, #{erl_dist := false, session_tickets_tracker(ListenSocket, Lifetime, TicketStoreSize, MaxEarlyDataSize, #{erl_dist := false, session_tickets := Mode, - anti_replay := AntiReplay}) -> + anti_replay := AntiReplay, + stateless_tickets_seed := Seed}) -> tls_server_session_ticket_sup:start_child([ListenSocket, Mode, Lifetime, - TicketStoreSize, MaxEarlyDataSize, AntiReplay]); + TicketStoreSize, MaxEarlyDataSize, + AntiReplay, Seed]); session_tickets_tracker(ListenSocket, Lifetime, TicketStoreSize, MaxEarlyDataSize, #{erl_dist := true, session_tickets := Mode, - anti_replay := AntiReplay}) -> + anti_replay := AntiReplay, + stateless_tickets_seed := Seed}) -> SupName = tls_server_session_ticket_sup:sup_name(dist), Children = supervisor:count_children(SupName), Workers = proplists:get_value(workers, Children), case Workers of 0 -> tls_server_session_ticket_sup:start_child([ListenSocket, Mode, Lifetime, - TicketStoreSize, MaxEarlyDataSize, AntiReplay]); + TicketStoreSize, MaxEarlyDataSize, + AntiReplay, Seed]); 1 -> [{_,Child,_, _}] = supervisor:which_children(SupName), {ok, Child} diff --git a/lib/ssl/test/inet_crypto_dist.erl b/lib/ssl/test/inet_crypto_dist.erl index 902bb5c252..ed950ddb20 100644 --- a/lib/ssl/test/inet_crypto_dist.erl +++ b/lib/ssl/test/inet_crypto_dist.erl @@ -36,8 +36,6 @@ -export([gen_listen/2, gen_accept/2, gen_accept_connection/6, gen_setup/6, gen_close/2, gen_select/2]). --export([nodelay/0]). - %% Debug %%%-compile(export_all). -export([dbg/0, test_server/0, test_client/1]). @@ -463,7 +461,7 @@ do_accept( {Acceptor, controller, Socket} -> Timer = dist_util:start_timer(SetupTime), HSData = - hs_data_common( + hs_data( NetKernel, MyNode, DistCtrl, Timer, Socket, Driver:family()), HSData_1 = @@ -554,7 +552,7 @@ do_setup_connect( end, %% DistCtrl is a "socket" HSData = - hs_data_common( + hs_data( NetKernel, MyNode, DistCtrl, Timer, Socket, Driver:family()), HSData_1 = @@ -583,7 +581,7 @@ gen_close(Socket, Driver) -> %% ------------------------------------------------------------------------- -hs_data_common(NetKernel, MyNode, DistCtrl, Timer, Socket, Family) -> +hs_data(NetKernel, MyNode, DistCtrl, Timer, Socket, Family) -> %% Field 'socket' below is set to DistCtrl, which makes %% the distribution handshake process (ticker) call %% the funs below with DistCtrl as the S argument. @@ -764,20 +762,6 @@ connect_options(Opts) -> Opts end. -%% we may not always want the nodelay behaviour -%% for performance reasons -nodelay() -> - case application:get_env(kernel, dist_nodelay) of - undefined -> - {nodelay, true}; - {ok, true} -> - {nodelay, true}; - {ok, false} -> - {nodelay, false}; - _ -> - {nodelay, true} - end. - %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% The DistCtrl process(es). @@ -1401,7 +1385,7 @@ input_handler(#params{socket = Socket} = Params, Seq, DistHandle) -> try ok = inet:setopts( - Socket, [{active, ?TCP_ACTIVE}, nodelay()]), + Socket, [{active, ?TCP_ACTIVE}, inet_tcp_dist:nodelay()]), input_handler( Params#params{dist_handle = DistHandle}, Seq) diff --git a/lib/ssl/test/openssl_npn_SUITE.erl b/lib/ssl/test/openssl_npn_SUITE.erl index 485efd7f45..5524a9293d 100644 --- a/lib/ssl/test/openssl_npn_SUITE.erl +++ b/lib/ssl/test/openssl_npn_SUITE.erl @@ -37,6 +37,7 @@ %% Test cases -export([erlang_client_openssl_server_npn/0, erlang_client_openssl_server_npn/1, + erlang_server_openssl_client_npn/0, erlang_server_openssl_client_npn/1, erlang_server_openssl_client_npn_only_client/1, erlang_server_openssl_client_npn_only_server/1, @@ -225,8 +226,8 @@ erlang_server_openssl_client_npn(Config) when is_list(Config) -> %%-------------------------------------------------------------------------- -erlang_server_openssl_client_npn_renegotiate() -> - [{doc,"Test erlang server with openssl client and npn negotiation with renegotiation"}]. +%% erlang_server_openssl_client_npn_renegotiate() -> +%% [{doc,"Test erlang server with openssl client and npn negotiation with renegotiation"}]. erlang_server_openssl_client_npn_renegotiate(Config) when is_list(Config) -> ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), @@ -317,17 +318,18 @@ erlang_server_openssl_client_npn_only_server(Config) when is_list(Config) -> ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), Server = ssl_test_lib:start_server(erlang, [{from, self()}], - [{server_opts, [{client_preferred_next_protocols, - {client, [<<"spdy/2">>], <<"http/1.1">>} - } | ServerOpts]} | Config]), + [{server_opts, + [{next_protocols_advertised, + [<<"spdy/2">>, <<"http/1.1">>]} + | ServerOpts]} | Config]), Port = ssl_test_lib:inet_port(Server), - {_Client, OpenSSLPort} = ssl_test_lib:start_client(openssl, [{port, Port}, - {options, ClientOpts}, + {_Client, OpenSSLPort} = ssl_test_lib:start_client(openssl, [{port, Port}, + {options, ClientOpts}, return_port], Config), - + Server ! get_socket, - SSocket = - receive + SSocket = + receive {Server, {socket, Socket}} -> Socket end, diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 4b43954139..7d4633095d 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -245,8 +245,10 @@ ecc_unknown_curve(Config) -> ecdhe_ecdsa, ecdhe_ecdsa, Config), COpts = ssl_test_lib:ssl_options(COpts0, Config), SOpts = ssl_test_lib:ssl_options(SOpts0, Config), - ECCOpts = [{eccs, ['123_fake_curve']}], - ssl_test_lib:ecc_test_error(COpts, SOpts, [], ECCOpts, Config). + ECCALL = ssl:eccs(), + SECCOpts = [{eccs, [hd(ECCALL)]}], + CECCOpts = [{eccs, tl(ECCALL)}], + ssl_test_lib:ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config). client_ecdh_rsa_server_ecdhe_ecdsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), diff --git a/lib/ssl/test/ssl_api_SUITE.erl b/lib/ssl/test/ssl_api_SUITE.erl index 6058672c4b..cb48a1f8a1 100644 --- a/lib/ssl/test/ssl_api_SUITE.erl +++ b/lib/ssl/test/ssl_api_SUITE.erl @@ -25,6 +25,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("ssl/src/ssl_api.hrl"). +-include_lib("ssl/src/ssl_internal.hrl"). -include_lib("public_key/include/public_key.hrl"). %% Common test @@ -133,6 +134,7 @@ options_not_proplist/1, invalid_options/0, invalid_options/1, + options_whitebox/0, options_whitebox/1, cb_info/0, cb_info/1, log_alert/0, @@ -175,6 +177,8 @@ server_options_negative_version_gap/1, server_options_negative_dependency_role/0, server_options_negative_dependency_role/1, + server_options_negative_stateless_tickets_seed/0, + server_options_negative_stateless_tickets_seed/1, invalid_options_tls13/0, invalid_options_tls13/1, cookie/0, @@ -231,7 +235,7 @@ all() -> {group, 'tlsv1'}, {group, 'dtlsv1.2'}, {group, 'dtlsv1'} - ]. + ] ++ simple_api_tests(). groups() -> [ @@ -247,13 +251,9 @@ groups() -> {'tlsv1.2', [], gen_api_tests() ++ since_1_2() ++ handshake_paus_tests() ++ pre_1_3()}, {'tlsv1.1', [], gen_api_tests() ++ handshake_paus_tests() ++ pre_1_3()}, {'tlsv1', [], gen_api_tests() ++ handshake_paus_tests() ++ pre_1_3() ++ beast_mitigation_test()}, - {'dtlsv1.2', [], (gen_api_tests() -- - [invalid_keyfile, invalid_certfile, invalid_cacertfile, - invalid_options, new_options_in_handshake]) ++ + {'dtlsv1.2', [], gen_api_tests() -- [new_options_in_handshake] ++ handshake_paus_tests() -- [handshake_continue_tls13_client] ++ pre_1_3()}, - {'dtlsv1', [], (gen_api_tests() -- - [invalid_keyfile, invalid_certfile, invalid_cacertfile, - invalid_options, new_options_in_handshake]) ++ + {'dtlsv1', [], gen_api_tests() -- [new_options_in_handshake] ++ handshake_paus_tests() -- [handshake_continue_tls13_client] ++ pre_1_3()} ]. @@ -271,6 +271,17 @@ pre_1_3() -> prf ]. +simple_api_tests() -> + [ + invalid_keyfile, + invalid_certfile, + invalid_cacertfile, + invalid_options, + options_not_proplist, + options_whitebox + ]. + + gen_api_tests() -> [ peercert, @@ -281,6 +292,7 @@ gen_api_tests() -> secret_connection_info, keylog_connection_info, versions, + new_options_in_handshake, active_n, dh_params, hibernate, @@ -305,13 +317,7 @@ gen_api_tests() -> honor_client_cipher_order, ipv6, der_input, - new_options_in_handshake, max_handshake_size, - invalid_certfile, - invalid_cacertfile, - invalid_keyfile, - options_not_proplist, - invalid_options, cb_info, log_alert, getstat, @@ -354,6 +360,7 @@ tls13_group() -> server_options_negative_early_data, server_options_negative_version_gap, server_options_negative_dependency_role, + server_options_negative_stateless_tickets_seed, invalid_options_tls13, cookie ]. @@ -1871,7 +1878,7 @@ der_input(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true}, {dh, DHParams}, {cert, ServerCert}, {key, ServerKey}, {cacerts, ServerCaCerts}], - ClientOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true}, + ClientOpts = [{verify, verify_peer}, {dh, DHParams}, {cert, ClientCert}, {key, ClientKey}, {cacerts, ClientCaCerts}], {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -1897,7 +1904,7 @@ der_input(Config) when is_list(Config) -> {cert, ServerCert}, {key, ServerKey}, {cacerts, [ #cert{der=Der, otp=public_key:pkix_decode_cert(Der, otp)} || Der <- ServerCaCerts]}], - ClientOpts1 = [{verify, verify_peer}, {fail_if_no_peer_cert, true}, + ClientOpts1 = [{verify, verify_peer}, {dh, DHParams}, {cert, ClientCert}, {key, ClientKey}, {cacerts, [ #cert{der=Der, otp=public_key:pkix_decode_cert(Der, otp)} @@ -2083,7 +2090,7 @@ options_not_proplist() -> options_not_proplist(Config) when is_list(Config) -> BadOption = {client_preferred_next_protocols, client, [<<"spdy/3">>,<<"http/1.1">>], <<"http/1.1">>}, - {option_not_a_key_value_tuple, BadOption} = + {error, {options, {option_not_a_key_value_tuple, BadOption}}} = ssl:connect("twitter.com", 443, [binary, {active, false}, BadOption]). @@ -2110,7 +2117,6 @@ invalid_options(Config) when is_list(Config) -> TestOpts = [{versions, [sslv2, sslv3]}, - {verify, 4}, {verify_fun, function}, {fail_if_no_peer_cert, 0}, {depth, four}, @@ -2131,31 +2137,19 @@ invalid_options(Config) when is_list(Config) -> {key, 'key.pem' }], TestOpts2 = - [{[{anti_replay, '10k'}], - %% anti_replay is a server only option but tested with client - %% for simplicity - {options,dependency,{anti_replay,{versions,['tlsv1.3']}}}}, - {[{cookie, false}], - {options,dependency,{cookie,{versions,['tlsv1.3']}}}}, - {[{supported_groups, []}], - {options,dependency,{supported_groups,{versions,['tlsv1.3']}}}}, - {[{use_ticket, [<<1,2,3,4>>]}], - {options,dependency,{use_ticket,{versions,['tlsv1.3']}}}}, - {[{verify, verify_none}, {fail_if_no_peer_cert, true}], - {options, incompatible, - {verify, verify_none}, - {fail_if_no_peer_cert, true}}}], + [{[{supported_groups, []}, {versions, [tlsv1]}], + {options,incompatible,[supported_groups,{versions,['tlsv1']}]}}], [begin Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, - {options, [TestOpt | ServerOpts]}]), + {options, ServerOpts ++ [TestOpt]}]), %% Will never reach a point where port is used. Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, 0}, {host, Hostname}, {from, self()}, - {options, [TestOpt | ClientOpts]}]), + {options, ClientOpts ++ [TestOpt]}]), Check(Client, Server, TestOpt), ok end || TestOpt <- TestOpts], @@ -2166,6 +2160,800 @@ invalid_options(Config) when is_list(Config) -> end || {TestOpt, ErrorMsg} <- TestOpts2], ok. +options_whitebox() -> + [{doc,"Whitebox tests of option handling"}]. + + +patch_version(Opts, Role, Host) -> + case proplists:get_value(protocol, Opts, tls) of + dtls -> + {ok, #config{ssl=DOpts}} = ssl:handle_options([{protocol, dtls}], Role, Host), + {DOpts, Opts}; + tls -> + {ok, #config{ssl=DOpts}} = ssl:handle_options([], Role, Host), + case proplists:get_value(versions, Opts) of + undefined -> + {DOpts, [{versions, ['tlsv1.2','tlsv1.3']}|Opts]}; + _ -> + {DOpts, Opts} + end; + _ -> + {ok, #config{ssl=DOpts}} = ssl:handle_options([], Role, Host), + {DOpts, Opts} + end. + +-define(OK(EXP, Opts, Role), + fun() -> + Host = "dummy.host.org", + {__DefOpts, __Opts} = patch_version(Opts, Role, Host), + try ssl:handle_options(__Opts, Role, Host) of + {ok, #config{ssl=EXP}} -> ok; + Other -> + ct:pal("ssl:handle_options(~0p,~0p,~0p).",[__Opts,Role,Host]), + error({unexpected, Other}) + catch + throw:{error,{options,{insufficient_crypto_support,{'tlsv1.3',_}}}} -> ignored; + C:Other:ST -> + ct:pal("ssl:handle_options(~0p,~0p,~0p).",[__Opts,Role,Host]), + error({unexpected, C, Other,ST}) + end, + try ssl:update_options(__Opts, Role, __DefOpts) of + EXP -> ok; + Other2 -> + ct:pal("{ok,Cfg} = ssl:handle_options([],~p,~p)," + "ssl:update_options(~p,~p, element(2,Cfg)).", + [Role,Host,__Opts,Role]), + error({unexpected2, Other2}) + catch + throw:{error,{options,{insufficient_crypto_support,{'tlsv1.3',_}}}} -> ignored; + C2:Other2:ST2 -> + ct:pal("{ok,Cfg} = ssl:handle_options([],~p,~p)," + "ssl:update_options(~p,~p, element(2,Cfg)).", + [Role,Host,__Opts,Role]), + error({unexpected, C2, Other2, ST2}) + end + end()). + +-define(ERR(EXP, Opts, Role), + fun() -> + Host = "dummy.host.org", + {__DefOpts, __Opts} = patch_version(Opts, Role, Host), + try ssl:handle_options(__Opts, Role, Host) of + Other -> + ct:pal("ssl:handle_options(~0p,~0p,~0p).",[__Opts,Role,Host]), + error({unexpected, Other}) + catch + throw:{error,{options,{insufficient_crypto_support,{'tlsv1.3',_}}}} -> ignored; + throw:{error, {options, EXP}} -> ok; + throw:{error, EXP} -> ok; + C:Other:ST -> + ct:pal("ssl:handle_options(~0p,~0p,~0p).",[__Opts,Role,Host]), + error({unexpected, C, Other,ST}) + end, + try ssl:update_options(__Opts, Role, __DefOpts) of + Other2 -> + ct:pal("{ok,Cfg} = ssl:handle_options([],~p,~p)," + "ssl:update_options(~p,~p, element(2,Cfg)).", + [Role,Host,__Opts,Role]), + error({unexpected, Other2}) + catch + throw:{error,{options,{insufficient_crypto_support,{'tlsv1.3',_}}}} -> ignored; + throw:{error, {options, EXP}} -> ok; + throw:{error, EXP} -> ok; + C2:Other2:ST2 -> + ct:pal("{ok,Cfg} = ssl:handle_options([],~p,~p)," + "ssl:update_options(~p,~p, element(2,Cfg)).", + [Role,Host,__Opts,Role]), + error({unexpected, C2, Other2,ST2}) + end + end()). + +options_whitebox(Config) when is_list(Config) -> + Cert = proplists:get_value(cert, ssl_test_lib:ssl_options(server_rsa_der_opts, Config)), + true = is_binary(Cert), + ?OK(#{verify := verify_peer}, %% Check Option order last wins + [{verify, verify_none}, {verify,verify_peer}, {cacerts, [Cert]}], client), + options_protocol(Config), + options_version(Config), + options_alpn(Config), + options_anti_replay(Config), + options_beast_mitigation(Config), + options_cacerts(Config), + options_cert(Config), + options_certificate_authorities(Config), + options_ciphers(Config), + options_client_renegotiation(Config), + options_cookie(Config), + options_crl(Config), + options_hostname_check(Config), + options_dh(Config), + options_early_data(Config), + options_eccs(Config), + options_verify(Config), + options_fallback(Config), + options_handshake(Config), + options_process(Config), + options_honor(Config), + options_debug(Config), + options_renegotiate(Config), + options_middlebox(Config), + options_frag_len(Config), + options_oscp(Config), + options_padding(Config), + options_identity(Config), + options_reuse_session(Config), + options_sni(Config), + options_sign_alg(Config), + options_supported_groups(Config), + ok. + +options_protocol(_Config) -> + ?OK(#{protocol := tls}, [], client), + ?OK(#{protocol := tls}, [{protocol, tls}], client), + ?OK(#{protocol := dtls}, [{protocol, dtls}], client), + + %% Errors + ?ERR({protocol, foo}, [{protocol, 'foo'}], client), + + begin %% erl_dist + ?OK(#{erl_dist := false}, [], client), + ?OK(#{erl_dist := true}, [{erl_dist, true}], client), + ?OK(#{ktls := false}, [], client), + ?OK(#{ktls := true}, [{ktls, true}], client) + end, + ok. + +options_version(_Config) -> + ?OK(#{versions := [_|_]}, [], client), %% Hmm some machines still default only {3,3} + ?OK(#{versions := [{254,253}]}, [{protocol, dtls}], client), + + ?OK(#{versions := [{3,4},{3,3},{3,2},{3,1}]}, + [{versions, ['tlsv1','tlsv1.1','tlsv1.2','tlsv1.3']}], + client), + ?OK(#{versions := [{3,4}]}, [{versions, ['tlsv1.3']}], client), + + ?OK(#{versions := [{254,253},{254,255}]}, + [{protocol, dtls}, {versions, ['dtlsv1', 'dtlsv1.2']}], client), + ?OK(#{versions := [{254,253}]}, [{protocol, dtls}, {versions, ['dtlsv1.2']}], client), + + + %% Errors + ?ERR({versions, []}, [{versions, []}], client), + ?ERR({versions, []}, [{protocol, dtls}, {versions, []}], client), + + ?ERR({'tlsv1.4',{versions, ['tlsv1.4']}}, + [{versions, ['tlsv1.4']}], client), + ?ERR({'dtlsv1', {versions, _}}, + [{versions, ['dtlsv1', 'dtlsv1.2']}], client), + + ?ERR({'tlsv1', {versions, _}}, + [{protocol, dtls}, {versions, ['tlsv1','tlsv1.1','tlsv1.2','tlsv1.3']}], + client), + ?ERR({'dtlsv1.3',{versions, ['dtlsv1.3']}}, + [{protocol, dtls}, {versions, ['dtlsv1.3']}], + client), + ?ERR({options,missing_version,{'tlsv1.2',_}}, + [{versions, ['tlsv1.1','tlsv1.3']}], + client), + ok. + +options_alpn(_Config) -> %% alpn & next_protocols + Http = <<"HTTP/2">>, + ?OK(#{alpn_advertised_protocols := undefined, alpn_preferred_protocols := undefined, + next_protocol_selector := undefined, next_protocols_advertised := undefined}, + [], client), + ?OK(#{alpn_advertised_protocols := undefined, alpn_preferred_protocols := undefined, + next_protocol_selector := undefined, next_protocols_advertised := undefined}, + [], server), + + ?OK(#{alpn_advertised_protocols := undefined, alpn_preferred_protocols := [Http], + next_protocol_selector := undefined, next_protocols_advertised := undefined}, + [{alpn_preferred_protocols, [Http]}], server), + ?OK(#{alpn_advertised_protocols := [Http], alpn_preferred_protocols := undefined, + next_protocol_selector := undefined, next_protocols_advertised := undefined}, + [{alpn_advertised_protocols, [Http]}], client), + + %% Note names have been swapped in client/server variants + + ?OK(#{alpn_advertised_protocols := undefined, alpn_preferred_protocols := undefined, + next_protocol_selector := undefined, next_protocols_advertised := [Http]}, + [{next_protocols_advertised, [Http]}], server), + ?OK(#{alpn_advertised_protocols := undefined, alpn_preferred_protocols := undefined, + next_protocol_selector := _, next_protocols_advertised := undefined}, + [{client_preferred_next_protocols, {server,[Http], Http}}], + client), + + %% Errors + ?ERR({alpn_preferred_protocols, {invalid_protocol, <<>>}}, [{alpn_preferred_protocols, [Http, <<>>]}], server), + ?ERR({alpn_advertised_protocols, Http}, [{alpn_advertised_protocols, Http}], client), + ?ERR({alpn_preferred_protocols, undefined}, [{alpn_preferred_protocols, undefined}], server), + ?ERR({alpn_advertised_protocols, undefined}, [{alpn_advertised_protocols, undefined}], client), + ?ERR({option, server_only, alpn_preferred_protocols}, [{alpn_preferred_protocols, [Http]}], client), + ?ERR({option, client_only, alpn_advertised_protocols}, [{alpn_advertised_protocols, [Http]}], server), + ?ERR({option, server_only, next_protocols_advertised}, [{next_protocols_advertised, [Http]}], client), + ?ERR({option, client_only, client_preferred_next_protocols}, [{client_preferred_next_protocols, [Http]}], server), + ok. + +options_anti_replay(_Config) -> + ?OK(#{anti_replay := undefined}, [], server), + ?OK(#{anti_replay := {_,_,_}}, + [{anti_replay, '10k'}, {session_tickets, stateless}], + server), + ?OK(#{anti_replay := {_,_,_}}, + [{anti_replay, {42,4711,21}}, {session_tickets, stateless}], + server), + + + %% Errors + ?ERR({option, server_only, anti_replay}, + [{anti_replay, '10k'}, {session_tickets, manual}], + client), + ?ERR({options, incompatible, [{anti_replay, _}, {session_tickets, disabled}]}, + [{anti_replay, '10k'}], + server), + ?ERR({options,incompatible, [session_tickets,{versions,['tlsv1']}]}, + [{anti_replay, '10k'}, {session_tickets, stateless}, {versions, ['tlsv1']}], + server), + ?ERR({anti_replay, '1k'}, + [{anti_replay, '1k'}, {session_tickets, stateless}], + server), + ?ERR({anti_replay, _}, + [{anti_replay, {1,1,1,1}}, {session_tickets, stateless}], + server), + ok. + +options_beast_mitigation(_Config) -> %% Beast mitigation + ?OK(#{beast_mitigation := one_n_minus_one}, [], client), + ?OK(#{beast_mitigation := disabled}, + [{beast_mitigation, disabled}, {versions, [tlsv1]}], client), + ?OK(#{beast_mitigation := zero_n}, + [{beast_mitigation, zero_n}, {versions, [tlsv1]}], client), + + %% Errors + ?ERR({beast_mitigation, enabled}, + [{beast_mitigation, enabled}, {versions, [tlsv1]}], client), + ?ERR({options, incompatible, [beast_mitigation, {versions, _}]}, %% ok? + [{beast_mitigation, disabled}], client), + ok. + +options_cacerts(Config) -> %% cacert[s]file + Cert = proplists:get_value(cert, ssl_test_lib:ssl_options(server_rsa_der_opts, Config)), + File = case os:type() of + {win32, _} -> <<"c:/tmp/foo">>; + _ -> <<"/tmp/foo">> + end, + ?OK(#{cacerts := undefined, cacertfile := <<>>}, + [], client), + ?OK(#{cacerts := undefined, cacertfile := File}, + [{cacertfile, File}], client), + ?OK(#{cacerts := [Cert], cacertfile := <<>>}, + [{cacerts, [Cert]}, {verify, verify_peer}], client), + ?OK(#{cacerts := [#cert{}], cacertfile := <<>>}, + [{cacerts, [#cert{der=Cert, otp=dummy}]}], client), + ?OK(#{cacerts := [Cert], cacertfile := _}, + [{cacerts, [Cert]}, {cacertfile, "/tmp/foo"}], client), + + %% Errors + ?ERR({options, incompatible, _}, [{verify, verify_peer}], server), + ?ERR({cacerts, Cert}, [{cacerts, Cert}], client), + ?ERR({cacertfile, cert}, [{cacertfile, cert}], client), + + begin %% depth + ?OK(#{depth := 10}, [], client), + ?OK(#{depth := 5}, [{depth, 5}], client), + %% Error + ?ERR({depth, not_an_int}, [{depth, not_an_int}], client) + end, + ok. + +options_cert(Config) -> %% cert[file] cert_keys keys password + Cert = proplists:get_value(cert, ssl_test_lib:ssl_options(server_rsa_der_opts, Config)), + {ok, #config{ssl = DefMap}} = ssl:handle_options([], client, "dummy.host.org"), + false = maps:is_key(certs_keys, DefMap), %% ?? + + ?OK(#{cert := undefined, certfile := <<>>, key := undefined, keyfile := <<>>, password := ""}, + [], client), + ?OK(#{cert := [Cert], certfile := <<>>, key := undefined, keyfile := <<>>, password := ""}, + [{cert,Cert}], client), + ?OK(#{cert := [Cert], certfile := <<>>, key := undefined, keyfile := <<>>}, + [{cert,[Cert]}], client), + ?OK(#{cert := undefined, certfile := <<"/tmp/foo">>, key := undefined, keyfile := <<"/tmp/foo">>}, + [{certfile, <<"/tmp/foo">>}], client), + + ?OK(#{certs_keys := [#{}]}, + [{certs_keys, [#{}]}], client), + + ?OK(#{cert := undefined, certfile := <<>>, key := {rsa, <<>>}, keyfile := <<>>}, + [{key, {rsa, <<>>}}], client), + ?OK(#{cert := undefined, certfile := <<>>, key := #{}, keyfile := <<>>}, + [{key, #{engine => foo, algorithm => foo, key_id => foo}}], client), + + ?OK(#{key := undefined, keyfile := <<>>, password := "foobar"}, + [{password, "foobar"}], client), + ?OK(#{key := undefined, keyfile := <<>>, password := <<"foobar">>}, + [{password, <<"foobar">>}], client), + Pwd = fun() -> "foobar" end, + ?OK(#{key := undefined, keyfile := <<>>, password := Pwd}, + [{password, Pwd}], client), + + ?OK(#{cert := undefined, certfile := <<"/tmp/foo">>, key := undefined, keyfile := <<"/tmp/baz">>}, + [{certfile, <<"/tmp/foo">>}, {keyfile, "/tmp/baz"}], client), + + ?OK(#{certs_keys := [#{}]}, + [{cert, Cert}, {certfile, "/tmp/foo"}, {certs_keys, [#{}]}], + client), + + %% Errors + ?ERR({cert, #{}}, [{cert, #{}}], client), + ?ERR({certfile, cert}, [{certfile, cert}], client), + ?ERR({certs_keys, #{}}, [{certs_keys, #{}}], client), + ?ERR({keyfile, #{}}, [{keyfile, #{}}], client), + ?ERR({key, <<>>}, [{key, <<>>}], client), + ?ERR({password, _}, [{password, fun(Arg) -> Arg end}], client), + ok. + +options_certificate_authorities(_Config) -> + ?OK(#{certificate_authorities := false}, [], client), + ?OK(#{certificate_authorities := true}, [], server), + ?OK(#{certificate_authorities := true}, + [{certificate_authorities, true}], client), + ?OK(#{certificate_authorities := false}, + [{certificate_authorities, false}], server), + + %% Errors + ?ERR({certificate_authorities, []}, + [{certificate_authorities, []}], client), + ?ERR({options, incompatible, [certificate_authorities, {versions, _}]}, + [{certificate_authorities, true}, {versions, ['tlsv1.2']}], + client), + ok. + +options_ciphers(_Config) -> + CipherSuite = ssl_test_lib:ecdh_dh_anonymous_suites({3,3}), + ?OK(#{ciphers := [_|_]}, [], client), + ?OK(#{ciphers := [_|_]}, [{ciphers, CipherSuite}], client), + ?OK(#{ciphers := [_|_]}, [{ciphers, "RC4-SHA:RC4-MD5"}], client), + ?OK(#{ciphers := [_|_]}, [{ciphers, ["RC4-SHA", "RC4-MD5"]}], client), + + %% FIXME extend this + ok. + +options_client_renegotiation(_Config) -> + ?OK(#{client_renegotiation := true}, [], server), + ?OK(#{client_renegotiation := false}, [{client_renegotiation, false}], server), + + %% Errors + ?ERR({client_renegotiation, []}, [{client_renegotiation, []}], server), + ?ERR({option, server_only, client_renegotiation}, [{client_renegotiation, true}], client), + ?ERR({options, incompatible, [client_renegotiation, {versions, _}]}, + [{client_renegotiation, true}, {versions, ['tlsv1.3']}], + server), + ok. + + +options_cookie(_Config) -> + ?OK(#{cookie := true}, [], server), + ?OK(#{cookie := false}, [{cookie, false}], server), + + %% Errors + ?ERR({cookie, []}, [{cookie, []}], server), + ?ERR({option, server_only, cookie}, [{cookie, true}], client), + ?ERR({options, incompatible, [cookie, {versions, _}]}, + [{cookie, true}, {versions, ['tlsv1.2']}], server), + ok. + +options_crl(_Config) -> + ?OK(#{crl_cache := {ssl_crl_cache, _}, crl_check := false}, [], server), + ?OK(#{crl_cache := {ssl_crl_cache, _}, crl_check := true}, [{crl_check, true}], server), + ?OK(#{crl_cache := {ssl_crl_hash_dir, {_,_}}, crl_check := true}, + [{crl_check, true}, {crl_cache, {ssl_crl_hash_dir, {internal, [{dir, "/tmp"}]}}}], + server), + ?OK(#{crl_cache := {ssl_crl_cache, {_,_}}, crl_check := true}, + [{crl_check, true}, {crl_cache, {ssl_crl_cache, {internal, [{http, 5000}]}}}], + server), + %% Errors + ?ERR({crl_check, foo}, [{crl_check, foo}], server), + ?ERR({crl_cache, {a,b,c}}, [{crl_cache, {a,b,c}}], server), + ok. + +options_hostname_check(_Config) -> + ?OK(#{customize_hostname_check := []}, [], client), + ?OK(#{customize_hostname_check := [{match_fun, _}]}, + [{customize_hostname_check, [{match_fun, fun() -> ok end}]}], + client), + %% Error + ?ERR({customize_hostname_check, _}, [{customize_hostname_check, {match_fun, pb_fun}}], client), + ok. + +options_dh(_Config) -> %% dh dhfile + ?OK(#{dh := undefined, dhfile := undefined}, [], client), + ?OK(#{dh := <<>>, dhfile := undefined}, [{dh, <<>>}], client), + ?OK(#{dh := undefined, dhfile := <<"/tmp/foo">>}, [{dhfile, <<"/tmp/foo">>}], client), + ?OK(#{dh := <<>>, dhfile := undefined}, [{dh, <<>>}, {dhfile, <<"/tmp/foo">>}], client), + + %% Should be and error + ?OK(#{dh := undefined, dhfile := <<"/tmp/foo">>}, %% Not available in 1.3 + [{dhfile, <<"/tmp/foo">>}, {versions, ['tlsv1.3']}], client), + + %% Error + ?ERR({dh, not_a_bin}, [{dh, not_a_bin}], client), + ?ERR({dhfile, not_a_filename}, [{dhfile, not_a_filename}], client), + ok. + +options_early_data(_Config) -> %% early_data, session_tickets and use_ticket + ?OK(#{early_data := undefined, session_tickets := disabled}, + [], client), + ?OK(#{early_data := disabled, session_tickets := disabled, stateless_tickets_seed := undefined}, + [], server), + + ?OK(#{early_data := <<>>, session_tickets := auto}, + [{early_data, <<>>}, {session_tickets, auto}], client), + ?OK(#{early_data := <<>>, session_tickets := manual, use_ticket := [<<1>>]}, + [{early_data, <<>>}, {session_tickets, manual}, {use_ticket, [<<1>>]}], + client), + + ?OK(#{early_data := enabled, stateless_tickets_seed := <<"foo">>}, + [{early_data, enabled}, {session_tickets, stateless}, {stateless_tickets_seed, <<"foo">>}], server), + + ?OK(#{early_data := disabled}, [{early_data, disabled}], server), + + %% Errors + ?ERR({option, client_only, use_ticket}, [{use_ticket, []}], server), + ?ERR({options, incompatible, _}, [{use_ticket, [<<>>]}], client), + + ?ERR({options, {session_tickets, foo}}, [{session_tickets, foo}], server), + ?ERR({options, {session_tickets, manual}}, [{session_tickets, manual}], server), + ?ERR({options, {session_tickets, stateful}}, [{session_tickets, stateful}], client), + ?ERR({options, incompatible, [session_tickets, {versions, _}]}, + [{session_tickets, stateful}, {versions, ['tlsv1.2']}], server), + + ?ERR({use_ticket, foo}, + [{use_ticket, foo}, {session_tickets, manual}], client), + + ?ERR({early_data,undefined}, [{early_data, undefined}], client), + ?ERR({options, incompatible, [early_data, {session_tickets, _}]}, + [{early_data, <<>>}], client), + ?ERR({options, {early_data, enabled}}, + [{early_data, enabled}, {session_tickets, auto}], client), + ?ERR({options, incompatible, [early_data, _, {use_ticket, _}]}, + [{early_data, <<>>}, {session_tickets, manual}], client), + + ?ERR({options, incompatible, [early_data, {session_tickets, _}]}, + [{early_data, enabled}], server), + ?ERR({options, {early_data, <<>>}}, + [{early_data, <<>>}, {session_tickets, stateless}], server), + + ?ERR({options, incompatible, [stateless_tickets_seed, {session_tickets, stateful}]}, + [{stateless_tickets_seed, <<"foo">>}, {session_tickets, stateful}], + server), + ?ERR({stateless_tickets_seed, foo}, + [{stateless_tickets_seed, foo}, {session_tickets, stateless}], + server), + ?ERR({option, server_only, stateless_tickets_seed}, + [{stateless_tickets_seed, <<"foo">>}], client), + ok. + +options_eccs(_Config) -> + Curves = tl(ssl:eccs()), + ?OK(#{eccs := {elliptic_curves, [_|_]}}, [], client), + ?OK(#{eccs := {elliptic_curves, [_|_]}}, [{eccs, Curves}], client), + + %% Errors + ?ERR({eccs, not_a_list}, [{eccs, not_a_list}], client), + ?ERR({eccs, none_valid}, [{eccs, []}], client), + ?ERR({eccs, none_valid}, [{eccs, [foo]}], client), + ok. + +options_verify(Config) -> %% fail_if_no_peer_cert, verify, verify_fun, partial_chain + Cert = proplists:get_value(cert, ssl_test_lib:ssl_options(server_rsa_der_opts, Config)), + {ok, #config{ssl = DefOpts = #{verify_fun := {DefVerify,_}}}} = ssl:handle_options([], client, "dummy.host.org"), + + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {DefVerify, []}, partial_chain := _}, + [], client), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {DefVerify, []}, partial_chain := _}, + [], server), + ?OK(#{fail_if_no_peer_cert := true, verify := verify_peer, verify_fun := undefined, partial_chain := _}, + [{fail_if_no_peer_cert, true}, {verify, verify_peer}, {cacerts, [Cert]}], + server), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {DefVerify, []}, partial_chain := _}, + [{verify, verify_none}], client), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_peer, verify_fun := undefined, partial_chain := _}, + [{verify, verify_peer}, {cacerts, [Cert]}], server), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {_, []}, partial_chain := _}, + [{partial_chain, fun(_) -> ok end}], client), + + OldF = fun(_) -> ok end, + NewF = fun(_,_,_) -> ok end, + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {_, OldF}, partial_chain := _}, + [{verify_fun, OldF}], client), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_none, verify_fun := {NewF, foo}, partial_chain := _}, + [{verify_fun, {NewF, foo}}], client), + ?OK(#{fail_if_no_peer_cert := false, verify := verify_peer, verify_fun := {NewF, foo}, partial_chain := _}, + [{verify_fun, {NewF, foo}}, {verify, verify_peer}, {cacerts, [Cert]}], + server), + + %% check verify_fun in update_options case + #{verify_fun := undefined} = ssl:update_options([{verify, verify_peer}, {cacerts, [Cert]}], client, DefOpts), + #{verify_fun := {NewF, bar}} = ssl:update_options([{verify, verify_peer}, {cacerts, [Cert]}, + {verify_fun, {NewF, bar}}], + client, DefOpts), + + %% Errors + ?ERR({partial_chain, undefined}, [{partial_chain, undefined}], client), + ?ERR({options, incompatible, [{verify, verify_none}, {fail_if_no_peer_cert, true}]}, + [{fail_if_no_peer_cert, true}], server), + ?ERR({verify, verify}, [{verify, verify}], client), + ?ERR({option, server_only, fail_if_no_peer_cert}, + [{fail_if_no_peer_cert, true}, {verify, verify_peer}, {cacerts, [Cert]}], + client), + ?ERR({options, incompatible, [{verify, _}, {cacerts, undefined}]}, [{verify, verify_peer}], server), + ?ERR({partial_chain, not_a_fun}, [{partial_chain, not_a_fun}], client), + ?ERR({verify_fun, not_a_fun}, [{verify_fun, not_a_fun}], client), + ok. + +options_fallback(_Config) -> + ?OK(#{fallback := false}, [], client), + ?OK(#{fallback := true}, [{fallback, true}], client), + + %% Errors + ?ERR({option, client_only, fallback}, [{fallback, true}], server), + ?ERR({fallback, []}, [{fallback, []}], client), + ?ERR({options, incompatible, [fallback, {versions, _}]}, + [{fallback, true}, {versions, ['tlsv1.3']}], client), + ok. + +options_handshake(_Config) -> %% handshake + ?OK(#{handshake := full, max_handshake_size := 262144}, + [], client), + ?OK(#{handshake := hello, max_handshake_size := 123800}, + [{handshake, hello}, {max_handshake_size, 123800}], client), + + + %% Errors + ?ERR({handshake, []}, [{handshake, []}], server), + ?ERR({max_handshake_size, 8388608}, [{max_handshake_size, 8388608}], server), + ?ERR({max_handshake_size, -1}, [{handshake, hello}, {max_handshake_size, -1}], client), + ok. + +options_process(_Config) -> % hibernate_after, spawn_opts + ?OK(#{hibernate_after := infinity, receiver_spawn_opts := [], sender_spawn_opts := []}, + [], client), + ?OK(#{hibernate_after := 10000, receiver_spawn_opts := [foo], sender_spawn_opts := [bar]}, + [{hibernate_after, 10000}, {receiver_spawn_opts, [foo]}, {sender_spawn_opts, [bar]}], + client), + %% Errors + ?ERR({hibernate_after, -1}, [{hibernate_after, -1}], server), + ?ERR({receiver_spawn_opts, not_a_list}, [{receiver_spawn_opts, not_a_list}], server), + ?ERR({sender_spawn_opts, not_a_list}, [{sender_spawn_opts, not_a_list}], server), + ok. + +options_honor(_Config) -> %% honor_cipher_order & honor_ecc_order + ?OK(#{honor_cipher_order := false, honor_ecc_order := false}, + [], server), + ?OK(#{honor_cipher_order := true, honor_ecc_order := true}, + [{honor_cipher_order, true}, {honor_ecc_order, true}], + server), + %% Errors + ?ERR({option, server_only, honor_cipher_order}, + [{honor_cipher_order, true}], client), + ?ERR({option, server_only, honor_ecc_order}, + [{honor_ecc_order, true}], client), + ?ERR({honor_ecc_order, foo}, [{honor_ecc_order, foo}], server), + ok. + +options_debug(_Config) -> %% debug log_level keep_secrets + ?OK(#{log_level := notice, keep_secrets := false}, [], server), + ?OK(#{log_level := debug, keep_secrets := true}, + [{log_level, debug}, {keep_secrets, true}], server), + %% Errors + ?ERR({log_level, foo}, [{log_level, foo}], server), + ?ERR({keep_secrets, foo}, [{keep_secrets, foo}], server), + ok. + +options_renegotiate(_Config) -> %% key_update_at renegotiate_at secure_renegotiate + ?OK(#{key_update_at := ?KEY_USAGE_LIMIT_AES_GCM, renegotiate_at := 268435456, secure_renegotiate := true}, + [], server), + ?OK(#{key_update_at := 123456, renegotiate_at := 64000, secure_renegotiate := false}, + [{key_update_at, 123456}, {renegotiate_at, 64000}, {secure_renegotiate, false}], + server), + + %% Errors + ?ERR({options, incompatible, [key_update_at, {versions, _}]}, + [{key_update_at, 123456}, {versions, ['tlsv1.2']}], server), + ?ERR({options, incompatible, [secure_renegotiate, {versions, _}]}, + [{secure_renegotiate, true}, {versions, ['tlsv1.3']}], server), + + ?ERR({key_update_at, -1}, [{key_update_at, -1}], server), + ?ERR({renegotiate_at, not_a_int}, [{renegotiate_at, not_a_int}], server), + ?ERR({renegotiate_at, -1}, [{renegotiate_at, -1}], server), + ok. + +options_middlebox(_Config) -> %% middlebox_comp_mode + ?OK(#{middlebox_comp_mode := true}, [], client), + ?OK(#{middlebox_comp_mode := false}, [{middlebox_comp_mode, false}], client), + + + %% Errors + ?ERR({middlebox_comp_mode, foo}, [{middlebox_comp_mode, foo}], server), + ?ERR({options, incompatible, [middlebox_comp_mode, {versions, _}]}, + [{middlebox_comp_mode, false}, {versions, ['tlsv1.2']}], server), + ok. + +options_frag_len(_Config) -> %% max_fragment_length + ?OK(#{max_fragment_length := undefined}, [], client), + ?OK(#{max_fragment_length := 2048}, [{max_fragment_length, 2048}], client), + + %% Errors + ?ERR({option, client_only, max_fragment_length}, + [{max_fragment_length, 2048}], server), + ?ERR({max_fragment_length,2000}, [{max_fragment_length, 2000}], client), + ok. + +options_oscp(Config) -> + Cert = proplists:get_value(cert, ssl_test_lib:ssl_options(server_rsa_der_opts, Config)), + ?OK(#{ocsp_stapling := false, ocsp_nonce := true, ocsp_responder_certs := []}, + [], client), + ?OK(#{ocsp_stapling := true}, + [{ocsp_stapling, true}], client), + ?OK(#{ocsp_stapling := true, ocsp_nonce := false, ocsp_responder_certs := [_,_]}, + [{ocsp_stapling, true}, {ocsp_nonce, false}, {ocsp_responder_certs, [Cert,Cert]}], + client), + %% Errors + ?ERR({ocsp_stapling, foo}, [{ocsp_stapling, 'foo'}], client), + ?ERR({ocsp_nonce, foo}, [{ocsp_nonce, 'foo'}], client), + ?ERR({ocsp_responder_certs, foo}, [{ocsp_responder_certs, 'foo'}], client), + ?ERR({options, incompatible, [{ocsp_nonce, false}, {ocsp_stapling, false}]}, + [{ocsp_nonce, false}], client), + ?ERR({options, incompatible, [ocsp_responder_certs, {ocsp_stapling, false}]}, + [{ocsp_responder_certs, [Cert]}], server), + ?ERR({ocsp_responder_certs, [_]}, + [{ocsp_stapling, true}, {ocsp_responder_certs, ['NOT A BINARY']}], + client), + ok. + +options_padding(_Config) -> + ?OK(#{padding_check := true}, [], server), + ?OK(#{padding_check := false}, [{padding_check, false}, {versions, [tlsv1]}], server), + %% Errors + ?ERR({padding_check, foo}, [{padding_check, foo}, {versions, [tlsv1]}], server), + ?ERR({options, incompatible, [padding_check, {versions, ['tlsv1.3','tlsv1.2']}]}, + [{padding_check, false}], server), + ok. + +options_identity(_Config) -> %% psk_identity srp_identity and user_lookup_fun + ?OK(#{psk_identity := undefined, srp_identity := undefined, user_lookup_fun := undefined}, + [], client), + ?OK(#{psk_identity := <<"foobar">>, srp_identity := undefined, user_lookup_fun := undefined}, + [{psk_identity, "foobar"}], server), + ?OK(#{psk_identity := undefined, srp_identity := {<<"user">>, <<"pwd">>}, user_lookup_fun := undefined}, + [{srp_identity, {"user", "pwd"}}], client), + ?OK(#{psk_identity := undefined, srp_identity := undefined, user_lookup_fun := {_, args}}, + [{user_lookup_fun, {fun(_,_,_) -> ok end, args}}], client), + + %% Should fail client option only + ?OK(#{psk_identity := undefined, srp_identity := {<<"user">>, <<"pwd">>}, user_lookup_fun := undefined}, + [{srp_identity, {"user", "pwd"}}], server), + + %% Errors + ?ERR({srp_identity, {_,_}}, %% FIXME doesn't like binary strings + [{srp_identity, {<<"user">>, <<"pwd">>}}], client), + ?ERR({psk_identity, _}, %% FIXME doesn't like binary strings + [{psk_identity, <<"user">>}], client), + ?ERR({srp_identity, _}, [{srp_identity, "user"}], client), + ?ERR({user_lookup_fun, _}, + [{user_lookup_fun, {fun(_,_) -> ok end, args}}], + client), + + ?ERR({options, incompatible, [psk_identity, _]}, + [{psk_identity, "foobar"},{versions, ['tlsv1.3']}], + server), + ?ERR({options, incompatible, [srp_identity, _]}, + [{srp_identity, {"user", "pwd"}},{versions, ['tlsv1.3']}], + client), + ?ERR({options, incompatible, [user_lookup_fun, _]}, + [{user_lookup_fun, {fun(_,_,_) -> ok end, args}}, {versions, ['tlsv1.3']}], + client), + ok. + +options_reuse_session(_Config) -> + ?OK(#{reuse_session := undefined, reuse_sessions := true}, [], client), + ?OK(#{reuse_session := _, reuse_sessions := true}, [], server), + + ?OK(#{reuse_session := <<>>, reuse_sessions := save}, + [{reuse_session, <<>>}, {reuse_sessions, save}], client), + ?OK(#{reuse_session := {<<>>, <<>>}, reuse_sessions := false}, + [{reuse_session, {<<>>, <<>>}}, {reuse_sessions, false}], client), + + RS_F = fun(_,_,_,_) -> ok end, + ?OK(#{reuse_session := RS_F, reuse_sessions := false}, + [{reuse_session, RS_F}, {reuse_sessions, false}], + server), + + %% Errors + ?ERR({options, incompatible, [reuse_session, _]}, + [{reuse_session, RS_F}, {versions, ['tlsv1.3']}], + server), + ?ERR({options, incompatible, [reuse_sessions, _]}, + [{reuse_sessions, true}, {versions, ['tlsv1.3']}], + server), + ?ERR({reuse_session, RS_F}, + [{reuse_session, RS_F},{reuse_sessions, false}], + client), + ?ERR({reuse_sessions, foo}, [{reuse_sessions, foo}], server), + ?ERR({reuse_session, foo}, [{reuse_session, foo}], server), + ?ERR({reuse_sessions, save}, [{reuse_sessions, save}], server), + ?ERR({reuse_session, <<>>}, [{reuse_session, <<>>}], server), + + ok. + +options_sni(_Config) -> %% server_name_indication + ?OK(#{server_name_indication := "dummy.host.org"}, [], client), + ?OK(#{server_name_indication := disable}, [{server_name_indication, disable}], client), + ?OK(#{server_name_indication := "dummy.org"}, [{server_name_indication, "dummy.org"}], client), + + ?OK(#{sni_fun := undefined, sni_hosts := []}, [], server), + + ?OK(#{sni_fun := undefined, sni_hosts := [{"a",[]}]}, + [{sni_hosts, [{"a", []}]}], server), + SNI_F = fun(_) -> sni end, + ?OK(#{sni_fun := SNI_F, sni_hosts := []}, [{sni_fun, SNI_F}], server), + + %% Errors + ?ERR({option, client_only, server_name_indication}, + [{server_name_indication, "dummy.org"}], server), + ?ERR({option, server_only, sni_hosts}, [{sni_hosts, [{"a", []}]}], client), + ?ERR({option, server_only, sni_fun}, [{sni_fun, SNI_F}], client), + + ?ERR({server_name_indication, foo}, [{server_name_indication, foo}], client), + ?ERR({sni_hosts, foo}, [{sni_hosts, foo}], server), + ?ERR({sni_fun, foo}, [{sni_fun, foo}], server), + + ?ERR({options, incompatible, [sni_fun, sni_hosts]}, + [{sni_fun, SNI_F}, {sni_hosts, [{"a", []}]}], server), + ok. + +options_sign_alg(_Config) -> %% signature_algs[_cert] + ?OK(#{signature_algs := [_|_], signature_algs_cert := undefined}, + [], client), + ?OK(#{signature_algs := [rsa_pss_rsae_sha512,{sha512,rsa}], signature_algs_cert := undefined}, + [{signature_algs, [rsa_pss_rsae_sha512,{sha512,rsa}]}], client), + ?OK(#{signature_algs := [_|_], signature_algs_cert := [eddsa_ed25519, rsa_pss_rsae_sha512]}, + [{signature_algs_cert, [eddsa_ed25519, rsa_pss_rsae_sha512]}], + client), + + %% Errors + ?ERR({signature_algs, not_a_list}, [{signature_algs, not_a_list}], client), + ?ERR({signature_algs_cert, not_a_list}, [{signature_algs_cert, not_a_list}], client), + ?ERR({signature_algs, [foobar]}, [{signature_algs, [foobar]}], client), + ?ERR({signature_algs_cert, [foobar]}, [{signature_algs_cert, [foobar]}], client), + ?ERR({options, incompatible, [signature_algs_cert, {versions, _}]}, + [{signature_algs_cert, [eddsa_ed25519, rsa_pss_rsae_sha512]}, {versions, ['tlsv1.1']}], + client), + ?ERR({options, incompatible, [signature_algs_cert, {versions, _}]}, + [{signature_algs_cert, [eddsa_ed25519, rsa_pss_rsae_sha512]}, {versions, ['tlsv1.1']}], + server), + ?ERR({options, {no_supported_algorithms, {signature_algs,[]}}}, + [{signature_algs, []}], client), + ?ERR({options, {no_supported_signature_schemes, {signature_algs_cert,[]}}}, + [{signature_algs_cert, []}], + client), + ok. + +options_supported_groups(_Config) -> + %% FIXME group() type doesn't cover the values below + ?OK(#{supported_groups := {supported_groups, [x25519,x448,secp256r1,secp384r1]}}, + [], client), + ?OK(#{supported_groups := {supported_groups, [secp521r1, ffdhe2048]}}, + [{supported_groups, [secp521r1, ffdhe2048]}], client), + + %% ERRORs + ?ERR({{'tlvs1.2'},{versions,[{'tlvs1.2'}]}}, + [{supported_groups, []}, {versions, [{'tlvs1.2'}]}], client), + ?ERR({supported_groups, not_a_list}, [{supported_groups, not_a_list}], client), + ?ERR({supported_groups, none_valid}, [{supported_groups, []}], client), + ?ERR({supported_groups, none_valid}, [{supported_groups, [foo]}], client), + ok. + %%------------------------------------------------------------------- default_reject_anonymous()-> @@ -2348,8 +3136,8 @@ client_options_negative_dependency_version() -> client_options_negative_dependency_version(Config) when is_list(Config) -> start_client_negative(Config, [{versions, ['tlsv1.1', 'tlsv1.2']}, {session_tickets, manual}], - {options,dependency, - {session_tickets,{versions,['tlsv1.3']}}}). + {options,incompatible, + [session_tickets,{versions,['tlsv1.2', 'tlsv1.1']}]}). %%-------------------------------------------------------------------- client_options_negative_dependency_stateless() -> @@ -2358,8 +3146,7 @@ client_options_negative_dependency_stateless(Config) when is_list(Config) -> start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {anti_replay, '10k'}, {session_tickets, manual}], - {options,dependency, - {anti_replay,{session_tickets,[stateless]}}}). + {option, server_only, anti_replay}). %%-------------------------------------------------------------------- @@ -2368,45 +3155,37 @@ client_options_negative_dependency_role() -> client_options_negative_dependency_role(Config) when is_list(Config) -> start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, stateless}], - {options,role, - {session_tickets,{stateless,{client,[disabled,manual,auto]}}}}). + {options,{session_tickets,stateless}}). %%-------------------------------------------------------------------- client_options_negative_early_data() -> [{doc,"Test client option early_data."}]. client_options_negative_early_data(Config) when is_list(Config) -> start_client_negative(Config, [{versions, ['tlsv1.2']}, - {early_data, "test"}], - {options,dependency, - {early_data,{versions,['tlsv1.3']}}}), + {session_tickets, manual}, + {early_data, <<"test">>}], + {options,incompatible, + [session_tickets,{versions,['tlsv1.2']}]}), start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, - {early_data, "test"}], - {options,dependency, - {early_data,{session_tickets,[manual,auto]}}}), + {early_data, <<"test">>}], + {options,incompatible, + [early_data,{session_tickets,disabled}]}), start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, stateful}, - {early_data, "test"}], - {options,role, - {session_tickets, - {stateful,{client,[disabled,manual,auto]}}}}), - start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, - {session_tickets, disabled}, - {early_data, "test"}], - {options,dependency, - {early_data,{session_tickets,[manual,auto]}}}), + {early_data, <<"test">>}], + {options, {session_tickets, stateful}}), start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, manual}, - {early_data, "test"}], - {options,dependency, - {early_data, use_ticket}}), + {early_data, <<"test">>}], + {options,incompatible, + [early_data, {session_tickets, manual}, {use_ticket, undefined}]}), start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, manual}, {use_ticket, [<<"ticket">>]}, {early_data, "test"}], - {options, type, - {early_data, {"test", not_binary}}}), + {options, {early_data, "test"}}), %% All options are ok but there is no server start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, manual}, @@ -2414,11 +3193,6 @@ client_options_negative_early_data(Config) when is_list(Config) -> {early_data, <<"test">>}], econnrefused), - start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, - {session_tickets, auto}, - {early_data, "test"}], - {options, type, - {early_data, {"test", not_binary}}}), %% All options are ok but there is no server start_client_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, auto}, @@ -2430,31 +3204,25 @@ server_options_negative_early_data() -> [{doc,"Test server option early_data."}]. server_options_negative_early_data(Config) when is_list(Config) -> start_server_negative(Config, [{versions, ['tlsv1.2']}, - {early_data, "test"}], - {options,dependency, - {early_data,{versions,['tlsv1.3']}}}), + {early_data, enabled}, + {session_tickets, stateful} + ], + {options,incompatible, + [session_tickets,{versions,['tlsv1.2']}]}), start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, - {early_data, "test"}], - {options,dependency, - {early_data,{session_tickets,[stateful,stateless]}}}), + {early_data, enabled}], + {options,incompatible, + [early_data,{session_tickets,disabled}]}), start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, manual}, - {early_data, "test"}], - {options,role, - {session_tickets, - {manual,{server,[disabled,stateful,stateless]}}}}), - start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, - {session_tickets, disabled}, - {early_data, "test"}], - {options,dependency, - {early_data,{session_tickets,[stateful,stateless]}}}), + {early_data, enabled}], + {options, {session_tickets, manual}}), start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, stateful}, {early_data, "test"}], - {options,role, - {early_data,{"test",{server,[disabled,enabled]}}}}). + {options, {early_data, "test"}}). %%-------------------------------------------------------------------- server_options_negative_version_gap() -> @@ -2470,8 +3238,36 @@ server_options_negative_dependency_role() -> server_options_negative_dependency_role(Config) when is_list(Config) -> start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, {session_tickets, manual}], - {options,role, - {session_tickets,{manual,{server,[disabled,stateful,stateless]}}}}). + {options,{session_tickets,manual}}). + +%%-------------------------------------------------------------------- +server_options_negative_stateless_tickets_seed() -> + [{doc, "Test server option stateless_tickets_seed"}]. +server_options_negative_stateless_tickets_seed(Config) -> + Seed = crypto:strong_rand_bytes(32), + + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {stateless_tickets_seed, Seed}], + {options, incompatible, + [stateless_tickets_seed, {session_tickets, disabled}]}), + + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, disabled}, + {stateless_tickets_seed, Seed}], + {options, incompatible, + [stateless_tickets_seed, {session_tickets, disabled}]}), + + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, stateful}, + {stateless_tickets_seed, Seed}], + {options, incompatible, + [stateless_tickets_seed, {session_tickets, stateful}]}), + + InvalidSeed1 = 12345, + start_server_negative(Config, [{versions, ['tlsv1.2', 'tlsv1.3']}, + {session_tickets, stateless}, + {stateless_tickets_seed, InvalidSeed1}], + {options, {stateless_tickets_seed, InvalidSeed1}}). %%-------------------------------------------------------------------- honor_server_cipher_order_tls13() -> @@ -2529,27 +3325,24 @@ invalid_options_tls13() -> invalid_options_tls13(Config) when is_list(Config) -> TestOpts = [{{beast_mitigation, one_n_minus_one}, - {options, dependency, - {beast_mitigation,{versions,[tlsv1]}}}, + {options, incompatible, + [beast_mitigation,{versions,['tlsv1.3']}]}, common}, {{next_protocols_advertised, [<<"http/1.1">>]}, - {options, dependency, - {next_protocols_advertised, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, + [next_protocols_advertised, {versions,['tlsv1.3']}]}, server}, {{client_preferred_next_protocols, {client, [<<"http/1.1">>]}}, - {options, dependency, - {client_preferred_next_protocols, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, + [client_preferred_next_protocols, {versions,['tlsv1.3']}]}, client}, {{client_renegotiation, false}, - {options, dependency, - {client_renegotiation, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, + [client_renegotiation, {versions,['tlsv1.3']}]}, server }, @@ -2559,49 +3352,36 @@ invalid_options_tls13(Config) when is_list(Config) -> }, {{padding_check, false}, - {options, dependency, - {padding_check,{versions,[tlsv1]}}}, + {options, incompatible, [padding_check,{versions,['tlsv1.3']}]}, common}, {{psk_identity, "Test-User"}, - {options, dependency, - {psk_identity,{versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [psk_identity,{versions,['tlsv1.3']}]}, common}, {{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, <<1,2,3>>}}, - {options, dependency, - {user_lookup_fun,{versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [user_lookup_fun,{versions,['tlsv1.3']}]}, common}, {{reuse_session, fun(_,_,_,_) -> false end}, - {options, dependency, - {reuse_session, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [reuse_session, {versions,['tlsv1.3']}]}, server}, {{reuse_session, <<1,2,3,4>>}, - {options, dependency, - {reuse_session, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [reuse_session, {versions,['tlsv1.3']}]}, client}, {{reuse_sessions, true}, - {options, dependency, - {reuse_sessions, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [reuse_sessions, {versions,['tlsv1.3']}]}, common}, {{secure_renegotiate, false}, - {options, dependency, - {secure_renegotiate, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {options, incompatible, [secure_renegotiate, {versions,['tlsv1.3']}]}, common}, - {{srp_identity, false}, - {options, dependency, - {srp_identity, - {versions,[tlsv1,'tlsv1.1','tlsv1.2']}}}, + {{srp_identity, {"user", "passwd"}}, + {options, incompatible, [srp_identity, {versions,['tlsv1.3']}]}, client} ], diff --git a/lib/ssl/test/ssl_cert_SUITE.erl b/lib/ssl/test/ssl_cert_SUITE.erl index 1dcac41b19..ea7a36c167 100644 --- a/lib/ssl/test/ssl_cert_SUITE.erl +++ b/lib/ssl/test/ssl_cert_SUITE.erl @@ -66,6 +66,8 @@ missing_root_cert_auth_user_verify_fun_accept/1, missing_root_cert_auth_user_verify_fun_reject/0, missing_root_cert_auth_user_verify_fun_reject/1, + missing_root_cert_auth_user_old_verify_fun_accept/0, + missing_root_cert_auth_user_old_verify_fun_accept/1, verify_fun_always_run_client/0, verify_fun_always_run_client/1, verify_fun_always_run_server/0, @@ -235,6 +237,7 @@ all_version_tests() -> missing_root_cert_auth, missing_root_cert_auth_user_verify_fun_accept, missing_root_cert_auth_user_verify_fun_reject, + missing_root_cert_auth_user_old_verify_fun_accept, verify_fun_always_run_client, verify_fun_always_run_server, incomplete_chain_auth, @@ -506,7 +509,8 @@ missing_root_cert_auth(Config) when is_list(Config) -> {options, no_reuse(n_version(Version)) ++ [{verify, verify_peer} | ServerOpts]}]), - ssl_test_lib:check_result(Server, {error, {options, {cacertfile, ""}}}), + Error = {error, {options, incompatible, [{verify,verify_peer}, {cacerts,undefined}]}}, + ssl_test_lib:check_result(Server, Error), ClientOpts = proplists:delete(cacertfile, ssl_test_lib:ssl_options(extra_client, client_cert_opts, Config)), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, 0}, @@ -514,7 +518,7 @@ missing_root_cert_auth(Config) when is_list(Config) -> {options, [{verify, verify_peer} | ClientOpts]}]), - ssl_test_lib:check_result(Client, {error, {options, {cacertfile, ""}}}). + ssl_test_lib:check_result(Client, Error). %%-------------------------------------------------------------------- missing_root_cert_auth_user_verify_fun_accept() -> @@ -523,6 +527,7 @@ missing_root_cert_auth_user_verify_fun_accept() -> missing_root_cert_auth_user_verify_fun_accept(Config) -> ServerOpts = ssl_test_lib:ssl_options(extra_server, server_cert_opts, Config), + ClientCaCerts = public_key:cacerts_get(), FunAndState = {fun(_,{bad_cert, unknown_ca}, UserState) -> {valid, UserState}; (_,{bad_cert, _} = Reason, _) -> @@ -534,16 +539,19 @@ missing_root_cert_auth_user_verify_fun_accept(Config) -> (_, valid_peer, UserState) -> {valid, UserState} end, []}, - ClientOpts = ssl_test_lib:ssl_options(extra_client, [{verify, verify_peer}, - {verify_fun, FunAndState}], Config), + ClientOpts = ssl_test_lib:ssl_options(extra_client, + [{verify, verify_peer}, {verify_fun, FunAndState}, + {cacerts, ClientCaCerts}], + Config), ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). %%-------------------------------------------------------------------- -missing_root_cert_auth_user_backwardscompatibility_verify_fun_accept() -> +missing_root_cert_auth_user_old_verify_fun_accept() -> [{doc, "Test old style verify fun"}]. -missing_root_cert_auth_user_backwardscompatibility_verify_fun_accept(Config) -> +missing_root_cert_auth_user_old_verify_fun_accept(Config) -> ServerOpts = ssl_test_lib:ssl_options(extra_server, server_cert_opts, Config), + ClientCaCerts = public_key:cacerts_get(), AcceptBadCa = fun({bad_cert,unknown_ca}, Acc) -> Acc; (Other, Acc) -> [Other | Acc] end, @@ -554,8 +562,10 @@ missing_root_cert_auth_user_backwardscompatibility_verify_fun_accept(Config) -> [_|_] -> false end end, - ClientOpts = ssl_test_lib:ssl_options(extra_client, [{verify, verify_peer}, - {verify_fun, VerifyFun}], Config), + ClientOpts = ssl_test_lib:ssl_options(extra_client, + [{verify, verify_peer}, + {verify_fun, VerifyFun}, + {cacerts, ClientCaCerts}], Config), ssl_test_lib:basic_test(ClientOpts, ServerOpts, Config). %%-------------------------------------------------------------------- @@ -565,6 +575,7 @@ missing_root_cert_auth_user_verify_fun_reject() -> missing_root_cert_auth_user_verify_fun_reject(Config) -> ServerOpts = ssl_test_lib:ssl_options(extra_server, server_cert_opts, Config), + ClientCaCerts = public_key:cacerts_get(), FunAndState = {fun(_,{bad_cert, unknown_ca} = Reason, _UserState) -> {fail, Reason}; (_,{bad_cert, _} = Reason, _) -> @@ -576,8 +587,11 @@ missing_root_cert_auth_user_verify_fun_reject(Config) -> (_, valid_peer, UserState) -> {valid, UserState} end, []}, - ClientOpts = ssl_test_lib:ssl_options(extra_client, [{verify, verify_peer}, - {verify_fun, FunAndState}], Config), + ClientOpts = ssl_test_lib:ssl_options(extra_client, + [{verify, verify_peer}, + {verify_fun, FunAndState}, + {cacerts, ClientCaCerts}], + Config), ssl_test_lib:basic_alert(ClientOpts, ServerOpts, Config, unknown_ca). diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index c6154855e5..45192acbbe 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -38,6 +38,12 @@ %% Test cases -export([basic/0, basic/1, + embedded/0, + embedded/1, + ktls_encrypt_decrypt/0, + ktls_encrypt_decrypt/1, + ktls_verify/0, + ktls_verify/1, monitor_nodes/1, payload/0, payload/1, @@ -107,6 +113,9 @@ start_ssl_node_name(Name, Args) -> %%-------------------------------------------------------------------- all() -> [basic, + embedded, + ktls_encrypt_decrypt, + ktls_verify, monitor_nodes, payload, dist_port_overload, @@ -143,18 +152,25 @@ init_per_suite(Config0) -> end_per_suite(_Config) -> application:stop(crypto). -init_per_testcase(plain_verify_options = Case, Config) when is_list(Config) -> - SslFlags = setup_tls_opts(Config), - Flags = case os:getenv("ERL_FLAGS") of - false -> - os:putenv("ERL_FLAGS", SslFlags), - ""; - OldFlags -> - os:putenv("ERL_FLAGS", OldFlags ++ " " ++ SslFlags), - OldFlags - end, - common_init(Case, [{old_flags, Flags} | Config]); - +init_per_testcase(Case, Config) + when Case =:= ktls_verify, is_list(Config) -> + %% We need a connected socket + {ok, Listen} = gen_tcp:listen(0, [{active, false}]), + {ok, Port} = inet:port(Listen), + {ok, Client} = + gen_tcp:connect({127,0,0,1}, Port, [{active, false}]), + {ok, Server} = gen_tcp:accept(Listen), + try ktls_encrypt_decrypt(Client, Server, false) of + ok -> + common_init(Case, Config); + Other -> + Other + after + _ = gen_tcp:close(Server), + _ = gen_tcp:close(Client), + _ = gen_tcp:close(Listen) + end; +%% init_per_testcase(Case, Config) when is_list(Config) -> common_init(Case, Config). @@ -162,12 +178,7 @@ common_init(Case, Config) -> ct:timetrap({seconds, ?DEFAULT_TIMETRAP_SECS}), [{testcase, Case}|Config]. -end_per_testcase(Case, Config) when is_list(Config) -> - Flags = proplists:get_value(old_flags, Config), - catch os:putenv("ERL_FLAGS", Flags), - common_end(Case, Config). - -common_end(_, _Config) -> +end_per_testcase(_, _Config) -> ok. %%-------------------------------------------------------------------- @@ -179,6 +190,222 @@ basic() -> basic(Config) when is_list(Config) -> gen_dist_test(basic_test, Config). +embedded() -> + [{doc,"Test that two nodes can connect via ssl distribution in embedded mode"}]. +embedded(Config) when is_list(Config) -> + ReleaseDir = filename:join(proplists:get_value(priv_dir,Config), "embedded"), + EbinDir = filename:join(ReleaseDir,"ebin/"), + + %% Create an application for the test modules + Modules = [ssl_dist_test_lib, ?MODULE], + App = {application, tls_test, + [{description, "Erlang/OTP SSL test application"}, + {vsn, "1.0"}, + {modules, Modules}, + {registered,[]}, + {applications, [kernel, stdlib]}]}, + ok = filelib:ensure_path(EbinDir), + [{ok,_} = file:copy(code:which(Mod), filename:join(EbinDir, atom_to_list(Mod)++".beam")) + || Mod <- Modules], + ok = file:write_file(filename:join(EbinDir,"tls_test.app"), + io_lib:format("~p.",[App])), + + %% Create a release that we can boot from + Rel = {release, {"tls","1.0"}, {erts, get_app_vsn(erts)}, + [{tls_test, "1.0"}, + {kernel, get_app_vsn(kernel)}, + {stdlib, get_app_vsn(stdlib)}, + {public_key, get_app_vsn(public_key)}, + {asn1, get_app_vsn(asn1)}, + {sasl, get_app_vsn(sasl)}, + {crypto, get_app_vsn(crypto)}, + {ssl, get_app_vsn(ssl)}]}, + TlsRel = filename:join(ReleaseDir, "tls"), + ok = file:write_file(TlsRel ++ ".rel", io_lib:format("~p.",[Rel])), + code:add_patha(EbinDir), + ok = systools:make_script(TlsRel), + ok = systools:script2boot(TlsRel), + + %% Start two nodes in embedded mode and make sure they can connect + %% There used to be a bug here where crypto was not loaded early enough + %% for the distributed connection to work. + NodeConfig = [{app_opts, "-boot "++TlsRel++" -mode embedded -pa "++EbinDir++" "} | Config], + Node1 = peer:random_name(), + Node2 = peer:random_name(), + + NH1 = start_ssl_node([{node_name,Node1}|NodeConfig]), + %% The second node does `sync_nodes_mandatory` with the first in order for + %% a connection to be established very early in the boot sequence + ok = file:write_file( + filename:join(ReleaseDir,"node2.config"), + io_lib:format( + "~p.",[[{kernel, + [{sync_nodes_timeout,infinity}, + {sync_nodes_mandatory, + [list_to_atom(Node1++"@"++inet_db:gethostname())]}]}]])), + NH2 = start_ssl_node([{node_name,Node2}|NodeConfig], + " -config " ++ filename:join(ReleaseDir,"node2")), + + try + basic_test(NH1, NH2, Config) + catch + _:Reason -> + stop_ssl_node(NH1), + stop_ssl_node(NH2), + ct:fail(Reason) + end, + stop_ssl_node(NH1), + stop_ssl_node(NH2), + success(Config). + +%%-------------------------------------------------------------------- +ktls_encrypt_decrypt() -> + [{doc,"Test that kTLS encryption offloading works"}]. +ktls_encrypt_decrypt(Config) when is_list(Config) -> + %% We need a connected socket + {ok, Listen} = gen_tcp:listen(0, [{active, false}]), + {ok, Port} = inet:port(Listen), + {ok, Client} = + gen_tcp:connect({127,0,0,1}, Port, [{active, false}]), + {ok, Server} = gen_tcp:accept(Listen), + try ktls_encrypt_decrypt(Client, Server, true) + after + _ = gen_tcp:close(Server), + _ = gen_tcp:close(Client), + _ = gen_tcp:close(Listen) + end. + +ktls_encrypt_decrypt(Client, Server, Test) -> + Done = make_ref(), + try + case {os:type(), os:version()} of + {{unix,linux}, OsVersion} when {5,2,0} =< OsVersion -> + ok; + OS -> + throw({Done, skip, {os,OS}}) + end, + %% + %% Test and verify setup of Client TX encryption + %% + SOL_TCP = 6, TCP_ULP = 31, + TLS_VER = ((3 bsl 8) bor 4), + TLS_CIPHER = 52, + TLS_SALT = <<1,1,1,1>>, + TLS_IV = <<2,2,2,2,2,2,2,2>>, + TLS_KEY = + <<3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3>>, + TLS_crypto_info = + <<TLS_VER:16/native, TLS_CIPHER:16/native, + TLS_IV/binary, TLS_KEY/binary, TLS_SALT/binary, + 0:64/native>>, + SOL_TLS = 282, TLS_TX = 1, TLS_RX = 2, + %% + inet:setopts(Client, [{raw, SOL_TCP, TCP_ULP, <<"tls">>}]) + =:= ok + orelse + throw({Done, skip, set_ulp}), + (GetULP = + inet:getopts(Client, [{raw, SOL_TCP, TCP_ULP, 4}])) + =:= {ok, [{raw, SOL_TCP, TCP_ULP, <<"tls",0>>}]} + orelse + throw({Done, skip, {get_ulp, GetULP}}), + %% + RawOptTX = {raw, SOL_TLS, TLS_TX, TLS_crypto_info}, + RawOptRX = {raw, SOL_TLS, TLS_RX, TLS_crypto_info}, + (SetoptsResult = inet:setopts(Client, [RawOptTX])) =:= ok + orelse throw({Done, skip, {setopts_error,SetoptsResult}}), + (GetCryptoInfo = + inet:getopts( + Client, + [{raw, SOL_TLS, TLS_TX, byte_size(TLS_crypto_info)}])) + =:= {ok, [RawOptTX]} + orelse throw({Done, skip, {get_crypto_info,GetCryptoInfo}}), + %% + %% + %% + Test orelse throw(Done), + %% + %% + %% + %% Test to transfer encrypted data, + %% and also to not activate RX encryption and transfer data. + %% + Data = "The quick brown fox jumps over a lazy dog 0123456789", + %% Send encrypted from Client before Server has activated decryption + ok = gen_tcp:send(Client, Data), + receive after 500 -> ok end, % Give time for data to arrive + %% + %% Activate Server TX encryption + ok = inet:setopts(Server, [{raw, SOL_TCP, TCP_ULP, <<"tls">>}]), + ok = inet:setopts(Server, [RawOptTX]), + %% Send encrypted from Server + ok = gen_tcp:send(Server, Data), + %% Receive encrypted data without decryption + case gen_tcp:recv(Client, 0, 1000) of + {ok, Data} -> + ct:fail(recv_cleartext_data); + {ok, RandomData} when length(Data) < length(RandomData) -> + %% A TLS block should be longer than Data + ok + end, + %% Finally, activate Server decryption + ok = inet:setopts(Server, [RawOptRX]), + %% Receive and decrypt the data that was first sent + {ok, Data} = gen_tcp:recv(Server, 0, 1000), + ok + catch + Done -> + ok; + {Done, skip,SkipReason} -> + {skip, + lists:flatten( + io_lib:format("kTLS not supported: ~p", [SkipReason]))} + end. + + + + +%%-------------------------------------------------------------------- +ktls_verify() -> + [{doc, + "Test that two nodes can connect via ssl distribution over kTLS"}]. +ktls_verify(Config) -> + KTLSOpts = "-ssl_dist_opt " + "client_versions tlsv1.3 " + "server_versions tlsv1.3 " + "client_ciphers TLS_AES_256_GCM_SHA384 " + "server_ciphers TLS_AES_256_GCM_SHA384 " + "client_ktls true " + "server_ktls true ", + KTLSConfig = [{tls_verify_opts, KTLSOpts} | Config], + gen_dist_test( + fun (NH1, NH2) -> + basic_test(NH1, NH2, KTLSConfig), + 0 = ktls_count_tls_dist(NH1), + 0 = ktls_count_tls_dist(NH2), + ok + end, KTLSConfig). + +%% Verify that kTLS was activated (whitebox verification); +%% check that a specific supervisor has no child supervisor +%% which indicates that ssl_gen_statem:ktls_handover/1 has succeeded +%% +ktls_count_tls_dist(Node) -> + Key = supervisors, + case + lists:keyfind( + Key, 1, + apply_on_ssl_node( + Node, supervisor, count_children, + [tls_dist_connection_sup])) + of + {Key, N} -> + N; + false -> + 0 + end. + %%-------------------------------------------------------------------- %% Test net_kernel:monitor_nodes with nodedown_reason (OTP-17838) monitor_nodes(Config) when is_list(Config) -> @@ -259,11 +486,15 @@ plain_options(Config) when is_list(Config) -> plain_verify_options() -> [{doc,"Test specifying tls options including certificate verification options"}]. plain_verify_options(Config) when is_list(Config) -> - TLSOpts = "-ssl_dist_opt server_secure_renegotiate true " - "client_secure_renegotiate true " - "server_hibernate_after 500 client_hibernate_after 500" - "server_reuse_sessions true client_reuse_sessions true " - "server_depth 1 client_depth 1 ", + TLSOpts = "-ssl_dist_opt " + "server_secure_renegotiate true " + "client_secure_renegotiate true " + "server_hibernate_after 500 " + "client_hibernate_after 500 " + "server_reuse_sessions true " + "client_reuse_sessions true " + "server_depth 1 " + "client_depth 1 ", gen_dist_test(plain_verify_options_test, [{tls_verify_opts, TLSOpts} | Config]). %%-------------------------------------------------------------------- @@ -454,16 +685,20 @@ address_please(_, _, _) -> gen_dist_test(Test, Config) -> NH1 = start_ssl_node(Config), NH2 = start_ssl_node(Config), - try - ?MODULE:Test(NH1, NH2, Config) + try + if + is_atom(Test) -> + ?MODULE:Test(NH1, NH2, Config); + is_function(Test, 2) -> + Test(NH1, NH2) + end catch - _:Reason -> - stop_ssl_node(NH1), - stop_ssl_node(NH2), - ct:fail(Reason) + Class:Reason:Stacktrace -> + ct:fail({Class,Reason,Stacktrace}) + after + stop_ssl_node(NH1), + stop_ssl_node(NH2) end, - stop_ssl_node(NH1), - stop_ssl_node(NH2), success(Config). %% ssl_node side api @@ -486,6 +721,7 @@ try_setting_priority(TestFun, Config) -> {error,_} -> {skip, "Can not set priority on socket"} end. + basic_test(NH1, NH2, _) -> Node1 = NH1#node_handle.nodename, Node2 = NH2#node_handle.nodename, @@ -493,6 +729,8 @@ basic_test(NH1, NH2, _) -> [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end), + verify_tls(NH1, NH2), + %% The test_server node has the same cookie as the ssl nodes %% but it should not be able to communicate with the ssl nodes %% via the erlang distribution. @@ -582,6 +820,8 @@ payload_test(NH1, NH2, _) -> pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), + verify_tls(NH1, NH2), + [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end), @@ -618,6 +858,8 @@ plain_options_test(NH1, NH2, _) -> pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), + verify_tls(NH1, NH2), + [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end). @@ -627,6 +869,8 @@ plain_verify_options_test(NH1, NH2, _) -> pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), + verify_tls(NH1, NH2), + [Node2] = apply_on_ssl_node(NH1, fun () -> nodes() end), [Node1] = apply_on_ssl_node(NH2, fun () -> nodes() end). @@ -758,37 +1002,51 @@ start_ssl_node(Config, XArgs) -> App = proplists:get_value(app_opts, Config), SSLOpts = setup_tls_opts(Config), start_ssl_node_name( - Name, App ++ " " ++ SSLOpts ++ XArgs). + Name, App ++ " " ++ SSLOpts ++ " " ++ XArgs). mk_node_name(Config) -> - N = erlang:unique_integer([positive]), - Case = proplists:get_value(testcase, Config), - Hostname = - case proplists:get_value(hostname, Config) of - undefined -> ""; - Host -> "@" ++ Host - end, - atom_to_list(?MODULE) - ++ "_" - ++ atom_to_list(Case) - ++ "_" - ++ integer_to_list(N) ++ Hostname. - + case proplists:get_value(node_name, Config) of + undefined -> + N = erlang:unique_integer([positive]), + Case = proplists:get_value(testcase, Config), + Hostname = + case proplists:get_value(hostname, Config) of + undefined -> ""; + Host -> "@" ++ Host + end, + atom_to_list(?MODULE) + ++ "_" + ++ atom_to_list(Case) + ++ "_" + ++ integer_to_list(N) ++ Hostname; + Name -> + Name + end. setup_certs(Config) -> + {ok,Host} = inet:gethostname(), + Extensions = + {extensions, + [#'Extension'{ + extnID = ?'id-ce-subjectAltName', + extnValue = [{dNSName, Host}], + critical = false}]}, PrivDir = proplists:get_value(priv_dir, Config), - DerConfig = public_key:pkix_test_data(#{server_chain => #{root => rsa_root_key(1), - intermediates => [rsa_intermediate(2)], - peer => rsa_peer_key(3)}, - client_chain => #{root => rsa_root_key(1), - intermediates => [rsa_intermediate(5)], - peer => rsa_peer_key(6)}}), + DerConfig = + public_key:pkix_test_data( + #{server_chain => + #{root => [rsa_root_key(1)], + intermediates => [rsa_intermediate_conf(2)], + peer => [rsa_peer_key(3), Extensions]}, + client_chain => + #{root => [rsa_root_key(1)], + intermediates => [rsa_intermediate_conf(5)], + peer => [rsa_peer_key(6), Extensions]}}), ClientBase = filename:join([PrivDir, "rsa"]), - SeverBase = filename:join([PrivDir, "rsa"]), - - _ = x509_test:gen_pem_config_files(DerConfig, ClientBase, SeverBase). - + ServerBase = filename:join([PrivDir, "rsa"]), + _ = x509_test:gen_pem_config_files(DerConfig, ClientBase, ServerBase). + setup_tls_opts(Config) -> PrivDir = proplists:get_value(priv_dir, Config), SC = filename:join([PrivDir, "rsa_server_cert.pem"]), @@ -842,9 +1100,9 @@ add_ssl_opts_config(Config) -> KrnlDir = filename:join([LibDir, "kernel-" ++ KRNL_VSN]), {ok, _} = file:read_file_info(StdlDir), {ok, _} = file:read_file_info(KrnlDir), - SSL_VSN = vsn(ssl), - VSN_CRYPTO = vsn(crypto), - VSN_PKEY = vsn(public_key), + SSL_VSN = get_app_vsn(ssl), + VSN_CRYPTO = get_app_vsn(crypto), + VSN_PKEY = get_app_vsn(public_key), SslDir = filename:join([LibDir, "ssl-" ++ SSL_VSN]), {ok, _} = file:read_file_info(SslDir), @@ -896,19 +1154,12 @@ success(Config) -> _ -> ok end. -vsn(App) -> - application:start(App), - try - {value, - {ssl, - _, - VSN}} = lists:keysearch(App, - 1, - application:which_applications()), - VSN - after - application:stop(ssl) - end. +get_app_vsn(erts) -> + erlang:system_info(version); +get_app_vsn(App) -> + application:load(App), + {ok, AppKeys} = application:get_all_key(App), + proplists:get_value(vsn, AppKeys). verify_fail_always(_Certificate, _Event, _State) -> %% Create an ETS table, to record the fact that the verify function ran. @@ -940,6 +1191,18 @@ verify_pass_always(_Certificate, _Event, State) -> receive go_ahead -> ok end, {valid, State}. +verify_tls(NH1, NH2) -> + %% Verify that distribution protocol between nodes is TLS + Node1 = NH1#node_handle.nodename, + Node2 = NH2#node_handle.nodename, + {ok,NodeInfo2} = apply_on_ssl_node(NH1, net_kernel, node_info, [Node2]), + {ok,NodeInfo1} = apply_on_ssl_node(NH2, net_kernel, node_info, [Node1]), + {address,#net_address{protocol = tls}} = + lists:keyfind(address, 1, NodeInfo1), + {address,#net_address{protocol = tls}} = + lists:keyfind(address, 1, NodeInfo2), + ok. + localhost_ip(InetVer) -> {ok, Addr} = inet:getaddr(net_adm:localhost(), InetVer), Addr. @@ -963,13 +1226,13 @@ inet_ver() -> rsa_root_key(N) -> %% As rsa keygen is not guaranteed to be fast - [{key, ssl_test_lib:hardcode_rsa_key(N)}]. + {key, ssl_test_lib:hardcode_rsa_key(N)}. rsa_peer_key(N) -> %% As rsa keygen is not guaranteed to be fast - [{key, ssl_test_lib:hardcode_rsa_key(N)}]. + {key, ssl_test_lib:hardcode_rsa_key(N)}. -rsa_intermediate(N) -> +rsa_intermediate_conf(N) -> [{key, ssl_test_lib:hardcode_rsa_key(N)}]. diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index 1bde66a80b..a679ad5d82 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -129,8 +129,8 @@ decode_hello_handshake(_Config) -> 16#70, 16#64, 16#79, 16#2f, 16#32>>, Version = {3, 0}, - {Records, _Buffer} = tls_handshake:get_tls_handshakes(Version, HelloPacket, <<>>, - default_options_map()), + DefOpts = ssl:update_options([], client, #{}), + {Records, _Buffer} = tls_handshake:get_tls_handshakes(Version, HelloPacket, <<>>, DefOpts), {Hello, _Data} = hd(Records), Extensions = Hello#server_hello.extensions, @@ -280,7 +280,3 @@ is_supported(Hash) -> Algos = crypto:supports(), Hashs = proplists:get_value(hashs, Algos), lists:member(Hash, Hashs). - -default_options_map() -> - Fun = fun (_Key, {Default, _}) -> Default end, - maps:map(Fun, ?RULES). diff --git a/lib/ssl/test/ssl_mfl_SUITE.erl b/lib/ssl/test/ssl_mfl_SUITE.erl index a111c40339..100a3eaab2 100644 --- a/lib/ssl/test/ssl_mfl_SUITE.erl +++ b/lib/ssl/test/ssl_mfl_SUITE.erl @@ -107,11 +107,10 @@ client_option(Config) when is_list(Config) -> ok. %-------------------------------------------------------------------------------- -%% check max_fragment_length option on the server is ignored -%% and both sides can successfully send > 512 bytes +%% check default max_fragment_length both sides can successfully send > 512 bytes server_option(Config) when is_list(Config) -> Data = "mfl_server_options " ++ lists:duplicate(512, $x), - run_mfl_handshake(Config, undefined, Data, [], [{max_fragment_length, 512}]). + run_mfl_handshake(Config, undefined, Data, [], []). %-------------------------------------------------------------------------------- %% check max_fragment_length option on the client is accepted and reused diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index ee8825a724..0cc344f5d6 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -171,5 +171,4 @@ create_connection_states() -> }. default_options_map() -> - Fun = fun (_Key, {Default, _}) -> Default end, - maps:map(Fun, ?RULES). + ssl:update_options([], client, #{}). diff --git a/lib/ssl/test/ssl_session_ticket_SUITE.erl b/lib/ssl/test/ssl_session_ticket_SUITE.erl index 0c981b5b83..31d260beaa 100644 --- a/lib/ssl/test/ssl_session_ticket_SUITE.erl +++ b/lib/ssl/test/ssl_session_ticket_SUITE.erl @@ -45,6 +45,8 @@ ticket_reuse_anti_replay/1, ticket_reuse_anti_replay_server_restart/0, ticket_reuse_anti_replay_server_restart/1, + ticket_reuse_anti_replay_server_restart_reused_seed/0, + ticket_reuse_anti_replay_server_restart_reused_seed/1, basic_stateful_stateless/0, basic_stateful_stateless/1, basic_stateless_stateful/0, @@ -78,7 +80,9 @@ early_data_basic/0, early_data_basic/1, early_data_basic_auth/0, - early_data_basic_auth/1]). + early_data_basic_auth/1, + stateless_multiple_servers/0, + stateless_multiple_servers/1]). -include("tls_handshake.hrl"). @@ -104,7 +108,9 @@ groups() -> [ticketage_smaller_than_windowsize_anti_replay, ticketage_bigger_than_windowsize_anti_replay, ticketage_out_of_lifetime_anti_replay, ticket_reuse_anti_replay, - ticket_reuse_anti_replay_server_restart]}, + ticket_reuse_anti_replay_server_restart, + ticket_reuse_anti_replay_server_restart_reused_seed, + stateless_multiple_servers]}, {mixed, [], mixed_tests()}]. session_tests() -> @@ -243,7 +249,7 @@ ticketage_smaller_than_windowsize_anti_replay(Config) when is_list(Config) -> ticketage_bigger_than_windowsize_anti_replay() -> [{doc, "Session resumption with stateless tickets and anti_replay enabled." "Fresh ClientHellos." - "Ticket age bigger than windowsize. 0-RTT is expected to fail." + "Ticket age bigger than windowsize. 0-RTT is expected to succeed." "(Erlang client - Erlang server)"}]. ticketage_bigger_than_windowsize_anti_replay(Config) when is_list(Config) -> WindowSize = 3, @@ -252,10 +258,10 @@ ticketage_bigger_than_windowsize_anti_replay(Config) when is_list(Config) -> ssl_test_lib:check_result(Server0, ok, Client0, ok), Client1 = anti_replay_helper_connect(Server0, Client0, Port0, ClientNode, Hostname, ClientOpts, - {seconds, WindowSize + 2}, false), + {seconds, WindowSize + 2}, true), Client2 = anti_replay_helper_connect(Server0, Client0, Port0, ClientNode, Hostname, ClientOpts, - {seconds, 2*WindowSize + 2}, false), + {seconds, 2*WindowSize + 2}, true), process_flag(trap_exit, false), [ssl_test_lib:close(A) || A <- [Server0, Client0, Client1, Client2]]. @@ -325,6 +331,36 @@ ticket_reuse_anti_replay_server_restart(Config) when is_list(Config) -> process_flag(trap_exit, false), [ssl_test_lib:close(A) || A <- [Server0, Client2, Server1]]. +ticket_reuse_anti_replay_server_restart_reused_seed() -> + [{doc, "Verify 2 connection attempts with same stateless tickets " + "and server restart between, with the server using the same session " + "ticket encryption seed between restarts. Second attempt is expected to " + "fail as long as the Bloom filter window overlaps with startup time." + }]. +ticket_reuse_anti_replay_server_restart_reused_seed(Config) when is_list(Config) -> + WindowSize = 10, + Seed = crypto:strong_rand_bytes(32), + Config1 = [{server_ticket_seed, Seed} | Config], + {Server1 , Port1} = anti_replay_helper_start_server(Config1, WindowSize), + {ClientNode, _ServerNode, Hostname} = ssl_test_lib:run_where(Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ClientOpts1 = [{session_tickets, manual}, + {versions, ['tlsv1.2','tlsv1.3']} | ClientOpts0], + Client1 = ssl_test_lib:start_client([{node, ClientNode}, + {port, Port1}, {host, Hostname}, + {mfa, {ssl_test_lib, %% full handshake + verify_active_session_resumption, + [false, wait_reply, {tickets, 1}]}}, + {from, self()}, {options, ClientOpts1}]), + [Ticket] = ssl_test_lib:check_tickets(Client1), + ssl_test_lib:check_result(Server1, ok), + ClientOpts2 = [{use_ticket, [Ticket]} | ClientOpts1], + {Server2, Port2} = anti_replay_helper_start_server(Config1, WindowSize), + Client2 = anti_replay_helper_connect(Server2, Client1, Port2, ClientNode, + Hostname, ClientOpts2, 0, false, false), + process_flag(trap_exit, false), + [ssl_test_lib:close(A) || A <- [Server1, Client2, Server2]]. + anti_replay_helper_init(Config, Mode, WindowSize) -> DefaultLifetime = ssl_config:get_ticket_lifetime(), anti_replay_helper_init(Config, Mode, WindowSize, DefaultLifetime). @@ -360,9 +396,17 @@ anti_replay_helper_start_server(Config, WindowSize) -> {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), ServerTicketMode = proplists:get_value(server_ticket_mode, Config), + ServerTicketSeed = + case proplists:get_value(server_ticket_seed, Config) of + undefined -> + []; + Seed -> + [{stateless_tickets_seed, Seed}] + end, ServerOpts = [{session_tickets, ServerTicketMode}, {anti_replay, {WindowSize, 5, 72985}}, - {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0 + ] ++ ServerTicketSeed, Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1227,6 +1271,69 @@ early_data_basic_auth(Config) when is_list(Config) -> ssl_test_lib:close(Server0), ssl_test_lib:close(Client1). +stateless_multiple_servers() -> + [{doc, "Test session resumption with session tickets, resuming on different server"}]. +stateless_multiple_servers(Config) when is_list(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Seed = crypto:strong_rand_bytes(64), + + %% Configure session tickets + ClientOpts = [{session_tickets, auto}, + {versions, ['tlsv1.2','tlsv1.3']} | ClientOpts0], + ServerOpts = [{session_tickets, stateless}, + {stateless_tickets_seed, Seed}, + {versions, ['tlsv1.2','tlsv1.3']} | ServerOpts0], + + Server0 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + verify_active_session_resumption, + [false]}}, + {options, ServerOpts}]), + Port0 = ssl_test_lib:inet_port(Server0), + + Server1 = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {options, ServerOpts}]), + Port1 = ssl_test_lib:inet_port(Server1), + + %% Store ticket from first connection to server 0 + Client0 = + ssl_test_lib:start_client([{node, ClientNode}, + {port, Port0}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Full handshake + verify_active_session_resumption, + [false]}}, + {from, self()}, {options, ClientOpts}]), + ssl_test_lib:check_result(Server0, ok, Client0, ok), + + %% Wait for session ticket + ct:sleep(100), + + ssl_test_lib:close(Client0), + + %% Use ticket when connecting to server 1 + Client1 = + ssl_test_lib:start_client([{node, ClientNode}, + {port, Port1}, {host, Hostname}, + {mfa, {ssl_test_lib, %% Short handshake + verify_active_session_resumption, + [true]}}, + {from, self()}, {options, ClientOpts}]), + ssl_test_lib:check_result(Server1, ok, Client1, ok), + + process_flag(trap_exit, false), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Server1), + ssl_test_lib:close(Client1). %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index a190053b37..a29d5944ef 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1402,7 +1402,9 @@ format_subject({rdnSequence, Seq}) -> format_subject([[{'AttributeTypeAndValue', ?'id-at-commonName', {_, String}}]|_]) -> String; format_subject([_|R]) -> - format_subject(R). + format_subject(R); +format_subject([]) -> + "no commonname". cert_options(Config) -> ClientCaCertFile = filename:join([proplists:get_value(priv_dir, Config), @@ -2818,6 +2820,8 @@ init_protocol_version(Version, Config) -> [{protocol, tls} | NewConfig]. clean_protocol_version(Config) -> + application:unset_env(ssl, protocol_version), + application:unset_env(ssl, dtls_protocol_version), proplists:delete(version, proplists:delete(protocol_opts, proplists:delete(protocol, Config))). sufficient_crypto_support(Version) diff --git a/lib/ssl/test/tls_server_session_ticket_SUITE.erl b/lib/ssl/test/tls_server_session_ticket_SUITE.erl index 954afa9fb5..ee6a87fa2f 100644 --- a/lib/ssl/test/tls_server_session_ticket_SUITE.erl +++ b/lib/ssl/test/tls_server_session_ticket_SUITE.erl @@ -43,14 +43,17 @@ main_test/0, main_test/1, misc_test/0, - misc_test/1]). + misc_test/1, + valid_ticket_older_than_windowsize_test/0, + valid_ticket_older_than_windowsize_test/1]). --define(LIFETIME, 1). % tickets expire after 1s +-define(LIFETIME, 3). % tickets expire after 3s -define(TICKET_STORE_SIZE, 1). -define(MASTER_SECRET, "master_secret"). -define(PRF, sha). -define(VERSION, {3,4}). -define(PSK, <<15,168,18,43,216,33,227,142,114,190,70,183,137,57,64,64,66,152,115,94>>). +-define(WINDOW_SIZE, 1). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -61,7 +64,7 @@ all() -> groups() -> [{stateful, [], [main_test, expired_ticket_test, invalid_ticket_test]}, {stateless, [], [expired_ticket_test, invalid_ticket_test, main_test]}, - {stateless_antireplay, [], [main_test, misc_test]} + {stateless_antireplay, [], [main_test, misc_test, valid_ticket_older_than_windowsize_test]} ]. init_per_suite(Config0) -> @@ -80,7 +83,7 @@ end_per_suite(_Config) -> init_per_group(stateless_antireplay, Config) -> check_environment([{server_session_tickets, stateless}, - {anti_replay, {10, 20, 30}}] + {anti_replay, {?WINDOW_SIZE, 20, 30}}] ++ Config); init_per_group(Group = stateless, Config) -> check_environment([{server_session_tickets, Group} | Config]); @@ -167,6 +170,29 @@ expired_ticket_test(Config) when is_list(Config) -> [iolist_to_binary(HandshakeHist)]), true = is_process_alive(Pid). +valid_ticket_older_than_windowsize_test() -> + [{doc, "Verify valid ticket handling of tickets older than WindowSize"}]. + +valid_ticket_older_than_windowsize_test(Config) when is_list(Config) -> + Pid = ?config(server_pid, Config), + % Fill in GB tree store for stateful setup (Stateless tests also fail without this) + tls_server_session_ticket:new(Pid, ?PRF, ?MASTER_SECRET), + % Reach ticket store size limit - force GB tree pruning + SessionTicket = #new_session_ticket{} = + tls_server_session_ticket:new(Pid, ?PRF, ?MASTER_SECRET), + TicketRecvTime = erlang:system_time(millisecond), + %% Sleep more than the window length (which is in seconds) + ct:sleep({seconds, 2 * ?WINDOW_SIZE}), + {HandshakeHist, OferredPsks} = get_handshake_hist(SessionTicket, TicketRecvTime, ?PSK), + AcceptResponse = {ok, {0, ?PSK}}, + AcceptResponse = tls_server_session_ticket:use(Pid, OferredPsks, ?PRF, + [iolist_to_binary(HandshakeHist)]), + % check replay attempt result + ExpReplyResult = get_replay_expected_result(Config, AcceptResponse), + ExpReplyResult = tls_server_session_ticket:use(Pid, OferredPsks, ?PRF, + [iolist_to_binary(HandshakeHist)]), + true = is_process_alive(Pid). + misc_test() -> [{doc, "Miscellaneous functionality"}]. misc_test(Config) when is_list(Config) -> diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index 8cd2ceb53c..5b1bc2b483 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -43,6 +43,7 @@ XML_REF3_FILES = \ dict.xml \ digraph.xml \ digraph_utils.xml \ + edlin_expand.xml \ epp.xml \ erl_anno.xml \ erl_error.xmlsrc \ diff --git a/lib/stdlib/doc/src/base64.xml b/lib/stdlib/doc/src/base64.xml index bb45927c3f..b58e7394f2 100644 --- a/lib/stdlib/doc/src/base64.xml +++ b/lib/stdlib/doc/src/base64.xml @@ -40,7 +40,17 @@ <datatypes> <datatype> <name name="base64_alphabet"/> - <desc><p>Base 64 Encoding alphabet, see <url href="https://www.ietf.org/rfc/rfc4648.txt">RFC 4648</url>.</p> + <desc><p>Base 64 Encoding alphabet, see + <url href="https://datatracker.ietf.org/doc/html/rfc4648">RFC 4648</url>.</p> + </desc> + </datatype> + <datatype> + <name name="base64_mode"/> + <desc> + <p>Selector for the Base 64 Encoding alphabet used for encoding and decoding, + see <url href="https://datatracker.ietf.org/doc/html/rfc4648">RFC 4648</url> + Sections <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">4</url> + and <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">5</url>.</p> </desc> </datatype> <datatype> @@ -67,12 +77,13 @@ <name name="mime_decode" arity="1" since=""/> <name name="mime_decode_to_string" arity="1" since=""/> <fsummary>Decode a base64 encoded string to data.</fsummary> - <type variable="Base64" name_i="1"/> + <type variable="Base64"/> <type variable="Data" name_i="1"/> <type variable="DataString" name_i="2"/> <desc> - <p>Decodes a base64-encoded string to plain ASCII. See - <url href="https://www.ietf.org/html/rfc4648">RFC 4648</url>.</p> + <p>Decodes a base64 string encoded using the standard alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url> to plain ASCII.</p> <p><c>mime_decode/1</c> and <c>mime_decode_to_string/1</c> strip away illegal characters, while <c>decode/1</c> and <c>decode_to_string/1</c> only strip away whitespace characters.</p> @@ -80,6 +91,41 @@ </func> <func> + <name name="decode" arity="2" since="OTP @OTP-18247@"/> + <name name="decode_to_string" arity="2" since="OTP @OTP-18247@"/> + <name name="mime_decode" arity="2" since="OTP @OTP-18247@"/> + <name name="mime_decode_to_string" arity="2" since="OTP @OTP-18247@"/> + <fsummary>Decode a base64 encoded string to data.</fsummary> + <type variable="Base64"/> + <type variable="Mode" name_i="1"/> + <type variable="Data" name_i="1"/> + <type variable="DataString" name_i="2"/> + <desc> + <p>Decodes a base64 string encoded using the alphabet indicated by the + <c><anno>Mode</anno></c> parameter to plain ASCII.</p> + <p><c>mime_decode/2</c> and <c>mime_decode_to_string/2</c> strip away + illegal characters, while <c>decode/2</c> and + <c>decode_to_string/2</c> only strip away whitespace characters.</p> + <p>The <c><anno>Mode</anno></c> parameter can be one of the following:</p> + <taglist> + <tag><c>standard</c></tag> + <item>Decode the given string using the standard base64 alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>, that is <c>"+"</c> and <c>"/"</c> are representing bytes <c>62</c> + and <c>63</c> respectively, while <c>"-"</c> and <c>"_"</c> are illegal + characters.</item> + <tag><c>urlsafe</c></tag> + <item>Decode the given string using the alternative "URL and Filename safe" base64 + alphabet according to + <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">RFC 4648 + Section 5</url>, that is <c>"-"</c> and <c>"_"</c> are representing bytes <c>62</c> + and <c>63</c> respectively, while <c>"+"</c> and <c>"/"</c> are illegal + characters.</item> + </taglist> + </desc> + </func> + + <func> <name name="encode" arity="1" since=""/> <name name="encode_to_string" arity="1" since=""/> <fsummary>Encode data into base64.</fsummary> @@ -87,8 +133,35 @@ <type variable="Base64" name_i="1"/> <type variable="Base64String"/> <desc> - <p>Encodes a plain ASCII string into base64. The result is 33% larger - than the data.</p> + <p>Encodes a plain ASCII string into base64 using the standard alphabet + according to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>. The result is 33% larger than the data.</p> + </desc> + </func> + + <func> + <name name="encode" arity="2" since="OTP @OTP-18247@"/> + <name name="encode_to_string" arity="2" since="OTP @OTP-18247@"/> + <fsummary>Encode data into base64.</fsummary> + <type variable="Data"/> + <type variable="Mode"/> + <type variable="Base64" name_i="1"/> + <type variable="Base64String"/> + <desc> + <p>Encodes a plain ASCII string into base64 using the alphabet indicated by + the <c><anno>Mode</anno></c> parameter. The result is 33% larger than the data.</p> + <p>The <c><anno>Mode</anno></c> parameter can be one of the following:</p> + <taglist> + <tag><c>standard</c></tag> + <item>Encode the given string using the standard base64 alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>.</item> + <tag><c>urlsafe</c></tag> + <item>Encode the given string using the alternative "URL and Filename safe" base64 + alphabet according to + <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">RFC 4648 + Section 5</url>.</item> + </taglist> </desc> </func> </funcs> diff --git a/lib/stdlib/doc/src/edlin_expand.xml b/lib/stdlib/doc/src/edlin_expand.xml new file mode 100644 index 0000000000..0240f02f9f --- /dev/null +++ b/lib/stdlib/doc/src/edlin_expand.xml @@ -0,0 +1,117 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>1996</year><year>2022</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>edlin_expand</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <module since="OTP @OTP-14835@">edlin_expand</module> + <modulesummary>Shell expansion and formatting of expansion suggestions.</modulesummary> + <description> + <p>This module provides an expand_fun for the erlang shell + <seemfa marker="#expand/1"><c>expand/1,2</c></seemfa>. + It is possible to override this expand_fun + <seemfa marker="io#setopts/1"><c>io:setopts/1,2</c></seemfa>.</p> + </description> + <funcs> + <func> + <name name="expand" arity="1" since="OTP @OTP-14835@"/> + <name name="expand" arity="2" since="OTP @OTP-14835@"/> + <fsummary>Standard expanion function for the erl shell.</fsummary> + <desc> + <p>The standard expansion function is able to expand strings to + valid erlang terms. This includes module names:</p> + <pre> +1> erla +modules +erlang: + </pre> + <p>function names:</p> + <pre> +1> is_ato +functions +is_atom( +2> erlang:is_ato +functions +is_atom( + </pre> +<p> + function types: +</p> +<pre> +1> erlang:is_atom( +typespecs +erlang:is_atom(Term) +any() +</pre> +<p> + and automatically add , or closing parenthesis when no other + valid expansion is possible. The expand function also completes: + shell bindings, record names, record fields and map keys. +</p> +<p> + As seen below, function headers are grouped together if they've got the same + expansion suggestion, in this case all had the same suggestions, that is '}'. + There is also limited support for filtering out function typespecs that that does + not match the types on the terms on the prompt. Only 4 suggestions are shown below + but there exists plenty more typespecs for <c>erlang:system_info</c>. + </p> +<pre> +1> erlang:system_info({allocator, my_allocator +typespecs +erlang:system_info(wordsize | {wordsize, ...} | {wordsize, ...}) +erlang:system_info({allocator, ...}) +erlang:system_info({allocator_sizes, ...}) +erlang:system_info({cpu_topology, ...}) +} +</pre> + <p>The return type of <c>expand</c> function specifies either a list of <c>Element</c> + tuples or a list of <c>Section</c> maps. The section concept was introduced to enable + more formatting options for the expansion results. For example, the shell expansion has + support to highlight text and hide suggestions. + There are also a <c>{highlight, Text}</c> that highlights all occurances of + <c>Text</c> in the title, and a <c>highlight_all</c> for simplicity which + highlights the whole title, as can be seen above for <c>functions</c> and <c>typespecs</c>.</p> + + <p>By setting the <c>{hide, result}</c> or <c>{hide, title}</c> options you may hide + suggestions. Sometimes the title isn't useful and just produces text noise, in the example + above the <c>any()</c> result is part of a section with title <c>Types</c>. Hiding results + is currently not in use, but the idea is that a section can be selected in the expand area + and all the other section entries should be collapsed. + </p> + + <p>Its possible to set a custom separator between the title and the results. This can be + done with <c>{separator, Separator}</c>. + By default its set to be <c>\n</c>, some results display a <c>type_name() :: </c> + followed by all types that define <c>type_name()</c>. + </p> + + <p>The <c>{ending, Text}</c> ElementOption just appends Text to the <c>Element</c>. + </p> + </desc> + </func> + </funcs> +</erlref> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 507d27d8d6..e323713668 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -956,6 +956,32 @@ Error: fun containing local Erlang function calls </func> <func> + <name name="lookup_element" arity="4" since="OTP @OTP-18279@"/> + <fsummary>Return the <c>Pos</c>:th element of all objects with a + specified key in an ETS table, or <c>Default</c> if there is no such object.</fsummary> + <desc> + <p>For a table <c><anno>Table</anno></c> of type <c>set</c> or + <c>ordered_set</c>, the function returns the + <c><anno>Pos</anno></c>:th + element of the object with key <c><anno>Key</anno></c>.</p> + <p>For tables of type <c>bag</c> or <c>duplicate_bag</c>, + the functions returns a list with the <c><anno>Pos</anno></c>:th + element of every object with key <c><anno>Key</anno></c>.</p> + <p>If no object with key <c><anno>Key</anno></c> exists, the + function returns <c><anno>Default</anno></c>.</p> + <p>If <c><anno>Pos</anno></c> is larger than the size of + any tuple with a matching key, the function exits with + reason <c>badarg</c>.</p> + <p>The difference between <c>set</c>, <c>bag</c>, and + <c>duplicate_bag</c> on one hand, and <c>ordered_set</c> on + the other, regarding the fact that <c>ordered_set</c> + view keys as equal when they <em>compare equal</em> + whereas the other table types regard them equal only when + they <em>match</em>, holds for <c>lookup_element/4</c>.</p> + </desc> + </func> + + <func> <name name="match" arity="1" since=""/> <fsummary>Continues matching objects in an ETS table.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/io.xml b/lib/stdlib/doc/src/io.xml index a400d2af23..faadbcc574 100644 --- a/lib/stdlib/doc/src/io.xml +++ b/lib/stdlib/doc/src/io.xml @@ -83,6 +83,9 @@ <datatype> <name name="opt_pair"/> </datatype> + <datatype> + <name name="get_opt_pair"/> + </datatype> <datatype> <name name="expand_fun"/> </datatype> @@ -779,9 +782,14 @@ enter><input>:</input> <input>alan</input> <input>:</input> <input>joe</in [{expand_fun,#Fun<group.0.120017273>}, {echo,true}, {binary,false}, - {encoding,unicode}]</pre> + {encoding,unicode}, + {terminal,true}]</pre> <p>This example is, as can be seen, run in an environment where the terminal supports Unicode input and output.</p> + <p>The <c>terminal</c> option is read only and indicates whether + the output stream is a terminal or not. + See <seemfa marker="#setopts/1"><c>setopts/1</c></seemfa> for a description + of the other options.</p> </desc> </func> @@ -1135,13 +1143,18 @@ enter><input>1.0er.</input> <seemfa marker="#get_line/1"><c>get_line/1,2</c></seemfa>.</p> <p>The function is called with the current line, up to the cursor, as a reversed string. It is to return a - three-tuple: <c>{yes|no, string(), [string(), ...]}</c>. The + three-tuple: <c>{yes|no, string(), list()}</c>. The first element gives a beep if <c>no</c>, otherwise the expansion is silent; the second is a string that will be entered at the cursor position; the third is a list of possible expansions. If this list is not empty, - it is printed and the current input line is written - once again.</p> + it is printed below the current input line. + The list of possible expansions can be formatted in + different ways to make more advanced expansion suggestions + more readable to the user, see + <seemfa marker="edlin_expand#expand/2"> + <c>edlin_expand:expand/2</c></seemfa> for + documentation of that.</p> <p>Trivial example (beep on anything except empty line, which is expanded to <c>"quit"</c>):</p> <code type="none"> diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index 98b764ab70..ce9fbc230a 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -87,7 +87,7 @@ returns <c>true</c>, the association is copied to the result map. If it returns <c>false</c>, the association is not copied. If it returns <c>{true, NewValue}</c>, the value for <c><anno>Key</anno></c> is - replaced with <c>NewValue</c>at this position is replaced in the + replaced with <c>NewValue</c> at this position is replaced in the result map.</p> <p>The call fails with a <c>{badmap,Map}</c> exception if <c><anno>MapOrIter</anno></c> is not a map or valid iterator, diff --git a/lib/stdlib/doc/src/peer.xml b/lib/stdlib/doc/src/peer.xml index d8ac800605..71bf71e75e 100644 --- a/lib/stdlib/doc/src/peer.xml +++ b/lib/stdlib/doc/src/peer.xml @@ -99,60 +99,60 @@ of the same test suite running in parallel</item> </list> <code type="erl"> - -module(my_SUITE). - -behaviour(ct_suite). - -export([all/0, groups/0]). - -export([basic/1, args/1, named/1, restart_node/1, multi_node/1]). - - -include_lib("common_test/include/ct.hrl"). - - groups() -> - [{quick, [parallel], - [basic, args, named, restart_node, multi_node]}]. - - all() -> - [{group, quick}]. - - basic(Config) when is_list(Config) -> - {ok, Peer, _Node} = ?CT_PEER(), - peer:stop(Peer). - - args(Config) when is_list(Config) -> - %% specify additional arguments to the new node - {ok, Peer, _Node} = ?CT_PEER(["-emu_flavor", "smp"]), - peer:stop(Peer). - - named(Config) when is_list(Config) -> - %% pass test case name down to function starting nodes - Peer = start_node_impl(named_test), - peer:stop(Peer). - - start_node_impl(ActualTestCase) -> - {ok, Peer, Node} = ?CT_PEER(#{name => ?CT_PEER_NAME(ActualTestCase)}), - %% extra setup needed for multiple test cases - ok = rpc:call(Node, application, set_env, [kernel, key, value]), - Peer. - - restart_node(Config) when is_list(Config) -> - Name = ?CT_PEER_NAME(), - {ok, Peer, Node} = ?CT_PEER(#{name => Name}), - peer:stop(Peer), - %% restart the node with the same name as before - {ok, Peer2, Node} = ?CT_PEER(#{name => Name, args => ["+fnl"]}), - peer:stop(Peer2). +-module(my_SUITE). +-behaviour(ct_suite). +-export([all/0, groups/0]). +-export([basic/1, args/1, named/1, restart_node/1, multi_node/1]). + +-include_lib("common_test/include/ct.hrl"). + +groups() -> + [{quick, [parallel], + [basic, args, named, restart_node, multi_node]}]. + +all() -> + [{group, quick}]. + +basic(Config) when is_list(Config) -> + {ok, Peer, _Node} = ?CT_PEER(), + peer:stop(Peer). + +args(Config) when is_list(Config) -> + %% specify additional arguments to the new node + {ok, Peer, _Node} = ?CT_PEER(["-emu_flavor", "smp"]), + peer:stop(Peer). + +named(Config) when is_list(Config) -> + %% pass test case name down to function starting nodes + Peer = start_node_impl(named_test), + peer:stop(Peer). + +start_node_impl(ActualTestCase) -> + {ok, Peer, Node} = ?CT_PEER(#{name => ?CT_PEER_NAME(ActualTestCase)}), + %% extra setup needed for multiple test cases + ok = rpc:call(Node, application, set_env, [kernel, key, value]), + Peer. + +restart_node(Config) when is_list(Config) -> + Name = ?CT_PEER_NAME(), + {ok, Peer, Node} = ?CT_PEER(#{name => Name}), + peer:stop(Peer), + %% restart the node with the same name as before + {ok, Peer2, Node} = ?CT_PEER(#{name => Name, args => ["+fnl"]}), + peer:stop(Peer2). </code> <p> The next example demonstrates how to start multiple nodes concurrently: </p> <code type="erl"> - multi_node(Config) when is_list(Config) -> - Peers = [?CT_PEER(#{wait_boot => {self(), tag}}) - || _ <- lists:seq(1, 4)], - %% wait for all nodes to complete boot process, get their names: - _Nodes = [receive {tag, {started, Node, Peer}} -> Node end - || {ok, Peer} <- Peers], - [peer:stop(Peer) || {ok, Peer} <- Peers]. +multi_node(Config) when is_list(Config) -> + Peers = [?CT_PEER(#{wait_boot => {self(), tag}}) + || _ <- lists:seq(1, 4)], + %% wait for all nodes to complete boot process, get their names: + _Nodes = [receive {tag, {started, Node, Peer}} -> Node end + || {ok, Peer} <- Peers], + [peer:stop(Peer) || {ok, Peer} <- Peers]. </code> <p> @@ -161,9 +161,9 @@ prompt. </p> <code type="erl"> - Ssh = os:find_executable("ssh"), - peer:start_link(#{exec => {Ssh, ["another_host", "erl"]}, - connection => standard_io}), +Ssh = os:find_executable("ssh"), +peer:start_link(#{exec => {Ssh, ["another_host", "erl"]}, + connection => standard_io}), </code> <p> @@ -172,76 +172,76 @@ running inside containers form an Erlang cluster. </p> <code type="erl"> - docker(Config) when is_list(Config) -> - Docker = os:find_executable("docker"), - PrivDir = proplists:get_value(priv_dir, Config), - build_release(PrivDir), - build_image(PrivDir), - - %% start two Docker containers - {ok, Peer, Node} = peer:start_link(#{name => lambda, - connection => standard_io, - exec => {Docker, ["run", "-h", "one", "-i", "lambda"]}}), - {ok, Peer2, Node2} = peer:start_link(#{name => lambda, - connection => standard_io, - exec => {Docker, ["run", "-h", "two", "-i", "lambda"]}}), - - %% find IP address of the second node using alternative connection RPC - {ok, Ips} = peer:call(Peer2, inet, getifaddrs, []), - {"eth0", Eth0} = lists:keyfind("eth0", 1, Ips), - {addr, Ip} = lists:keyfind(addr, 1, Eth0), - - %% make first node to discover second one - ok = peer:call(Peer, inet_db, set_lookup, [[file]]), - ok = peer:call(Peer, inet_db, add_host, [Ip, ["two"]]), - - %% join a cluster - true = peer:call(Peer, net_kernel, connect_node, [Node2]), - %% verify that second peer node has only the first node visible - [Node] = peer:call(Peer2, erlang, nodes, []), - - %% stop peers, causing containers to also stop - peer:stop(Peer2), - peer:stop(Peer). - - build_release(Dir) -> - %% load sasl.app file, otherwise application:get_key will fail - application:load(sasl), - %% create *.rel - release file - RelFile = filename:join(Dir, "lambda.rel"), - Release = {release, {"lambda", "1.0.0"}, - {erts, erlang:system_info(version)}, - [{App, begin {ok, Vsn} = application:get_key(App, vsn), Vsn end} - || App <- [kernel, stdlib, sasl]]}, - ok = file:write_file(RelFile, list_to_binary(lists:flatten( - io_lib:format("~tp.", [Release])))), - RelFileNoExt = filename:join(Dir, "lambda"), - - %% create boot script - {ok, systools_make, []} = systools:make_script(RelFileNoExt, - [silent, {outdir, Dir}]), - %% package release into *.tar.gz - ok = systools:make_tar(RelFileNoExt, [{erts, code:root_dir()}]). - - build_image(Dir) -> - %% Create Dockerfile example, working only for Ubuntu 20.04 - %% Expose port 4445, and make Erlang distribution to listen - %% on this port, and connect to it without EPMD - %% Set cookie on both nodes to be the same. - BuildScript = filename:join(Dir, "Dockerfile"), - Dockerfile = - "FROM ubuntu:20.04 as runner\n" - "EXPOSE 4445\n" - "WORKDIR /opt/lambda\n" - "COPY lambda.tar.gz /tmp\n" - "RUN tar -zxvf /tmp/lambda.tar.gz -C /opt/lambda\n" - "ENTRYPOINT [\"/opt/lambda/erts-" ++ erlang:system_info(version) ++ - "/bin/dyn_erl\", \"-boot\", \"/opt/lambda/releases/1.0.0/start\"," - " \"-kernel\", \"inet_dist_listen_min\", \"4445\"," - " \"-erl_epmd_port\", \"4445\"," - " \"-setcookie\", \"secret\"]\n", - ok = file:write_file(BuildScript, Dockerfile), - os:cmd("docker build -t lambda " ++ Dir). +docker(Config) when is_list(Config) -> + Docker = os:find_executable("docker"), + PrivDir = proplists:get_value(priv_dir, Config), + build_release(PrivDir), + build_image(PrivDir), + + %% start two Docker containers + {ok, Peer, Node} = peer:start_link(#{name => lambda, + connection => standard_io, + exec => {Docker, ["run", "-h", "one", "-i", "lambda"]}}), + {ok, Peer2, Node2} = peer:start_link(#{name => lambda, + connection => standard_io, + exec => {Docker, ["run", "-h", "two", "-i", "lambda"]}}), + + %% find IP address of the second node using alternative connection RPC + {ok, Ips} = peer:call(Peer2, inet, getifaddrs, []), + {"eth0", Eth0} = lists:keyfind("eth0", 1, Ips), + {addr, Ip} = lists:keyfind(addr, 1, Eth0), + + %% make first node to discover second one + ok = peer:call(Peer, inet_db, set_lookup, [[file]]), + ok = peer:call(Peer, inet_db, add_host, [Ip, ["two"]]), + + %% join a cluster + true = peer:call(Peer, net_kernel, connect_node, [Node2]), + %% verify that second peer node has only the first node visible + [Node] = peer:call(Peer2, erlang, nodes, []), + + %% stop peers, causing containers to also stop + peer:stop(Peer2), + peer:stop(Peer). + +build_release(Dir) -> + %% load sasl.app file, otherwise application:get_key will fail + application:load(sasl), + %% create *.rel - release file + RelFile = filename:join(Dir, "lambda.rel"), + Release = {release, {"lambda", "1.0.0"}, + {erts, erlang:system_info(version)}, + [{App, begin {ok, Vsn} = application:get_key(App, vsn), Vsn end} + || App <- [kernel, stdlib, sasl]]}, + ok = file:write_file(RelFile, list_to_binary(lists:flatten( + io_lib:format("~tp.", [Release])))), + RelFileNoExt = filename:join(Dir, "lambda"), + + %% create boot script + {ok, systools_make, []} = systools:make_script(RelFileNoExt, + [silent, {outdir, Dir}]), + %% package release into *.tar.gz + ok = systools:make_tar(RelFileNoExt, [{erts, code:root_dir()}]). + +build_image(Dir) -> + %% Create Dockerfile example, working only for Ubuntu 20.04 + %% Expose port 4445, and make Erlang distribution to listen + %% on this port, and connect to it without EPMD + %% Set cookie on both nodes to be the same. + BuildScript = filename:join(Dir, "Dockerfile"), + Dockerfile = + "FROM ubuntu:20.04 as runner\n" + "EXPOSE 4445\n" + "WORKDIR /opt/lambda\n" + "COPY lambda.tar.gz /tmp\n" + "RUN tar -zxvf /tmp/lambda.tar.gz -C /opt/lambda\n" + "ENTRYPOINT [\"/opt/lambda/erts-" ++ erlang:system_info(version) ++ + "/bin/dyn_erl\", \"-boot\", \"/opt/lambda/releases/1.0.0/start\"," + " \"-kernel\", \"inet_dist_listen_min\", \"4445\"," + " \"-erl_epmd_port\", \"4445\"," + " \"-setcookie\", \"secret\"]\n", + ok = file:write_file(BuildScript, Dockerfile), + os:cmd("docker build -t lambda " ++ Dir). </code> </section> @@ -270,13 +270,6 @@ is, <c>peer</c> follows compatibility behaviour and uses the origin node name. </p> </item> - <tag><c>host</c></tag> - <item> - <p> - Enforces a specific host name. Can be used to override the default - behaviour and start "node@localhost" instead of "node@realhostname". - </p> - </item> <tag><c>longnames</c></tag> <item> <p> @@ -285,6 +278,13 @@ short names is the default. </p> </item> + <tag><c>host</c></tag> + <item> + <p> + Enforces a specific host name. Can be used to override the default + behaviour and start "node@localhost" instead of "node@realhostname". + </p> + </item> <tag><c>peer_down</c></tag> <item> <p> @@ -296,6 +296,11 @@ the controlling process to exit abnormally. </p> </item> + <tag><c>connection</c></tag> + <item> + <p>Alternative connection specification. See the + <seetype marker="#connection"><c>connection</c> datatype</seetype>.</p> + </item> <tag><c>exec</c></tag> <item> <p> @@ -303,16 +308,32 @@ default bash. </p> </item> - <tag><c>connection</c></tag> + <tag><c>detached</c></tag> <item> - <p>Alternative connection specification. See the - <seetype marker="#connection"><c>connection</c> datatype</seetype>.</p> + <p>Defines whether to pass the <c>-detached</c> flag to the started peer. + This option cannot be set to <c>false</c> using the standard_io alternative + connection type. Default is <c>true</c>. + </p> </item> <tag><c>args</c></tag> <item> <p>Extra command line arguments to append to the "erl" command. Arguments are passed as is, no escaping or quoting is needed or accepted.</p> </item> + <tag><c>post_process_args</c></tag> + <item> + <p>Allows the user to change the arguments passed to <c>exec</c> before the + peer is started. This can for example be useful when the <c>exec</c> program + wants the arguments to "erl" as a single argument. Example: + </p> + <code type="erl"> +peer:start(#{ name => peer:random_name(), + exec => {os:find_executable("bash"),["-c","erl"]}, + post_process_args => + fun(["-c"|Args]) -> ["-c", lists:flatten(lists:join($\s, Args))] end + }). + </code> + </item> <tag><c>env</c></tag> <item> <p> diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index e16ef12f16..43bdb142f4 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -75,6 +75,9 @@ <datatype> <name name="compile_option"/> </datatype> + <datatype> + <name name="replace_fun"/> + </datatype> </datatypes> <funcs> @@ -363,7 +366,7 @@ elements with Replacement.</fsummary> <desc> <p>Replaces the matched part of the <c><anno>Subject</anno></c> string - with the contents of <c><anno>Replacement</anno></c>.</p> + with <c><anno>Replacement</anno></c>.</p> <p>The permissible options are the same as for <seemfa marker="#run/3"><c>run/3</c></seemfa>, except that option<c> capture</c> is not allowed. Instead a <c>{return, @@ -378,8 +381,8 @@ <c>unicode</c> compilation option is specified to this function, both the regular expression and <c><anno>Subject</anno></c> are to specified as valid Unicode <c>charlist()</c>s.</p> - <p>The replacement string can contain the special character - <c>&</c>, which inserts the whole matching expression in the + <p>If the replacement is given as a string, it can contain the special + character <c>&</c>, which inserts the whole matching expression in the result, and the special sequence <c>\</c>N (where N is an integer > 0), <c>\g</c>N, or <c>\g{</c>N<c>}</c>, resulting in the subexpression number N, is inserted in the result. If no subexpression with that @@ -401,6 +404,35 @@ re:replace("abcd","c","[\\&]",[{return,list}]).</code> <p>gives</p> <code> "ab[&]d"</code> + <p>If the replacement is given as a fun, it will be called with the + whole matching expression as the first argument and a list of subexpression + matches in the order in which they appear in the regular expression. + The returned value will be inserted in the result.</p> + <p><em>Example:</em></p> + <code> +re:replace("abcd", ".(.)", fun(Whole, [<<C>>]) -> <<$#, Whole/binary, $-, (C - $a + $A), $#>> end, [{return, list}]).</code> + <p>gives</p> + <code> +"#ab-B#cd"</code> + <note> + <p>Non-matching optional subexpressions will not be included in the list + of subexpression matches if they are the last subexpressions in the + regular expression.</p> + <p><em>Example:</em></p> + <p>The regular expression <c>"(a)(b)?(c)?"</c> ("a", optionally followed + by "b", optionally followed by "c") will create the following subexpression + lists:</p> + <list> + <item><c>[<<"a">>, <<"b">>, <<"c">>]</c> + when applied to the string <c>"abc"</c></item> + <item><c>[<<"a">>, <<>>, <<"c">>]</c> + when applied to the string <c>"acx"</c></item> + <item><c>[<<"a">>, <<"b">>]</c> + when applied to the string <c>"abx"</c></item> + <item><c>[<<"a">>]</c> + when applied to the string <c>"axx"</c></item> + </list> + </note> <p>As with <c>run/3</c>, compilation errors raise the <c>badarg</c> exception. <seemfa marker="#compile/2"><c>compile/2</c></seemfa> can be used to get more information about the error.</p> @@ -972,7 +1004,7 @@ re:run("ABCabcdABC",".*(?<FOO>abcd).*",[{capture,['FOO']}]).</code> <p>Here the empty binary (<c><<>></c>) represents the unassigned subpattern. In the <c>binary</c> case, some information about the matching is therefore lost, as - <c><<>></c> can + <c><<>></c> can also be an empty string captured.</p> <p>If differentiation between empty matches and non-existing subpatterns is necessary, use the <c>type</c> <c>index</c> and do diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index e63c455ec8..961c5a0a77 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -43,6 +43,7 @@ <xi:include href="dict.xml"/> <xi:include href="digraph.xml"/> <xi:include href="digraph_utils.xml"/> + <xi:include href="edlin_expand.xml"/> <xi:include href="epp.xml"/> <xi:include href="erl_anno.xml"/> <xi:include href="erl_error.xml"/> diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml index 928d2686b6..e33db34b28 100644 --- a/lib/stdlib/doc/src/shell.xml +++ b/lib/stdlib/doc/src/shell.xml @@ -957,6 +957,85 @@ q - quit erlang </func> <func> + <name name="start_interactive" arity="0" since="OTP @OTP-17932@"/> + <fsummary>Start the interactive shell</fsummary> + <desc> + <p>Starts the interactive shell if it has not already been started. + It can be used to programatically start the shell from an escript + or when erl is started with the -noinput or -noshell flags.</p> + </desc> + </func> + + <func> + <name name="start_interactive" arity="1" clause_i="1" since="OTP @OTP-17932@"/> + <name name="start_interactive" arity="1" clause_i="2" since="OTP @OTP-17932@"/> + <name name="start_interactive" arity="1" clause_i="3" since="OTP @OTP-17932@"/> + <fsummary>Start the interactive shell</fsummary> + <desc> + <p>Starts the interactive shell if it has not already been started. + It can be used to programatically start the shell from an + <seecom marker="erts:escript"><c>escript</c></seecom> or when + <seecom marker="erts:erl"><c>erl</c></seecom> is started with the + <seecom marker="erts:erl#noinput"><c>-noinput</c></seecom> or + <seecom marker="erts:erl#noshell"><c>-noshell</c></seecom> flags. + The following options are allowed:</p> + <taglist> + <tag>noshell</tag> + <item> + <p>Starts the interactive shell as if <seecom marker="erts:erl#noshell"> + <c>-noshell</c></seecom> was given to <seecom marker="erts:erl"><c>erl</c></seecom>. + This is only useful when erl is started with + <seecom marker="erts:erl#noinput"><c>-noinput</c></seecom> and the + system want to read input data. + </p> + </item> + <tag><seetype marker="erts:erlang#mfa">mfa()</seetype></tag> + <item> + <p>Starts the interactive shell using + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype> + as the default shell.</p> + </item> + <tag>{<seetype marker="erts:erlang#node">node()</seetype>, + <seetype marker="erts:erlang#mfa">mfa()</seetype>}</tag> + <item> + <p>Starts the interactive shell using + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype> on + <seetype marker="erts:erlang#node"><c>node()</c></seetype> as the default shell.</p> + </item> + <tag>{remote, <seetype marker="erts:erlang#string"><c>string()</c></seetype>}</tag> + <item> + <p>Starts the interactive shell using as if + <seecom marker="erts:erl#remsh"><c>-remsh</c></seecom> + was given to <seecom marker="erts:erl"><c>erl</c></seecom>.</p> + </item> + <tag>{remote, + <seetype marker="erts:erlang#string"><c>string()</c></seetype>, + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype>}</tag> + <item> + <p>Starts the interactive shell using as if + <seecom marker="erts:erl#remsh"><c>-remsh</c></seecom> + was given to <seecom marker="erts:erl"><c>erl</c></seecom> + but with an alternative shell implementation.</p> + </item> + </taglist> + <p>On error this function will return:</p> + <taglist> + <tag>already_started</tag> + <item>if an interactive shell is already started.</item> + <tag>noconnection</tag> + <item>if a remote shell was requested but it could not be connected to.</item> + <tag>badfile | nofile | on_load_failure</tag> + <item>if a remote shell was requested with a custom + <seetype marker="erts:erlang#mfa">mfa()</seetype>, + but the module could not be loaded. See <seeerl marker="kernel:code#error_reasons"> + Error Reasons for Code-Loading Functions</seeerl> + for a description of the error reasons. + </item> + </taglist> + </desc> + </func> + + <func> <name name="start_restricted" arity="1" since=""/> <fsummary>Exit a normal shell and starts a restricted shell.</fsummary> <desc> @@ -994,6 +1073,17 @@ q - quit erlang string syntax.</p> </desc> </func> + + <func> + <name name="whereis" arity="0" since="OTP @OTP-17932@"/> + <fsummary>Return the current shell process.</fsummary> + <desc> + <p>Returns the current shell process on the node where the + calling process' group_leader is located. If that node + has no shell this function will return undefined.</p> + </desc> + </func> + </funcs> </erlref> diff --git a/lib/stdlib/doc/src/specs.xml b/lib/stdlib/doc/src/specs.xml index 9fdd0d1c21..8279c5a5d8 100644 --- a/lib/stdlib/doc/src/specs.xml +++ b/lib/stdlib/doc/src/specs.xml @@ -10,6 +10,7 @@ <xi:include href="../specs/specs_dict.xml"/> <xi:include href="../specs/specs_digraph.xml"/> <xi:include href="../specs/specs_digraph_utils.xml"/> + <xi:include href="../specs/specs_edlin_expand.xml"/> <xi:include href="../specs/specs_epp.xml"/> <xi:include href="../specs/specs_erl_anno.xml"/> <xi:include href="../specs/specs_erl_error.xml"/> diff --git a/lib/stdlib/doc/src/stdlib_app.xml b/lib/stdlib/doc/src/stdlib_app.xml index 243853140b..e0f50292d6 100644 --- a/lib/stdlib/doc/src/stdlib_app.xml +++ b/lib/stdlib/doc/src/stdlib_app.xml @@ -57,6 +57,11 @@ <p>Can be used to set the exception handling of the evaluator process of Erlang shell.</p> </item> + <tag><marker id="shell_expand_location"/><c>shell_expand_location = above | below</c></tag> + <item> + <p>Sets where the tab expansion text should appear in the shell. + The default is <c>below</c>.</p> + </item> <tag><marker id="shell_history_length"/><c>shell_history_length = integer() >= 0</c></tag> <item> <p>Can be used to determine how many commands are saved by the Erlang @@ -76,6 +81,29 @@ <p>Can be used to determine how many results are saved by the Erlang shell.</p> </item> + <tag><marker id="shell_session_slogan"/><c>shell_session_slogan = string() | fun() -> string())</c></tag> + <item> + <p>The slogan printed when starting an Erlang shell. Example: </p> + <code type="erl"> +$ erl -stdlib shell_session_slogan '"Test slogan"' +Erlang/OTP 26 [DEVELOPMENT] [erts-13.0.2] [source] [64-bit] [smp:8:8] [ds:8:8:10] [async-threads:1] [jit:ns] + +Test slogan +1> + </code> + </item> + <tag><marker id="shell_slogan"/><c>shell_slogan = string() | fun(() -> string())</c></tag> + <item> + <p>The slogan printed when starting the Erlang shell subsystem. Example: </p> + <code type="erl"> +$ erl -stdlib shell_slogan '"Test slogan"' +Test slogan +Eshell V13.0.2 (abort with ^G) +1> + </code> + <p>The default is the return value of <seeerl marker="erts:erlang#system_info_system_version"> + <c>erlang:system_info(system_version)</c></seeerl>.</p> + </item> <tag><marker id="shell_strings"/><c>shell_strings = boolean()</c></tag> <item> <p>Can be used to determine how the Erlang shell outputs lists of diff --git a/lib/stdlib/doc/src/timer.xml b/lib/stdlib/doc/src/timer.xml index 11279ff410..2cbf7977a9 100644 --- a/lib/stdlib/doc/src/timer.xml +++ b/lib/stdlib/doc/src/timer.xml @@ -71,10 +71,10 @@ <funcs> <func> <name name="apply_after" arity="4" since=""/> - <fsummary>Apply <c>Module:Function(Arguments)</c> after a specified - <c>Time</c>.</fsummary> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + after a specified <c>Time</c>.</fsummary> <desc> - <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> after <c><anno>Time</anno></c> milliseconds.</p> <p>Returns <c>{ok, <anno>TRef</anno>}</c> or @@ -84,12 +84,53 @@ <func> <name name="apply_interval" arity="4" since=""/> - <fsummary>Evaluate <c>Module:Function(Arguments)</c> repeatedly at - intervals of <c>Time</c>.</fsummary> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + repeatedly at intervals of <c>Time</c>.</fsummary> + <desc> + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, + <anno>Arguments</anno>)</c> repeatedly at intervals of + <c><anno>Time</anno></c>, irrespective of whether a previously + spawned process has finished or not.</p> + <warning> + <p>If the execution time of the spawned process is, on average, + greater than the given <c><anno>Time</anno></c>, multiple such + processes will run at the same time. With long execution times, + short intervals, and many interval timers running, this may even + lead to exceeding the number of allowed processes. As an extreme + example, consider + <c>[timer:apply_interval(1, timer, sleep, [1000]) || _ <- lists:seq(1, 1000)]</c>, + that is, 1,000 interval timers executing a process that takes 1s + to complete, started in intervals of 1ms, which would result in + 1,000,000 processes running at the same time, far more than a node + started with default settings allows (see the + <seeguide marker="system/efficiency_guide:advanced#system-limits">System + Limits section in the Effiency Guide</seeguide>).</p> + </warning> + <p>Returns <c>{ok, <anno>TRef</anno>}</c> or + <c>{error, <anno>Reason</anno>}</c>.</p> + </desc> + </func> + + <func> + <name name="apply_repeatedly" arity="4" since="OTP @OTP-18236@"/> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + repeatedly at intervals of <c>Time</c>.</fsummary> <desc> - <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> repeatedly at intervals of - <c><anno>Time</anno></c>.</p> + <c><anno>Time</anno></c>, waiting for the spawned process to + finish before starting the next.</p> + <p>If the execution time of the spawned process is greater than the + given <c><anno>Time</anno></c>, the next process is spawned immediately + after the one currently running has finished. Assuming that execution + times of the spawned processes performing the applies on average + are smaller than <c><anno>Time</anno></c>, the amount of applies + made over a large amount of time will be the same even if some + individual execution times are larger than + <c><anno>Time</anno></c>. The system will try to catch up as soon + as possible. For example, if one apply takes + <c>2.5*<anno>Time</anno></c>, the following two applies will be + made immediately one after the other in sequence.</p> <p>Returns <c>{ok, <anno>TRef</anno>}</c> or <c>{error, <anno>Reason</anno>}</c>.</p> </desc> diff --git a/lib/stdlib/doc/src/unicode.xml b/lib/stdlib/doc/src/unicode.xml index 1f0133de67..1ebd5de55a 100644 --- a/lib/stdlib/doc/src/unicode.xml +++ b/lib/stdlib/doc/src/unicode.xml @@ -37,7 +37,7 @@ representations. It converts between ISO Latin-1 characters and Unicode characters, but it can also convert between different Unicode encodings (like UTF-8, UTF-16, and UTF-32).</p> - <p>The default Unicode encoding in Erlang is in binaries UTF-8, which is also + <p>The default Unicode encoding in Erlang binaries is UTF-8, which is also the format in which built-in functions and libraries in OTP expect to find binary Unicode data. In lists, Unicode data is encoded as integers, each integer representing one character and encoded simply as the Unicode code diff --git a/lib/stdlib/doc/src/zip.xml b/lib/stdlib/doc/src/zip.xml index a056621ca1..03c28f5d3c 100644 --- a/lib/stdlib/doc/src/zip.xml +++ b/lib/stdlib/doc/src/zip.xml @@ -507,6 +507,14 @@ </func> <func> + <name name="zip_get_crc32" arity="2" since="OTP @OTP-18159@"/> + <fsummary>Extracts a crc32 checksum from an open archive.</fsummary> + <desc> + <p>Extracts one crc32 checksum from an open archive.</p> + </desc> + </func> + + <func> <name name="zip_list_dir" arity="1" since=""/> <fsummary>Return a table of files in open zip archive.</fsummary> <desc> diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 761d6c4c28..e546172856 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -56,7 +56,9 @@ MODULES= \ digraph \ digraph_utils \ edlin \ + edlin_context \ edlin_expand \ + edlin_type_suggestion \ epp \ erl_abstract_code \ erl_anno \ @@ -196,7 +198,8 @@ primary_bootstrap_compiler: \ $(BOOTSTRAP_COMPILER)/ebin/erl_parse.beam \ $(BOOTSTRAP_COMPILER)/ebin/erl_lint.beam \ $(BOOTSTRAP_COMPILER)/ebin/io.beam \ - $(BOOTSTRAP_COMPILER)/ebin/otp_internal.beam + $(BOOTSTRAP_COMPILER)/ebin/otp_internal.beam \ + $(BOOTSTRAP_COMPILER)/ebin/erl_internal.beam $(BOOTSTRAP_COMPILER)/ebin/erl_parse.beam: erl_parse.yrl diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 1504326c61..03dedabd55 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -462,7 +462,7 @@ fix_test_() -> -spec relax(Array :: array(Type)) -> array(Type). -relax(#array{size = N}=A) -> +relax(#array{size = N}=A) when is_integer(N), N >= 0 -> A#array{max = find_max(N-1, ?LEAFSIZE)}. @@ -489,7 +489,9 @@ relax_test_() -> array(Type). resize(Size, #array{size = N, max = M, elements = E}=A) - when is_integer(Size), Size >= 0 -> + when is_integer(Size), Size >= 0, + is_integer(N), N >= 0, + is_integer(M), M >= 0 -> if Size > N -> {E1, M1} = grow(Size-1, E, if M > 0 -> M; @@ -570,7 +572,7 @@ resize_test_() -> -spec set(I :: array_indx(), Value :: Type, Array :: array(Type)) -> array(Type). set(I, Value, #array{size = N, max = M, default = D, elements = E}=A) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> A#array{elements = set_1(I, E, Value, D)}; I < M -> @@ -599,7 +601,7 @@ set_1(I, E, X, _D) -> %% Enlarging the array upwards to accommodate an index `I' -grow(I, E, _M) when is_integer(E) -> +grow(I, E, _M) when is_integer(I), is_integer(E) -> M1 = find_max(I, E), {M1, M1}; grow(I, E, M) -> @@ -633,7 +635,7 @@ expand(I, _S, X, D) -> -spec get(I :: array_indx(), Array :: array(Type)) -> Value :: Type. get(I, #array{size = N, max = M, elements = E, default = D}) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> get_1(I, E, D); M > 0 -> @@ -673,7 +675,7 @@ get_1(I, E, _D) -> -spec reset(I :: array_indx(), Array :: array(Type)) -> array(Type). reset(I, #array{size = N, max = M, default = D, elements = E}=A) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> try A#array{elements = reset_1(I, E, D)} catch throw:default -> A @@ -760,7 +762,7 @@ set_get_test_() -> to_list(#array{size = 0}) -> []; -to_list(#array{size = N, elements = E, default = D}) -> +to_list(#array{size = N, elements = E, default = D}) when is_integer(N) -> to_list_1(E, D, N - 1); to_list(_) -> erlang:error(badarg). @@ -833,7 +835,7 @@ to_list_test_() -> sparse_to_list(#array{size = 0}) -> []; -sparse_to_list(#array{size = N, elements = E, default = D}) -> +sparse_to_list(#array{size = N, elements = E, default = D}) when is_integer(N) -> sparse_to_list_1(E, D, N - 1); sparse_to_list(_) -> erlang:error(badarg). @@ -1011,7 +1013,7 @@ from_list_test_() -> to_orddict(#array{size = 0}) -> []; -to_orddict(#array{size = N, elements = E, default = D}) -> +to_orddict(#array{size = N, elements = E, default = D}) when is_integer(N) -> I = N - 1, to_orddict_1(E, I, D, I); to_orddict(_) -> @@ -1030,7 +1032,7 @@ to_orddict_1(E, R, D, I) when is_integer(E) -> to_orddict_1(E, R, _D, I) -> push_tuple_pairs(I+1, R, E, []). -to_orddict_2(E=?NODEPATTERN(S), R, D, L) -> +to_orddict_2(E=?NODEPATTERN(S), R, D, L) when is_integer(S) -> to_orddict_3(?NODESIZE, R, D, L, E, S); to_orddict_2(E, R, D, L) when is_integer(E) -> push_pairs(E, R, D, L); @@ -1103,7 +1105,8 @@ to_orddict_test_() -> sparse_to_orddict(#array{size = 0}) -> []; -sparse_to_orddict(#array{size = N, elements = E, default = D}) -> +sparse_to_orddict(#array{size = N, elements = E, default = D}) + when is_integer(N) -> I = N - 1, sparse_to_orddict_1(E, I, D, I); sparse_to_orddict(_) -> @@ -1122,7 +1125,7 @@ sparse_to_orddict_1(E, _R, _D, _I) when is_integer(E) -> sparse_to_orddict_1(E, R, D, I) -> sparse_push_tuple_pairs(I+1, R, D, E, []). -sparse_to_orddict_2(E=?NODEPATTERN(S), R, D, L) -> +sparse_to_orddict_2(E=?NODEPATTERN(S), R, D, L) when is_integer(S) -> sparse_to_orddict_3(?NODESIZE, R, D, L, E, S); sparse_to_orddict_2(E, _R, _D, L) when is_integer(E) -> L; @@ -1223,7 +1226,7 @@ from_orddict_0([], N, _Max, _D, Es) -> end; from_orddict_0(Xs=[{Ix1, _}|_], Ix, Max0, D, Es0) - when Ix1 > Max0, is_integer(Ix1) -> + when is_integer(Ix1), Ix1 > Max0 -> %% We have a hole larger than a leaf Hole = Ix1-Ix, Step = Hole - (Hole rem ?LEAFSIZE), @@ -1393,7 +1396,7 @@ from_orddict_test_() -> Function :: fun((Index :: array_indx(), Type1) -> Type2). map(Function, Array=#array{size = N, elements = E, default = D}) - when is_function(Function, 2) -> + when is_function(Function, 2), is_integer(N) -> if N > 0 -> A = Array#array{elements = []}, % kill reference, for GC A#array{elements = map_1(N-1, E, 0, Function, D)}; @@ -1485,7 +1488,7 @@ map_test_() -> Function :: fun((Index :: array_indx(), Type1) -> Type2). sparse_map(Function, Array=#array{size = N, elements = E, default = D}) - when is_function(Function, 2) -> + when is_function(Function, 2), is_integer(N) -> if N > 0 -> A = Array#array{elements = []}, % kill reference, for GC A#array{elements = sparse_map_1(N-1, E, 0, Function, D)}; @@ -1581,7 +1584,7 @@ sparse_map_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). foldl(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> foldl_1(N-1, E, A, 0, Function, D); true -> @@ -1653,7 +1656,7 @@ foldl_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). sparse_foldl(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> sparse_foldl_1(N-1, E, A, 0, Function, D); true -> @@ -1730,7 +1733,7 @@ sparse_foldl_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). foldr(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> I = N - 1, foldr_1(I, E, I, A, Function, D); @@ -1808,7 +1811,7 @@ foldr_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). sparse_foldr(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> I = N - 1, sparse_foldr_1(I, E, I, A, Function, D); @@ -1862,7 +1865,7 @@ sparse_size(A) -> try sparse_foldr(F, [], A) of [] -> 0 catch - {value, I} -> + {value, I} when is_integer(I) -> I + 1 end. diff --git a/lib/stdlib/src/base64.erl b/lib/stdlib/src/base64.erl index be4d9d42b4..eae9527a2d 100644 --- a/lib/stdlib/src/base64.erl +++ b/lib/stdlib/src/base64.erl @@ -21,12 +21,19 @@ -module(base64). --export([encode/1, decode/1, mime_decode/1, - encode_to_string/1, decode_to_string/1, mime_decode_to_string/1]). - +-export([encode/1, encode/2, + decode/1, decode/2, + mime_decode/1, mime_decode/2, + encode_to_string/1, encode_to_string/2, + decode_to_string/1, decode_to_string/2, + mime_decode_to_string/1, mime_decode_to_string/2]). %% RFC 4648: Base 64 Encoding alphabet --type base64_alphabet() :: $A..$Z | $a..$z | $0..$9 | $+ | $/ | $=. +-type base64_alphabet() :: $A..$Z | $a..$z | $0..$9 | $+ | $/ | $- | $_ | $=. + +%% Selector for the Base 64 alphabet, `standard' for RFC 4648 +%% Section 4, `urlsafe' for RFC 4648 Section 5. +-type base64_mode() :: 'standard' | 'urlsafe'. %% The following type is a subtype of string() for return values %% of encoding functions. @@ -40,67 +47,95 @@ Data :: byte_string() | binary(), Base64String :: base64_string(). -encode_to_string(Bin) when is_binary(Bin) -> - encode_to_string(binary_to_list(Bin)); -encode_to_string(List) when is_list(List) -> - encode_list_to_string(List). +encode_to_string(Data) -> + encode_to_string(Data, standard). + +-spec encode_to_string(Data, Mode) -> Base64String when + Data :: byte_string() | binary(), + Mode :: base64_mode(), + Base64String :: base64_string(). + +encode_to_string(Bin, Mode) when is_binary(Bin) -> + encode_to_string(binary_to_list(Bin), Mode); +encode_to_string(List, Mode) when is_list(List) -> + encode_list_to_string(get_encoding_offset(Mode), List). -spec encode(Data) -> Base64 when Data :: byte_string() | binary(), Base64 :: base64_binary(). -encode(Bin) when is_binary(Bin) -> - encode_binary(Bin, <<>>); -encode(List) when is_list(List) -> - encode_list(List, <<>>). +encode(Data) -> + encode(Data, standard). + +-spec encode(Data, Mode) -> Base64 when + Data :: byte_string() | binary(), + Mode :: base64_mode(), + Base64 :: base64_binary(). + +encode(Bin, Mode) when is_binary(Bin) -> + encode_binary(get_encoding_offset(Mode), Bin, <<>>); +encode(List, Mode) when is_list(List) -> + encode_list(get_encoding_offset(Mode), List, <<>>). -encode_list_to_string([]) -> +encode_list_to_string(_ModeOffset, []) -> []; -encode_list_to_string([B1]) -> - [b64e(B1 bsr 2), - b64e((B1 band 3) bsl 4), $=, $=]; -encode_list_to_string([B1,B2]) -> - [b64e(B1 bsr 2), - b64e(((B1 band 3) bsl 4) bor (B2 bsr 4)), - b64e((B2 band 15) bsl 2), $=]; -encode_list_to_string([B1,B2,B3|Ls]) -> +encode_list_to_string(ModeOffset, [B1]) -> + [b64e(B1 bsr 2, ModeOffset), + b64e((B1 band 3) bsl 4, ModeOffset), $=, $=]; +encode_list_to_string(ModeOffset, [B1,B2]) -> + [b64e(B1 bsr 2, ModeOffset), + b64e(((B1 band 3) bsl 4) bor (B2 bsr 4), ModeOffset), + b64e((B2 band 15) bsl 2, ModeOffset), $=]; +encode_list_to_string(ModeOffset, [B1,B2,B3|Ls]) -> BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - [b64e(BB bsr 18), - b64e((BB bsr 12) band 63), - b64e((BB bsr 6) band 63), - b64e(BB band 63) | encode_list_to_string(Ls)]. - -encode_binary(<<>>, A) -> + [b64e(BB bsr 18, ModeOffset), + b64e((BB bsr 12) band 63, ModeOffset), + b64e((BB bsr 6) band 63, ModeOffset), + b64e(BB band 63, ModeOffset) | encode_list_to_string(ModeOffset, Ls)]. + +encode_binary(ModeOffset, <<B1:6, B2:6, B3:6, B4:6, B5:6, B6:6, B7:6, B8:6, Ls/bits>>, A) -> + encode_binary(ModeOffset, + Ls, + <<A/bits, + (b64e(B1, ModeOffset)):8, + (b64e(B2, ModeOffset)):8, + (b64e(B3, ModeOffset)):8, + (b64e(B4, ModeOffset)):8, + (b64e(B5, ModeOffset)):8, + (b64e(B6, ModeOffset)):8, + (b64e(B7, ModeOffset)):8, + (b64e(B8, ModeOffset)):8>>); +encode_binary(_ModeOffset, <<>>, A) -> A; -encode_binary(<<B1:8>>, A) -> - <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; -encode_binary(<<B1:8, B2:8>>, A) -> - <<A/bits,(b64e(B1 bsr 2)):8, - (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, - (b64e((B2 band 15) bsl 2)):8, $=:8>>; -encode_binary(<<B1:8, B2:8, B3:8, Ls/bits>>, A) -> - BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - encode_binary(Ls, - <<A/bits,(b64e(BB bsr 18)):8, - (b64e((BB bsr 12) band 63)):8, - (b64e((BB bsr 6) band 63)):8, - (b64e(BB band 63)):8>>). - -encode_list([], A) -> +encode_binary(ModeOffset, <<B1:6, B2:6, B3:6, B4:6, Ls/bits>>, A) -> + encode_binary(ModeOffset, + Ls, + <<A/bits, + (b64e(B1, ModeOffset)):8, + (b64e(B2, ModeOffset)):8, + (b64e(B3, ModeOffset)):8, + (b64e(B4, ModeOffset)):8>>); +encode_binary(ModeOffset, <<B1:6, B2:2>>, A) -> + <<A/bits,(b64e(B1, ModeOffset)):8,(b64e(B2 bsl 4, ModeOffset)):8,$=:8,$=:8>>; +encode_binary(ModeOffset, <<B1:6, B2:6, B3:4>>, A) -> + <<A/bits,(b64e(B1, ModeOffset)):8,(b64e(B2, ModeOffset)):8,(b64e(B3 bsl 2, ModeOffset)):8, $=:8>>. + +encode_list(_ModeOffset, [], A) -> A; -encode_list([B1], A) -> - <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; -encode_list([B1,B2], A) -> - <<A/bits,(b64e(B1 bsr 2)):8, - (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, - (b64e((B2 band 15) bsl 2)):8, $=:8>>; -encode_list([B1,B2,B3|Ls], A) -> +encode_list(ModeOffset, [B1], A) -> + <<A/bits,(b64e(B1 bsr 2, ModeOffset)):8,(b64e((B1 band 3) bsl 4, ModeOffset)):8,$=:8,$=:8>>; +encode_list(ModeOffset, [B1,B2], A) -> + <<A/bits,(b64e(B1 bsr 2, ModeOffset)):8, + (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4), ModeOffset)):8, + (b64e((B2 band 15) bsl 2, ModeOffset)):8, $=:8>>; +encode_list(ModeOffset, [B1,B2,B3|Ls], A) -> BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - encode_list(Ls, - <<A/bits,(b64e(BB bsr 18)):8, - (b64e((BB bsr 12) band 63)):8, - (b64e((BB bsr 6) band 63)):8, - (b64e(BB band 63)):8>>). + encode_list(ModeOffset, + Ls, + <<A/bits,(b64e(BB bsr 18, ModeOffset)):8, + (b64e((BB bsr 12) band 63, ModeOffset)):8, + (b64e((BB bsr 6) band 63, ModeOffset)):8, + (b64e(BB band 63, ModeOffset)):8>>). %% mime_decode strips away all characters not Base64 before %% converting, whereas decode crashes if an illegal character is found @@ -109,19 +144,35 @@ encode_list([B1,B2,B3|Ls], A) -> Base64 :: base64_string() | base64_binary(), Data :: binary(). -decode(Bin) when is_binary(Bin) -> - decode_binary(Bin, <<>>); -decode(List) when is_list(List) -> - decode_list(List, <<>>). +decode(Base64) -> + decode(Base64, standard). + +-spec decode(Base64, Mode) -> Data when + Base64 :: base64_string() | base64_binary(), + Mode :: base64_mode(), + Data :: binary(). + +decode(Bin, Mode) when is_binary(Bin) -> + decode_binary(get_decoding_offset(Mode), Bin, <<>>); +decode(List, Mode) when is_list(List) -> + decode_list(get_decoding_offset(Mode), List, <<>>). -spec mime_decode(Base64) -> Data when Base64 :: base64_string() | base64_binary(), Data :: binary(). -mime_decode(Bin) when is_binary(Bin) -> - mime_decode_binary(Bin, <<>>); -mime_decode(List) when is_list(List) -> - mime_decode_list(List, <<>>). +mime_decode(Base64) -> + mime_decode(Base64, standard). + +-spec mime_decode(Base64, Mode) -> Data when + Base64 :: base64_string() | base64_binary(), + Mode :: base64_mode(), + Data :: binary(). + +mime_decode(Bin, Mode) when is_binary(Bin) -> + mime_decode_binary(get_decoding_offset(Mode), Bin, <<>>); +mime_decode(List, Mode) when is_list(List) -> + mime_decode_list(get_decoding_offset(Mode), List, <<>>). %% mime_decode_to_string strips away all characters not Base64 before %% converting, whereas decode_to_string crashes if an illegal @@ -131,309 +182,338 @@ mime_decode(List) when is_list(List) -> Base64 :: base64_string() | base64_binary(), DataString :: byte_string(). -decode_to_string(Bin) when is_binary(Bin) -> - decode_to_string(binary_to_list(Bin)); -decode_to_string(List) when is_list(List) -> - decode_list_to_string(List). +decode_to_string(Base64) -> + decode_to_string(Base64, standard). + +-spec decode_to_string(Base64, Mode) -> DataString when + Base64 :: base64_string() | base64_binary(), + Mode :: base64_mode(), + DataString :: byte_string(). + +decode_to_string(Bin, Mode) when is_binary(Bin) -> + decode_to_string(binary_to_list(Bin), Mode); +decode_to_string(List, Mode) when is_list(List) -> + decode_list_to_string(get_decoding_offset(Mode), List). -spec mime_decode_to_string(Base64) -> DataString when Base64 :: base64_string() | base64_binary(), DataString :: byte_string(). -mime_decode_to_string(Bin) when is_binary(Bin) -> - mime_decode_to_string(binary_to_list(Bin)); -mime_decode_to_string(List) when is_list(List) -> - mime_decode_list_to_string(List). +mime_decode_to_string(Base64) -> + mime_decode_to_string(Base64, standard). + +-spec mime_decode_to_string(Base64, Mode) -> DataString when + Base64 :: base64_string() | base64_binary(), + Mode :: base64_mode(), + DataString :: byte_string(). + +mime_decode_to_string(Bin, Mode) when is_binary(Bin) -> + mime_decode_to_string(binary_to_list(Bin), Mode); +mime_decode_to_string(List, Mode) when is_list(List) -> + mime_decode_list_to_string(get_decoding_offset(Mode), List). %% Skipping pad character if not at end of string. Also liberal about %% excess padding and skipping of other illegal (non-base64 alphabet) %% characters. See section 3.3 of RFC4648 -mime_decode_list([0 | Cs], A) -> - mime_decode_list(Cs, A); -mime_decode_list([C1 | Cs], A) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_list(Cs, A, B1); - _ -> mime_decode_list(Cs, A) % eq is padding +mime_decode_list(ModeOffset, [C1 | Cs], A) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_list(ModeOffset, Cs, A, B1); + _ -> mime_decode_list(ModeOffset, Cs, A) % eq is padding end; -mime_decode_list([], A) -> +mime_decode_list(_ModeOffset, [], A) -> A. -mime_decode_list([0 | Cs], A, B1) -> - mime_decode_list(Cs, A, B1); -mime_decode_list([C2 | Cs], A, B1) -> - case b64d(C2) of +mime_decode_list(ModeOffset, [C2 | Cs], A, B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_list(Cs, A, B1, B2); - _ -> mime_decode_list(Cs, A, B1) % eq is padding + mime_decode_list(ModeOffset, Cs, A, B1, B2); + _ -> mime_decode_list(ModeOffset, Cs, A, B1) % eq is padding end. -mime_decode_list([0 | Cs], A, B1, B2) -> - mime_decode_list(Cs, A, B1, B2); -mime_decode_list([C3 | Cs], A, B1, B2) -> - case b64d(C3) of +mime_decode_list(ModeOffset, [C3 | Cs], A, B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_list(Cs, A, B1, B2, B3); + mime_decode_list(ModeOffset, Cs, A, B1, B2, B3); eq=B3 -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_list(Cs, A, B1, B2) + mime_decode_list_after_eq(ModeOffset, Cs, A, B1, B2, B3); + _ -> mime_decode_list(ModeOffset, Cs, A, B1, B2) end. -mime_decode_list([0 | Cs], A, B1, B2, B3) -> - mime_decode_list(Cs, A, B1, B2, B3); -mime_decode_list([C4 | Cs], A, B1, B2, B3) -> - case b64d(C4) of +mime_decode_list(ModeOffset, [C4 | Cs], A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> - mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + mime_decode_list(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); eq -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_list(Cs, A, B1, B2, B3) + mime_decode_list_after_eq(ModeOffset, Cs, A, B1, B2, B3); + _ -> mime_decode_list(ModeOffset, Cs, A, B1, B2, B3) end. -mime_decode_list_after_eq([0 | Cs], A, B1, B2, B3) -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); -mime_decode_list_after_eq([C | Cs], A, B1, B2, B3) -> - case b64d(C) of +mime_decode_list_after_eq(ModeOffset, [C | Cs], A, B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_list(Cs, A, B1, B2, B); - _ -> mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + eq -> mime_decode_list(ModeOffset, Cs, A, B1, B2, B); + _ -> mime_decode_list(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) end; - _ -> mime_decode_list_after_eq(Cs, A, B1, B2, B3) + _ -> mime_decode_list_after_eq(ModeOffset, Cs, A, B1, B2, B3) end; -mime_decode_list_after_eq([], A, B1, B2, eq) -> +mime_decode_list_after_eq(_ModeOffset, [], A, B1, B2, eq) -> <<A/bits,B1:6,(B2 bsr 4):2>>; -mime_decode_list_after_eq([], A, B1, B2, B3) -> +mime_decode_list_after_eq(_ModeOffset, [], A, B1, B2, B3) -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. -mime_decode_binary(<<0:8, Cs/bits>>, A) -> - mime_decode_binary(Cs, A); -mime_decode_binary(<<C1:8, Cs/bits>>, A) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_binary(Cs, A, B1); - _ -> mime_decode_binary(Cs, A) % eq is padding +mime_decode_binary(ModeOffset, <<C1:8, Cs/bits>>, A) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_binary(ModeOffset, Cs, A, B1); + _ -> mime_decode_binary(ModeOffset, Cs, A) % eq is padding end; -mime_decode_binary(<<>>, A) -> +mime_decode_binary(_ModeOffset, <<>>, A) -> A. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1) -> - mime_decode_binary(Cs, A, B1); -mime_decode_binary(<<C2:8, Cs/bits>>, A, B1) -> - case b64d(C2) of +mime_decode_binary(ModeOffset, <<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_binary(Cs, A, B1, B2); - _ -> mime_decode_binary(Cs, A, B1) % eq is padding + mime_decode_binary(ModeOffset, Cs, A, B1, B2); + _ -> mime_decode_binary(ModeOffset, Cs, A, B1) % eq is padding end. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2) -> - mime_decode_binary(Cs, A, B1, B2); -mime_decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> - case b64d(C3) of +mime_decode_binary(ModeOffset, <<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_binary(Cs, A, B1, B2, B3); + mime_decode_binary(ModeOffset, Cs, A, B1, B2, B3); eq=B3 -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_binary(Cs, A, B1, B2) + mime_decode_binary_after_eq(ModeOffset, Cs, A, B1, B2, B3); + _ -> mime_decode_binary(ModeOffset, Cs, A, B1, B2) end. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2, B3) -> - mime_decode_binary(Cs, A, B1, B2, B3); -mime_decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C4) of +mime_decode_binary(ModeOffset, <<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> - mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + mime_decode_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); eq -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_binary(Cs, A, B1, B2, B3) + mime_decode_binary_after_eq(ModeOffset, Cs, A, B1, B2, B3); + _ -> mime_decode_binary(ModeOffset, Cs, A, B1, B2, B3) end. -mime_decode_binary_after_eq(<<0:8, Cs/bits>>, A, B1, B2, B3) -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); -mime_decode_binary_after_eq(<<C:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C) of +mime_decode_binary_after_eq(ModeOffset, <<C:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_binary(Cs, A, B1, B2, B); - _ -> mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + eq -> mime_decode_binary(ModeOffset, Cs, A, B1, B2, B); + _ -> mime_decode_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) end; - _ -> mime_decode_binary_after_eq(Cs, A, B1, B2, B3) + _ -> mime_decode_binary_after_eq(ModeOffset, Cs, A, B1, B2, B3) end; -mime_decode_binary_after_eq(<<>>, A, B1, B2, eq) -> +mime_decode_binary_after_eq(_ModeOffset, <<>>, A, B1, B2, eq) -> <<A/bits,B1:6,(B2 bsr 4):2>>; -mime_decode_binary_after_eq(<<>>, A, B1, B2, B3) -> +mime_decode_binary_after_eq(_ModeOffset, <<>>, A, B1, B2, B3) -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. -mime_decode_list_to_string([0 | Cs]) -> - mime_decode_list_to_string(Cs); -mime_decode_list_to_string([C1 | Cs]) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_list_to_string(Cs, B1); - _ -> mime_decode_list_to_string(Cs) % eq is padding +mime_decode_list_to_string(ModeOffset, [C1 | Cs]) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_list_to_string(ModeOffset, Cs, B1); + _ -> mime_decode_list_to_string(ModeOffset, Cs) % eq is padding end; -mime_decode_list_to_string([]) -> +mime_decode_list_to_string(_ModeOffset, []) -> []. -mime_decode_list_to_string([0 | Cs], B1) -> - mime_decode_list_to_string(Cs, B1); -mime_decode_list_to_string([C2 | Cs], B1) -> - case b64d(C2) of +mime_decode_list_to_string(ModeOffset, [C2 | Cs], B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_list_to_string(Cs, B1, B2); - _ -> mime_decode_list_to_string(Cs, B1) % eq is padding + mime_decode_list_to_string(ModeOffset, Cs, B1, B2); + _ -> mime_decode_list_to_string(ModeOffset, Cs, B1) % eq is padding end. -mime_decode_list_to_string([0 | Cs], B1, B2) -> - mime_decode_list_to_string(Cs, B1, B2); -mime_decode_list_to_string([C3 | Cs], B1, B2) -> - case b64d(C3) of +mime_decode_list_to_string(ModeOffset, [C3 | Cs], B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_list_to_string(Cs, B1, B2, B3); - eq=B3 -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); - _ -> mime_decode_list_to_string(Cs, B1, B2) + mime_decode_list_to_string(ModeOffset, Cs, B1, B2, B3); + eq=B3 -> mime_decode_list_to_string_after_eq(ModeOffset, Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(ModeOffset, Cs, B1, B2) end. -mime_decode_list_to_string([0 | Cs], B1, B2, B3) -> - mime_decode_list_to_string(Cs, B1, B2, B3); -mime_decode_list_to_string([C4 | Cs], B1, B2, B3) -> - case b64d(C4) of +mime_decode_list_to_string(ModeOffset, [C4 | Cs], B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)]; + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(ModeOffset, Cs)]; eq -> - mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); - _ -> mime_decode_list_to_string(Cs, B1, B2, B3) + mime_decode_list_to_string_after_eq(ModeOffset, Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(ModeOffset, Cs, B1, B2, B3) end. -mime_decode_list_to_string_after_eq([0 | Cs], B1, B2, B3) -> - mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); -mime_decode_list_to_string_after_eq([C | Cs], B1, B2, B3) -> - case b64d(C) of +mime_decode_list_to_string_after_eq(ModeOffset, [C | Cs], B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_list_to_string(Cs, B1, B2, B); + eq -> mime_decode_list_to_string(ModeOffset, Cs, B1, B2, B); _ -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)] + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(ModeOffset, Cs)] end; - _ -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3) + _ -> mime_decode_list_to_string_after_eq(ModeOffset, Cs, B1, B2, B3) end; -mime_decode_list_to_string_after_eq([], B1, B2, eq) -> +mime_decode_list_to_string_after_eq(_ModeOffset, [], B1, B2, eq) -> binary_to_list(<<B1:6,(B2 bsr 4):2>>); -mime_decode_list_to_string_after_eq([], B1, B2, B3) -> +mime_decode_list_to_string_after_eq(_ModeOffset, [], B1, B2, B3) -> binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>). -decode_list([C1 | Cs], A) -> - case b64d(C1) of - ws -> decode_list(Cs, A); - B1 -> decode_list(Cs, A, B1) +decode_list(ModeOffset, [C1 | Cs], A) -> + case b64d(C1, ModeOffset) of + ws -> decode_list(ModeOffset, Cs, A); + B1 -> decode_list(ModeOffset, Cs, A, B1) end; -decode_list([], A) -> +decode_list(_ModeOffset, [], A) -> A. -decode_list([C2 | Cs], A, B1) -> - case b64d(C2) of - ws -> decode_list(Cs, A, B1); - B2 -> decode_list(Cs, A, B1, B2) +decode_list(ModeOffset, [C2 | Cs], A, B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_list(ModeOffset, Cs, A, B1); + B2 -> decode_list(ModeOffset, Cs, A, B1, B2) end. -decode_list([C3 | Cs], A, B1, B2) -> - case b64d(C3) of - ws -> decode_list(Cs, A, B1, B2); - B3 -> decode_list(Cs, A, B1, B2, B3) +decode_list(ModeOffset, [C3 | Cs], A, B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_list(ModeOffset, Cs, A, B1, B2); + B3 -> decode_list(ModeOffset, Cs, A, B1, B2, B3) end. -decode_list([C4 | Cs], A, B1, B2, B3) -> - case b64d(C4) of - ws -> decode_list(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); - eq -> only_ws(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) +decode_list(ModeOffset, [C4 | Cs], A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of + ws -> decode_list(ModeOffset, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws(ModeOffset, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws(ModeOffset, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_list(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) end. -decode_binary(<<C1:8, Cs/bits>>, A) -> - case b64d(C1) of - ws -> decode_binary(Cs, A); - B1 -> decode_binary(Cs, A, B1) +decode_binary(ModeOffset, <<C1:8, C2:8, C3:8, C4:8, Cs/bits>>, A) -> + case {b64d(C1, ModeOffset), b64d(C2, ModeOffset), b64d(C3, ModeOffset), b64d(C4, ModeOffset)} of + {B1, B2, B3, B4} when is_integer(B1), is_integer(B2), + is_integer(B3), is_integer(B4) -> + decode_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + {B1, B2, B3, B4} -> + dec_bin(ModeOffset, Cs, B1, B2, B3, B4, A) end; -decode_binary(<<>>, A) -> - A. +decode_binary(_ModeOffset, <<>>, A) -> + A; +decode_binary(ModeOffset, <<C1:8, Cs/bits>>, A) -> + case b64d(C1, ModeOffset) of + ws -> decode_binary(ModeOffset, Cs, A); + B1 -> decode_binary(ModeOffset, Cs, A, B1) + end. -decode_binary(<<C2:8, Cs/bits>>, A, B1) -> - case b64d(C2) of - ws -> decode_binary(Cs, A, B1); - B2 -> decode_binary(Cs, A, B1, B2) +dec_bin(ModeOffset, Cs, ws, B2, B3, B4, A) -> + dec_bin(ModeOffset, Cs, B2, B3, B4, A); +dec_bin(ModeOffset, Cs, B1, ws, B3, B4, A) -> + dec_bin(ModeOffset, Cs, B1, B3, B4, A); +dec_bin(ModeOffset, Cs, B1, B2, ws, B4, A) -> + dec_bin(ModeOffset, Cs, B1, B2, B4, A); +dec_bin(ModeOffset, Cs, B1, B2, B3, B4, A) -> + case B4 of + ws -> decode_binary(ModeOffset, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws_binary(ModeOffset, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) end. -decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> - case b64d(C3) of - ws -> decode_binary(Cs, A, B1, B2); - B3 -> decode_binary(Cs, A, B1, B2, B3) +dec_bin(ModeOffset, Cs, ws, B2, B3, A) -> + dec_bin(ModeOffset, Cs, B2, B3, A); +dec_bin(ModeOffset, Cs, B1, ws, B3, A) -> + dec_bin(ModeOffset, Cs, B1, B3, A); +dec_bin(ModeOffset, Cs, B1, B2, ws, A) -> + dec_bin(ModeOffset, Cs, B1, B2, A); +dec_bin(ModeOffset, Cs, B1, B2, B3, A) -> + decode_binary(ModeOffset, Cs, A, B1, B2, B3). + +dec_bin(ModeOffset, Cs, ws, B2, A) -> + dec_bin(ModeOffset, Cs, B2, A); +dec_bin(ModeOffset, Cs, B1, ws, A) -> + dec_bin(ModeOffset, Cs, B1, A); +dec_bin(ModeOffset, Cs, B1, B2, A) -> + decode_binary(ModeOffset, Cs, A, B1, B2). + +dec_bin(ModeOffset, Cs, ws, A) -> + decode_binary(ModeOffset, Cs, A); +dec_bin(ModeOffset, Cs, B1, A) -> + decode_binary(ModeOffset, Cs, A, B1). + +decode_binary(ModeOffset, <<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_binary(ModeOffset, Cs, A, B1); + B2 -> decode_binary(ModeOffset, Cs, A, B1, B2) end. -decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C4) of - ws -> decode_binary(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws_binary(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); - eq -> only_ws_binary(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) +decode_binary(ModeOffset, <<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_binary(ModeOffset, Cs, A, B1, B2); + B3 -> decode_binary(ModeOffset, Cs, A, B1, B2, B3) end. -only_ws_binary(<<>>, A) -> +decode_binary(ModeOffset, <<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of + ws -> decode_binary(ModeOffset, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws_binary(ModeOffset, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_binary(ModeOffset, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) + end. + +only_ws_binary(_ModeOffset, <<>>, A) -> A; -only_ws_binary(<<C:8, Cs/bits>>, A) -> - case b64d(C) of - ws -> only_ws_binary(Cs, A) +only_ws_binary(ModeOffset, <<C:8, Cs/bits>>, A) -> + case b64d(C, ModeOffset) of + ws -> only_ws_binary(ModeOffset, Cs, A) end. -decode_list_to_string([C1 | Cs]) -> - case b64d(C1) of - ws -> decode_list_to_string(Cs); - B1 -> decode_list_to_string(Cs, B1) +decode_list_to_string(ModeOffset, [C1 | Cs]) -> + case b64d(C1, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Cs); + B1 -> decode_list_to_string(ModeOffset, Cs, B1) end; -decode_list_to_string([]) -> +decode_list_to_string(_ModeOffset, []) -> []. -decode_list_to_string([C2 | Cs], B1) -> - case b64d(C2) of - ws -> decode_list_to_string(Cs, B1); - B2 -> decode_list_to_string(Cs, B1, B2) +decode_list_to_string(ModeOffset, [C2 | Cs], B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Cs, B1); + B2 -> decode_list_to_string(ModeOffset, Cs, B1, B2) end. -decode_list_to_string([C3 | Cs], B1, B2) -> - case b64d(C3) of - ws -> decode_list_to_string(Cs, B1, B2); - B3 -> decode_list_to_string(Cs, B1, B2, B3) +decode_list_to_string(ModeOffset, [C3 | Cs], B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Cs, B1, B2); + B3 -> decode_list_to_string(ModeOffset, Cs, B1, B2, B3) end. -decode_list_to_string([C4 | Cs], B1, B2, B3) -> - case b64d(C4) of +decode_list_to_string(ModeOffset, [C4 | Cs], B1, B2, B3) -> + case b64d(C4, ModeOffset) of ws -> - decode_list_to_string(Cs, B1, B2, B3); + decode_list_to_string(ModeOffset, Cs, B1, B2, B3); eq when B3 =:= eq -> - only_ws(Cs, binary_to_list(<<B1:6,(B2 bsr 4):2>>)); + only_ws(ModeOffset, Cs, binary_to_list(<<B1:6,(B2 bsr 4):2>>)); eq -> - only_ws(Cs, binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>)); + only_ws(ModeOffset, Cs, binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>)); B4 -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | decode_list_to_string(Cs)] + [Octet1, Octet2, Octet3 | decode_list_to_string(ModeOffset, Cs)] end. -only_ws([], A) -> +only_ws(_ModeOffset, [], A) -> A; -only_ws([C | Cs], A) -> - case b64d(C) of - ws -> only_ws(Cs, A) +only_ws(ModeOffset, [C | Cs], A) -> + case b64d(C, ModeOffset) of + ws -> only_ws(ModeOffset, Cs, A) end. %%%======================================================================== @@ -441,14 +521,19 @@ only_ws([C | Cs], A) -> %%%======================================================================== %% accessors --compile({inline, [{b64d, 1}]}). -%% One-based decode map. -b64d(X) -> - element(X, - {bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %1-15 + +get_decoding_offset(standard) -> 1; +get_decoding_offset(urlsafe) -> 257. + +-compile({inline, [{b64d, 2}]}). +b64d(X, Off) -> + element(X + Off, + { + %% standard base64 alphabet (RFC 4648 Section 4) + bad,bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %0-15 bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad,bad,63, %32-47 - 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-63 + 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-61 bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,bad, bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, @@ -460,13 +545,44 @@ b64d(X) -> bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + + %% alternative base64url alphabet (RFC 4648 Section 5) + bad,bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %0-15 + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 + ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad, %32-47 + 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-61 + bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,63, + bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad}). --compile({inline, [{b64e, 1}]}). -b64e(X) -> - element(X+1, - {$A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, +get_encoding_offset(standard) -> 1; +get_encoding_offset(urlsafe) -> 65. + +-compile({inline, [{b64e, 2}]}). +b64e(X, Off) -> + element(X + Off, + { + %% standard base64 alphabet (RFC 4648 Section 4) + $A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, + $O, $P, $Q, $R, $S, $T, $U, $V, $W, $X, $Y, $Z, + $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, + $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z, + $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $+, $/, + + %% alternative base64url alphabet (RFC 4648 Section 5) + $A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, $O, $P, $Q, $R, $S, $T, $U, $V, $W, $X, $Y, $Z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z, - $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $+, $/}). + $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $-, $_}). + diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 5eed3c06b6..b5249df414 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -51,6 +51,7 @@ -export([make_crypto_key/2, get_crypto_key/1]). %Utilities used by compiler -export_type([attrib_entry/0, compinfo_entry/0, labeled_entry/0, label/0]). +-export_type([chunkid/0]). -export_type([chnk_rsn/0]). -import(lists, [append/1, delete/2, foreach/2, keysort/2, @@ -831,8 +832,7 @@ symbol(_, AT, I1, I2, _I3, _Cnt) -> {atm(AT, I1), I2}. atm(AT, N) -> - [{_N, S}] = ets:lookup(AT, N), - S. + ets:lookup_element(AT, N, 2). %% AT is updated. ensure_atoms({empty, AT}, Cs) -> diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 640ad7a81c..133d209ae1 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -93,7 +93,8 @@ tab_name/0]). -compile({inline, [{einval,2},{badarg,2},{undefined,1}, - {badarg_exit,2},{lookup_reply,2}]}). + {badarg_exit,2},{lookup_reply,2}, + {pidof,1},{resp,2}]}). -include_lib("kernel/include/file.hrl"). @@ -1237,16 +1238,25 @@ treq(Tab, R) -> req(Proc, R) -> Ref = erlang:monitor(process, Proc), - Proc ! ?DETS_CALL(self(), R), + Proc ! ?DETS_CALL({self(), Ref}, R), receive {'DOWN', Ref, process, Proc, _Info} -> badarg; - {Proc, Reply} -> + {Ref, Reply} -> erlang:demonitor(Ref, [flush]), Reply end. %% Inlined. +pidof({Pid, _Tag}) -> + Pid. + +%% Inlined. +resp({Pid, Tag} = _From, Message) -> + Pid ! {Tag, Message}, + ok. + +%% Inlined. einval({error, {file_error, _, einval}}, A) -> erlang:error(badarg, A); einval({error, {file_error, _, badarg}}, A) -> @@ -1398,7 +1408,7 @@ apply_op(Op, From, Head, N) -> true -> err({error, incompatible_arguments}) end, - From ! {self(), Res}, + resp(From, Res), ok; auto_save -> case Head#head.update_mode of @@ -1419,7 +1429,7 @@ apply_op(Op, From, Head, N) -> {0, Head} end; close -> - From ! {self(), fclose(Head)}, + resp(From, fclose(Head)), _NewHead = unlink_fixing_procs(Head), ?PROFILE(ep:done()), exit(normal); @@ -1427,25 +1437,25 @@ apply_op(Op, From, Head, N) -> %% Used from dets_server when Pid has closed the table, %% but the table is still opened by some process. NewHead = remove_fix(Head, Pid, close), - From ! {self(), status(NewHead)}, + resp(From, status(NewHead)), NewHead; {corrupt, Reason} -> {H2, Error} = dets_utils:corrupt_reason(Head, Reason), - From ! {self(), Error}, + resp(From, Error), H2; {delayed_write, WrTime} -> delayed_write(Head, WrTime); info -> {H2, Res} = finfo(Head), - From ! {self(), Res}, + resp(From, Res), H2; {info, Tag} -> {H2, Res} = finfo(Head, Tag), - From ! {self(), Res}, + resp(From, Res), H2; {is_compatible_bchunk_format, Term} -> Res = test_bchunk_format(Head, Term), - From ! {self(), Res}, + resp(From, Res), ok; {internal_open, Ref, Args} -> do_internal_open(Head#head.parent, Head#head.server, From, @@ -1462,27 +1472,27 @@ apply_op(Op, From, Head, N) -> end; {set_verbose, What} -> set_verbose(What), - From ! {self(), ok}, + resp(From, ok), ok; {where, Object} -> {H2, Res} = where_is_object(Head, Object), - From ! {self(), Res}, + resp(From, Res), H2; _Message when element(1, Head#head.update_mode) =:= error -> - From ! {self(), status(Head)}, + resp(From, status(Head)), ok; %% The following messages assume that the status of the table is OK. {bchunk_init, Tab} -> {H2, Res} = do_bchunk_init(Head, Tab), - From ! {self(), Res}, + resp(From, Res), H2; {bchunk, State} -> {H2, Res} = do_bchunk(Head, State), - From ! {self(), Res}, + resp(From, Res), H2; delete_all_objects -> {H2, Res} = fdelete_all_objects(Head), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), {0, H2}; {delete_key, _Keys} when Head#head.update_mode =:= dirty -> @@ -1492,16 +1502,16 @@ apply_op(Op, From, Head, N) -> true -> stream_op(Op, From, [], Head, N); false -> - From ! {self(), badarg}, + resp(From, badarg), ok end; first -> {H2, Res} = ffirst(Head), - From ! {self(), Res}, + resp(From, Res), H2; {initialize, InitFun, Format, MinNoSlots} -> {H2, Res} = finit(Head, InitFun, Format, MinNoSlots), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), H2; {insert, Objs} when Head#head.update_mode =:= dirty -> @@ -1509,12 +1519,12 @@ apply_op(Op, From, Head, N) -> true -> stream_op(Op, From, [], Head, N); false -> - From ! {self(), badarg}, + resp(From, badarg), ok end; {insert_new, Objs} when Head#head.update_mode =:= dirty -> {H2, Res} = finsert_new(Head, Objs), - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {lookup_keys, _Keys} -> stream_op(Op, From, [], Head, N); @@ -1523,48 +1533,48 @@ apply_op(Op, From, Head, N) -> H2 = case Res of {cont,_} -> H1; _ when Safe =:= no_safe-> H1; - _ when Safe =:= safe -> do_safe_fixtable(H1, From, false) + _ when Safe =:= safe -> do_safe_fixtable(H1, pidof(From), false) end, - From ! {self(), Res}, + resp(From, Res), H2; {match, MP, Spec, NObjs, Safe} -> {H2, Res} = fmatch(Head, MP, Spec, NObjs, Safe, From), - From ! {self(), Res}, + resp(From, Res), H2; {member, _Key} = Op -> stream_op(Op, From, [], Head, N); {next, Key} -> {H2, Res} = fnext(Head, Key), - From ! {self(), Res}, + resp(From, Res), H2; {match_delete, State} when Head#head.update_mode =:= dirty -> {H1, Res} = fmatch_delete(Head, State), H2 = case Res of {cont,_S,_N} -> H1; - _ -> do_safe_fixtable(H1, From, false) + _ -> do_safe_fixtable(H1, pidof(From), false) end, - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {match_delete_init, MP, Spec} when Head#head.update_mode =:= dirty -> {H2, Res} = fmatch_delete_init(Head, MP, Spec, From), - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {safe_fixtable, Bool} -> - NewHead = do_safe_fixtable(Head, From, Bool), - From ! {self(), ok}, + NewHead = do_safe_fixtable(Head, pidof(From), Bool), + resp(From, ok), NewHead; {slot, Slot} -> {H2, Res} = fslot(Head, Slot), - From ! {self(), Res}, + resp(From, Res), H2; sync -> {NewHead, Res} = perform_save(Head, true), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), {0, NewHead}; {update_counter, Key, Incr} when Head#head.update_mode =:= dirty -> {NewHead, Res} = do_update_counter(Head, Key, Incr), - From ! {self(), Res}, + resp(From, Res), {N + 1, NewHead}; WriteOp when Head#head.update_mode =:= new_dirty -> H2 = Head#head{update_mode = dirty}, @@ -1577,12 +1587,12 @@ apply_op(Op, From, Head, N) -> H2 = Head#head{update_mode = dirty}, apply_op(WriteOp, From, H2, 0); {NewHead, Error} when is_record(NewHead, head) -> - From ! {self(), Error}, + resp(From, Error), NewHead end; WriteOp when is_tuple(WriteOp), Head#head.access =:= read -> Reason = {access_mode, Head#head.filename}, - From ! {self(), err({error, Reason})}, + resp(From, err({error, Reason})), ok end. @@ -1603,7 +1613,7 @@ bug_found(Name, Op, Bad, Stacktrace, From) -> end, if From =/= self() -> - From ! {self(), {error, {dets_bug, Name, Op, Bad}}}, + resp(From, {error, {dets_bug, Name, Op, Bad}}), ok; true -> % auto_save | may_grow | {delayed_write, _} ok @@ -1613,10 +1623,10 @@ do_internal_open(Parent, Server, From, Ref, Args) -> ?PROFILE(ep:do()), case do_open_file(Args, Parent, Server, Ref) of {ok, Head} -> - From ! {self(), ok}, + resp(From, ok), Head; Error -> - From ! {self(), Error}, + resp(From, Error), exit(normal) end. @@ -1698,7 +1708,7 @@ stream_end1(Pids, Next, N, C, Head, PwriteList) -> stream_end2(Pids, Pids, Next, N, C, Head1, PR). stream_end2([Pid | Pids], Ps, Next, N, C, Head, Reply) -> - Pid ! {self(), Reply}, + resp(Pid, Reply), stream_end2(Pids, Ps, Next, N+1, C, Head, Reply); stream_end2([], Ps, no_more, N, C, Head, _Reply) -> penalty(Head, Ps, C), @@ -1710,7 +1720,7 @@ penalty(H, _Ps, _C) when H#head.fixed =:= false -> ok; penalty(_H, _Ps, [{{lookup,_Pids},_Keys}]) -> ok; -penalty(#head{fixed = {_,[{Pid,_}]}}, [Pid], _C) -> +penalty(#head{fixed = {_,[{Pid, _}]}}, [{Pid, _Tag} = _From], _C) -> ok; penalty(_H, _Ps, _C) -> timer:sleep(1). @@ -1729,9 +1739,9 @@ lookup_replies(P, O, [{P2,O2} | L]) -> %% If a list of Pid then op was {member, Key}. Inlined. lookup_reply([P], O) -> - P ! {self(), O =/= []}; + resp(P, O =/= []); lookup_reply(P, O) -> - P ! {self(), O}. + resp(P, O). %%----------------------------------------------------------------- %% Callback functions for system messages handling. @@ -2253,7 +2263,7 @@ fmatch(Head, MP, Spec, N, Safe, From) -> {Head1, []} -> NewHead = case Safe of - safe -> do_safe_fixtable(Head1, From, true); + safe -> do_safe_fixtable(Head1, pidof(From), true); no_safe -> Head1 end, C0 = init_scan(NewHead, N), @@ -2370,7 +2380,7 @@ do_fmatch_delete_var_keys(Head, _MP, ?PATTERN_TO_TRUE_MATCH_SPEC('_'), _From) Reply end; do_fmatch_delete_var_keys(Head, MP, _Spec, From) -> - Head1 = do_safe_fixtable(Head, From, true), + Head1 = do_safe_fixtable(Head, pidof(From), true), {NewHead, []} = write_cache(Head1), C0 = init_scan(NewHead, default), {NewHead, {cont, C0#dets_cont{match_program = MP}, 0}}. diff --git a/lib/stdlib/src/edlin.erl b/lib/stdlib/src/edlin.erl index 6078c5e67b..b2603c5402 100644 --- a/lib/stdlib/src/edlin.erl +++ b/lib/stdlib/src/edlin.erl @@ -91,8 +91,8 @@ edit([C|Cs], P, Line, {blink,_}, [_|Rs]) -> %Remove blink here edit([C|Cs], P, Line, none, Rs); edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> case key_map(C, Prefix) of - meta -> - edit(Cs, P, {Bef,Aft}, meta, Rs0); + meta -> + edit(Cs, P, {Bef,Aft}, meta, Rs0); meta_o -> edit(Cs, P, {Bef,Aft}, meta_o, Rs0); meta_csi -> @@ -101,56 +101,44 @@ edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> edit(Cs, P, {Bef,Aft}, meta_meta, Rs0); {csi, _} = Csi -> edit(Cs, P, {Bef,Aft}, Csi, Rs0); - meta_left_sq_bracket -> - edit(Cs, P, {Bef,Aft}, meta_left_sq_bracket, Rs0); - search_meta -> - edit(Cs, P, {Bef,Aft}, search_meta, Rs0); - search_meta_left_sq_bracket -> - edit(Cs, P, {Bef,Aft}, search_meta_left_sq_bracket, Rs0); - ctlx -> - edit(Cs, P, {Bef,Aft}, ctlx, Rs0); - new_line -> - {done, get_line(Bef, Aft ++ "\n"), Cs, - reverse(Rs0, [{move_rel,cp_len(Aft)},{put_chars,unicode,"\n"}])}; - redraw_line -> - Rs1 = erase(P, Bef, Aft, Rs0), - Rs = redraw(P, Bef, Aft, Rs1), - edit(Cs, P, {Bef,Aft}, none, Rs); + meta_left_sq_bracket -> + edit(Cs, P, {Bef,Aft}, meta_left_sq_bracket, Rs0); + search_meta -> + edit(Cs, P, {Bef,Aft}, search_meta, Rs0); + search_meta_left_sq_bracket -> + edit(Cs, P, {Bef,Aft}, search_meta_left_sq_bracket, Rs0); + ctlx -> + edit(Cs, P, {Bef,Aft}, ctlx, Rs0); + new_line -> + {done, get_line(Bef, Aft ++ "\n"), Cs, + reverse(Rs0, [{move_rel,cp_len(Aft)},{put_chars,unicode,"\n"}])}; + redraw_line -> + Rs1 = erase(P, Bef, Aft, Rs0), + Rs = redraw(P, Bef, Aft, Rs1), + edit(Cs, P, {Bef,Aft}, none, Rs); + clear -> + Rs = redraw(P, Bef, Aft, [clear | Rs0]), + edit(Cs, P, {Bef,Aft}, none, Rs); tab_expand -> {expand, Bef, Cs, - {line, P, {Bef, Aft}, none}, + {line, P, {Bef, Aft}, tab_expand}, reverse(Rs0)}; - -%% tab -> -%% %% Always redraw the line since expand/1 might have printed -%% %% possible expansions. -%% case expand(Bef) of -%% {yes,Str} -> -%% edit([redraw_line| -%% (Str ++ Cs)], P, {Bef,Aft}, none, Rs0); -%% no -> -%% %% don't beep if there's only whitespace before -%% %% us - user may have pasted in a lot of indented stuff. -%% case whitespace_only(Bef) of -%% false -> -%% edit([redraw_line|Cs], P, {Bef,Aft}, none, -%% [beep|Rs0]); -%% true -> -%% edit([redraw_line|Cs], P, {Bef,Aft}, none, [Rs0]) -%% end -%% end; - {undefined,C} -> - {undefined,{none,Prefix,C},Cs,{line,P,{Bef,Aft},none}, + tab_expand_full -> + {expand_full, Bef, Cs, + {line, P, {Bef, Aft}, tab_expand}, reverse(Rs0)}; - Op -> - case do_op(Op, Bef, Aft, Rs0) of - {blink,N,Line,Rs} -> - edit(Cs, P, Line, {blink,N}, Rs); - {Line, Rs, Mode} -> % allow custom modes from do_op - edit(Cs, P, Line, Mode, Rs); - {Line,Rs} -> - edit(Cs, P, Line, none, Rs) - end + {undefined,C} -> + {undefined,{none,Prefix,C},Cs,{line,P,{Bef,Aft},none}, + reverse(Rs0)}; + Op -> + case do_op(Op, Bef, Aft, Rs0) of + {blink,N,Line,Rs} -> + edit(Cs, P, Line, {blink,N}, Rs); + {Line, Rs, Mode} -> % allow custom modes from do_op + edit(Cs, P, Line, Mode, Rs); + {Line,Rs} -> + edit(Cs, P, Line, none, Rs) + end end; edit([], P, L, {blink,N}, Rs) -> {blink,{line,P,L,{blink,N}},reverse(Rs)}; @@ -183,7 +171,7 @@ prefix_arg(N) -> N. %% key_map(Char, Prefix) %% Map a character and a prefix to an action. -key_map(A, _) when is_atom(A) -> A; % so we can push keywords +key_map(A, _) when is_atom(A) -> A; % so we can push keywords key_map($\^A, none) -> beginning_of_line; key_map($\^B, none) -> backward_char; key_map($\^D, none) -> forward_delete_char; @@ -191,9 +179,11 @@ key_map($\^E, none) -> end_of_line; key_map($\^F, none) -> forward_char; key_map($\^H, none) -> backward_delete_char; key_map($\t, none) -> tab_expand; -key_map($\^L, none) -> redraw_line; -key_map($\n, none) -> new_line; +key_map($\t, tab_expand) -> tab_expand_full; +key_map(C, tab_expand) -> key_map(C, none); key_map($\^K, none) -> kill_line; +key_map($\^L, none) -> clear; +key_map($\n, none) -> new_line; key_map($\r, none) -> new_line; key_map($\^T, none) -> transpose_char; key_map($\^U, none) -> ctlu; @@ -214,11 +204,13 @@ key_map($], Prefix) when Prefix =/= meta, key_map($B, meta) -> backward_word; key_map($D, meta) -> kill_word; key_map($F, meta) -> forward_word; +key_map($L, meta) -> redraw_line; key_map($T, meta) -> transpose_word; key_map($Y, meta) -> yank_pop; key_map($b, meta) -> backward_word; key_map($d, meta) -> kill_word; key_map($f, meta) -> forward_word; +key_map($l, meta) -> redraw_line; key_map($t, meta) -> transpose_word; key_map($y, meta) -> yank_pop; key_map($O, meta) -> meta_o; @@ -320,9 +312,9 @@ do_op({search, backward_delete_char}, [_|Bef], Aft, Rs) -> {{Bef,NAft}, [{insert_chars, unicode, NAft}, {delete_chars,-Offset}|Rs], search}; -do_op({search, backward_delete_char}, [], _Aft, Rs) -> - Aft="': ", - {{[],Aft}, Rs, search}; +do_op({search, backward_delete_char}, [], Aft, Rs) -> + NAft="': ", + {{[],NAft}, [{insert_chars, unicode, NAft}, {delete_chars,-cp_len(Aft)}|Rs], search}; do_op({search, skip_up}, Bef, Aft, Rs) -> Offset= cp_len(Aft), NAft = "': ", @@ -621,148 +613,3 @@ cp_len(Str) -> cp_len([GC|R], Len) -> cp_len(R, Len + gc_len(GC)); cp_len([], Len) -> Len. - -%% %% expand(CurrentBefore) -> -%% %% {yes,Expansion} | no -%% %% Try to expand the word before as either a module name or a function -%% %% name. We can handle white space around the seperating ':' but the -%% %% function name must be on the same line. CurrentBefore is reversed -%% %% and over_word/3 reverses the characters it finds. In certain cases -%% %% possible expansions are printed. - -%% expand(Bef0) -> -%% {Bef1,Word,_} = over_word(Bef0, [], 0), -%% case over_white(Bef1, [], 0) of -%% {[$:|Bef2],_White,_Nwh} -> -%% {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), -%% {_,Mod,_Nm} = over_word(Bef3, [], 0), -%% expand_function_name(Mod, Word); -%% {_,_,_} -> -%% expand_module_name(Word) -%% end. - -%% expand_module_name(Prefix) -> -%% match(Prefix, code:all_loaded(), ":"). - -%% expand_function_name(ModStr, FuncPrefix) -> -%% Mod = list_to_atom(ModStr), -%% case erlang:module_loaded(Mod) of -%% true -> -%% L = apply(Mod, module_info, []), -%% case lists:keyfind(exports, 1, L) of -%% {_, Exports} -> -%% match(FuncPrefix, Exports, "("); -%% _ -> -%% no -%% end; -%% false -> -%% no -%% end. - -%% match(Prefix, Alts, Extra) -> -%% Matches = match1(Prefix, Alts), -%% case longest_common_head([N || {N,_} <- Matches]) of -%% {partial, []} -> -%% print_matches(Matches), -%% no; -%% {partial, Str} -> -%% case lists:nthtail(length(Prefix), Str) of -%% [] -> -%% print_matches(Matches), -%% {yes, []}; -%% Remain -> -%% {yes, Remain} -%% end; -%% {complete, Str} -> -%% {yes, lists:nthtail(length(Prefix), Str) ++ Extra}; -%% no -> -%% no -%% end. - -%% %% Print the list of names L in multiple columns. -%% print_matches(L) -> -%% io:nl(), -%% col_print(lists:sort(L)), -%% ok. - -%% col_print([]) -> ok; -%% col_print(L) -> col_print(L, field_width(L), 0). - -%% col_print(X, Width, Len) when Width + Len > 79 -> -%% io:nl(), -%% col_print(X, Width, 0); -%% col_print([{H0,A}|T], Width, Len) -> -%% H = if -%% %% If the second element is an integer, we assume it's an -%% %% arity, and meant to be printed. -%% integer(A) -> -%% H0 ++ "/" ++ integer_to_list(A); -%% true -> -%% H0 -%% end, -%% io:format("~-*s",[Width,H]), -%% col_print(T, Width, Len+Width); -%% col_print([], _, _) -> -%% io:nl(). - -%% field_width([{H,_}|T]) -> field_width(T, length(H)). - -%% field_width([{H,_}|T], W) -> -%% case length(H) of -%% L when L > W -> field_width(T, L); -%% _ -> field_width(T, W) -%% end; -%% field_width([], W) when W < 40 -> -%% W + 4; -%% field_width([], _) -> -%% 40. - -%% match1(Prefix, Alts) -> -%% match1(Prefix, Alts, []). - -%% match1(Prefix, [{H,A}|T], L) -> -%% case prefix(Prefix, Str = atom_to_list(H)) of -%% true -> -%% match1(Prefix, T, [{Str,A}|L]); -%% false -> -%% match1(Prefix, T, L) -%% end; -%% match1(_, [], L) -> -%% L. - -%% longest_common_head([]) -> -%% no; -%% longest_common_head(LL) -> -%% longest_common_head(LL, []). - -%% longest_common_head([[]|_], L) -> -%% {partial, reverse(L)}; -%% longest_common_head(LL, L) -> -%% case same_head(LL) of -%% true -> -%% [[H|_]|_] = LL, -%% LL1 = all_tails(LL), -%% case all_nil(LL1) of -%% false -> -%% longest_common_head(LL1, [H|L]); -%% true -> -%% {complete, reverse([H|L])} -%% end; -%% false -> -%% {partial, reverse(L)} -%% end. - -%% same_head([[H|_]|T1]) -> same_head(H, T1). - -%% same_head(H, [[H|_]|T]) -> same_head(H, T); -%% same_head(_, []) -> true; -%% same_head(_, _) -> false. - -%% all_tails(LL) -> all_tails(LL, []). - -%% all_tails([[_|T]|T1], L) -> all_tails(T1, [T|L]); -%% all_tails([], L) -> L. - -%% all_nil([]) -> true; -%% all_nil([[] | Rest]) -> all_nil(Rest); -%% all_nil(_) -> false. diff --git a/lib/stdlib/src/edlin_context.erl b/lib/stdlib/src/edlin_context.erl new file mode 100644 index 0000000000..0bc3677fc4 --- /dev/null +++ b/lib/stdlib/src/edlin_context.erl @@ -0,0 +1,649 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_context). +%% description +%% +-export([get_context/1, get_context/2, odd_quotes/2]). +%% The context record is a structure that helps with viewing a nested expression +%% Typically we do not know the types of a tuple or a map in isolation, but if +%% we make use of the type information available for a function or a record +%% then we are able to extract more information. +%% A nesting can be either be a tuple a list or a map, more types of nestings could +%% be supported in the future. But it is for these types that we have type information. +%% We do not need to care about nested records or nested functions since the type information +%% is available for the deepest nested one. +%% The context record stores the top most nesting so far, while the nesting field contains +%% a list of nestings we found, the last element being the deepest nesting. +-type nesting() :: {'tuple', [{atom(), string()}], {atom(), string()} | []} + | {'list', [{atom(), string()}], {atom(), string()} | []} + | {'map', [string()], string(), [{atom(), string()}], {atom(), string()} | []}. +-record(context,{ + arguments = [] :: [any()], + fields = [] :: [string()], %% The Field or Keys in this nesting + parameter_count = 0 :: non_neg_integer(), %% Number of parameters in this nesting + current_field = [] :: string(), %% field of a record or key of a map field=<tab> %% should be the last element in this context + nestings = [] :: [nesting()]}). +%% get_context - basically get the context to be able to deduce how we should complete the word +%% If the word is empty, then we do not want to complete with anything if we just closed +%% a bracket, ended a quote (user has to enter , or . themselves) +%% but maybe we can help to add ',',},] depending on context in the future +-spec get_context(Line) -> Context when + Line :: string(), %% The whole line in reverse excluding Word + Mod :: string(), + Fun :: string(), + Args :: list(any()), + Unfinished :: any(), + Binding :: string(), %% map variable + Keys :: [string()], %% list of keys in the map MapBind + Record :: string(), + Fields :: [string()], %% list of keys in the record + FieldToComplete :: string(), + Nesting :: [nesting()] | [], + Context :: {string} %% cursor is inside a string "_ + | {binding} %% cursor is inside a Binding statement + | {term} %% cursor is at a free position, where any value is expected + | {term, Args, Unfinished} + | {fun_} %% cursor is in a fun statement (either a NewVar, '(' or mfa() is expected) + | {fun_, Mod} %% cursor is in a fun mod: statement (mfa) + | {fun_, Mod, Fun} %% cursor is in a fun mod:fun statement + | {new_fun, Unfinished} + | {function} + | {function, Mod, Fun, Args, Unfinished, Nesting} + | {map, Binding, Keys} + | {map_or_record} + | {record} + | {record, Record, Fields, FieldToComplete, Args, Unfinished, Nesting} + | {error, integer()}. +get_context(Line) -> + {Bef0, Word} = edlin_expand:over_word(Line), + case {{Bef0, Word}, odd_quotes($", Bef0)} of + {_, true} -> {string}; + {{[$#|_], []}, _} -> {map_or_record}; + {{_Bef1, Word}, _} -> + case is_binding(Word) of + true -> {binding}; + false -> get_context(Bef0, Word) + end + end. +get_context(">-" ++ _, L) when is_list(L) -> {term}; +get_context([$?|_], _) -> + {macro}; +get_context(Bef0, Word) when is_list(Word) -> + get_context(lists:reverse(Word) ++ Bef0, #context{}); +get_context([], #context{arguments = Args, parameter_count = Count, nestings = Nestings} = _CR) -> + case Count+1 == length(Args) of + true -> {term, lists:droplast(Args), lists:last(Args)}; + _ -> + %% Nestings will not end up as an argument + case Nestings of + [] -> case Count of + 0 when length(Args) > 0 -> {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; + [{list, Args1, Arg}] -> {term, Args1, Arg}; + [{tuple, Args1, Arg}] -> {term, Args1, Arg}; + [{map, _, _, Args1, Arg}] -> {term, Args1, Arg} + end + end; +get_context([$(|Bef], CR) -> + %% We have an unclosed opening parenthesis + %% Check if we have a function call + %% We can deduce the minimum arity based on how many terms we trimmed + %% We can check the type of the following Term and suggest those in special cases + %% shell_default and erlang are imported, make sure we can do expansion for those + {Bef1, Fun} = edlin_expand:over_word(Bef), + case Fun of + [] -> {term}; % parenthesis + _ -> + {_, Mod} = over_module(Bef1, Fun), + case Mod of + "shell" -> {term}; + "shell_default" -> {term}; + _ -> + case CR#context.parameter_count+1 == length(CR#context.arguments) of + true -> + %% N arguments N-1 commas, this means that we have an argument + %% still being worked on. + {function, Mod, Fun, lists:droplast(CR#context.arguments), + lists:last(CR#context.arguments),CR#context.nestings}; + _ -> + {function, Mod, Fun, CR#context.arguments, + [], CR#context.nestings} + end + end + end; +get_context([${|Bef], #context{ fields=Fields, + current_field=FieldToComplete, + arguments = Arguments, + parameter_count = Count, + nestings=Nestings}) -> + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + case edlin_expand:over_word(Bef) of + {[$#|Bef1], []} -> %% Map + {Bef2, Map} = edlin_expand:over_word(Bef1), + case Map of + [] -> get_context(Bef2, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'map', Fields, FieldToComplete, Args, Unfinished}|Nestings]}); + _ -> {map, Map, Fields} + end; + {_, []} -> + get_context(Bef, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'tuple', Args, Unfinished}|Nestings]}); + {[$#|_Bef3], Record} -> %% Record + {record, Record, Fields, FieldToComplete, Args, Unfinished, Nestings} + end; +get_context([$[|Bef1], #context{arguments = Arguments, parameter_count = Count, nestings=Nestings}) -> + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + get_context(Bef1, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'list', Args, Unfinished}|Nestings]}); +get_context([$,|Bef1], #context{parameter_count=Count}=CR) -> + get_context(Bef1, CR#context{ + parameter_count = Count+1}); +get_context([$>,$=|Bef1], #context{ parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + case Count of + 0 -> %% If count is 0, then we know its a value we may want to complete. + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$=,$:|Bef1], #context{ parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + case Count of + 0 -> %% If count is 0, then we know its a value we may want to complete. + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$=|Bef1], #context{ + parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + % if we are here, its always going to be + case Count of + 0 -> %%[$=|_], + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$.|Bef2], CR) -> + Arguments = CR#context.arguments, + Count = CR#context.parameter_count, + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + case edlin_expand:over_word(Bef2) of + {[$#|_Bef3], Record} -> %% Record + {record, Record, CR#context.fields, CR#context.current_field, Args, Unfinished, CR#context.nestings}; + _ -> {'end'} + end; +get_context([$:|Bef2], _) -> + %% look backwards to see if its a fun + {Bef3, Mod} = edlin_expand:over_word(Bef2), + case edlin_expand:over_word(Bef3) of + {_, "fun"} -> {fun_, Mod}; + _ -> {function} + end; +get_context([$/|Bef1], _) -> + {Bef2, Fun} = edlin_expand:over_word(Bef1), + {_, Mod} = over_module(Bef2, Fun), + {fun_, Mod, Fun}; +get_context([$>,$-|_Bef2], #context{arguments = Args} = CR) -> + %% Inside a function + case CR#context.parameter_count+1 == length(Args) of + true -> + {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; +get_context("nehw " ++ _Bef2, #context{arguments = Args} = CR) -> + %% Inside a guard + case CR#context.parameter_count+1 == length(Args) of + true -> + {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; +get_context([$\ |Bef],CR) -> get_context(Bef, CR); %% matching space here simplifies the other clauses +get_context(Bef0, #context{arguments=Args, parameter_count=Count} = CR) -> + case over_to_opening(Bef0) of + {_,[]} -> {term}; + {error, _}=E -> E; + {record} -> {record}; + {fun_} -> {fun_}; + {new_fun, _}=F -> F; + {Bef1, {fun_, Str}=Arg} -> + case Count of + 0 -> + [_, Mod, Fun| _] = string:tokens(Str, " :/"), + {fun_, Mod, Fun}; + _ -> get_context(Bef1, CR#context{arguments=[Arg|Args]}) + end; + {Bef1, Arg} -> get_context(Bef1, CR#context{arguments=[Arg|Args]}) + end. + +read_operator(Bef) -> + read_operator1(Bef). + +operator_string() -> "-=><:+*/|&^~". +read_operator1([$\ |Bef]) -> read_operator1(Bef); +read_operator1("mer " ++ Bef) -> {Bef, "rem"}; +read_operator1("osladna " ++ Bef) -> {Bef, "andalso"}; +read_operator1("dna " ++ Bef) -> {Bef, "and"}; +read_operator1("eslero " ++ Bef) -> {Bef, "orelse"}; +read_operator1("ro " ++ Bef) -> {Bef, "or"}; +read_operator1([$>,$>,$>|Bef1]) -> {[$>,$>|Bef1], [$>]}; %% comparison with a binary or typo +read_operator1([$>,$>|Bef1]) -> {[$>|Bef1], [$>]}; %% comparison with a pid, or binary +read_operator1([$>,$-, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$-,$>]}; + false -> {Bef, []} + end; +read_operator1([$>,$=, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$=,$>]}; + false -> {Bef, []} + end; +read_operator1([$=,$:, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$:,$=]}; + false -> {Bef, []} + end; +read_operator1([$:|_]=Bef) -> {Bef, []}; %% this operator does not count +read_operator1([Op1,Op2,Op3|Bef])-> + case {lists:member(Op1, operator_string()), + lists:member(Op2, operator_string()), + lists:member(Op3, operator_string())} of + {true, true, true} -> {Bef, [Op3, Op2, Op1]}; + {true, true, false} -> {[Op3|Bef], [Op2, Op1]}; + {true, false, _} -> {[Op2,Op3|Bef], [Op1]}; + _ -> {[Op1,Op2,Op3|Bef], []} + end; +read_operator1([Op1, Op2]) -> + case {lists:member(Op1, operator_string()), + lists:member(Op2, operator_string())} of + {true, true} -> {[], [Op2, Op1]}; + {true, false} -> {[Op2], [Op1]}; + _ -> {[Op1,Op2], []} + end; +read_operator1([Op1]) -> + case lists:member(Op1, operator_string()) of + true -> {[], [Op1]}; + _ -> {[Op1], []} + end; +read_operator1(Bef) -> {Bef, []}. + +read_opening_char("nehw "++Bef) -> + {Bef, "when"}; +read_opening_char([OC|Bef]) when OC =:= $(; OC =:= $[; OC =:= ${; OC =:= $,; OC =:= $. -> + {Bef, [OC]}; +read_opening_char([$>,$-|_]=Bef) -> + case read_operator(Bef) of + {_, []} -> {Bef, "->"}; + _ -> {Bef, []} + end; +read_opening_char([$\ |Bef]) -> read_opening_char(Bef); +read_opening_char(Bef) -> {Bef, []}. + +over_to_opening(Bef) -> try + over_to_opening1(Bef,#{args => []}) + catch + throw:E -> E + end. +over_to_opening1([], #{'args' := Args}) -> + over_to_opening_return([], Args); +over_to_opening1(Bef, Acc = #{args := Args}) -> + case edlin_expand:over_word(Bef) of + {_, []} -> %% removed spaces + case read_opening_char(Bef) of + {Bef1, []} -> %% not an opening + case extract_argument2(Bef1) of + {stop} -> over_to_opening_return(Bef1, Args); + {Bef2, []} -> over_to_opening_return(Bef2, Args); + {Bef2, Arg} -> over_to_opening1(Bef2, Acc#{args => [Arg | Args]}) + end; + {_Bef1, _Opening} -> over_to_opening_return(Bef, Args) + end; + _ -> case extract_argument2(Bef) of + {stop} -> over_to_opening_return(Bef, Args); + {Bef2, []} -> over_to_opening_return(Bef2, Args); + {Bef2, Arg} -> over_to_opening1(Bef2, Acc#{args => [Arg | Args]}) + end + end. +over_to_opening_return(Bef, Args) -> + case Args of + [] -> {Bef, []}; + [Arg] -> {Bef, Arg}; + [{operator, "-"}, {integer, I}] -> {Bef, {integer, "-" ++ I}}; + [{operator, "-"}, {float, F}] -> {Bef, {float, "-" ++ F}}; + [{atom, "fun"}, {atom, _}] -> throw({fun_}); + _ -> + case look_for_non_operator_separator(Args) of + true -> {Bef, {operation, lists:flatten(lists:join(" ", lists:map(fun({_, Arg}) -> Arg end, Args)))}}; + false -> {error, length(Bef)} + end + end. +look_for_non_operator_separator([{string, _},{string, _}=A|Args]) -> + look_for_non_operator_separator([A|Args]); +look_for_non_operator_separator([{operator, _}, {operator, _}|_]) -> false; + +look_for_non_operator_separator([_, {operator, _}=B|Args]) -> + look_for_non_operator_separator([B|Args]); +look_for_non_operator_separator([{operator, _}, B|Args]) -> + look_for_non_operator_separator([B|Args]); +look_for_non_operator_separator([_]) -> true; +look_for_non_operator_separator(_) -> false. + +over_map_record_or_tuple(Bef0) -> + case over_to_opening_paren($},Bef0) of + {_, []} -> %% no matching { + throw({error, length(Bef0)}); + {Bef3, Clause} -> + {Bef4, MaybeRecord} = edlin_expand:over_word(Bef3), + case MaybeRecord of + [] -> case Bef4 of + [$#|Bef5] -> %% Map + {Bef6, _Var} = edlin_expand:over_word(Bef5), + {Bef6, {map, _Var++"#"++Clause}}; + _ -> %% Tuple + {Bef4, {tuple, Clause}} + end; + _Record -> %% Record + [$#|Bef5] = Bef4, + {Bef6, _Var} = edlin_expand:over_word(Bef5), + {Bef6, {record, _Var++"#"++_Record++Clause}} + end + end. +over_pid_port_or_ref(Bef2) -> + %% Extracts argument or part of an operation + %% Consume Pid, Ref, FunRef or Binary + case over_to_opening_paren($>,Bef2) of + {_, []} -> %% no matching <, maybe a '>' operator + throw({soft_error, length(Bef2)}); + {Bef3, Clause} -> + case Bef3 of + "feR#" ++ Bef4 -> + {Bef4, {ref, "#Ref" ++ Clause}}; + "nuF#" ++ Bef4 -> + {Bef4, {'funref', "#Fun" ++ Clause}}; + "troP#" ++ Bef4 -> + {Bef4, {port, "#Port" ++ Clause}}; + _ -> case edlin_expand:over_word(Bef3) of + {Bef3, []} -> + case Bef2 of + [$>|_] -> %% binary + {Bef3, {binary, Clause}}; + _ -> %% pid + %% match <Num.Num.Num> + {Bef3, {pid, Clause}} + end; + _ -> + throw({error, length(Bef3)}) + end + end + end. +over_list(Bef2) -> + case over_to_opening_paren($],Bef2) of + {_, []} -> %% no matching [ + throw({error, length(Bef2)}); + {Bef3, Clause} -> + {Bef3, {list, Clause}} + end. +over_parenthesis_or_call(Bef2) -> + case over_to_opening_paren($),Bef2) of + {_, []} -> %% no matching ( + throw({error, length(Bef2)}); + {Bef3, Clause} -> + {Bef4, Fun} = edlin_expand:over_word(Bef3), + {Bef5, ModFun} = case Bef4 of + [$:|Bef41] -> + {Bef42, Mod} = edlin_expand:over_word(Bef41), + {Bef42, Mod++[$:|Fun]}; + _ -> {Bef4, Fun} + end, + case ModFun of + [] -> {Bef5, {parenthesis, Clause}}; + "fun" -> throw({new_fun, Clause}); + _ -> {Bef5, {call, ModFun++Clause}} + end + end. +over_keyword_or_fun(Bef1) -> + case over_keyword_expression(Bef1) of + {Bef2, KeywordExpression} -> {Bef2, {keyword, KeywordExpression ++ " end"}}; + _ -> throw({error, length(Bef1)}) + end. +extract_argument2([$>|Bef0]=Bef)-> + case read_operator(Bef) of + {[$>|_]=Bef1, ">"=Operator} -> + try over_pid_port_or_ref(Bef0) + catch + %% not a pid, port, ref or binary + throw:{error, _}=E -> throw(E); + throw:{soft_error, _Col} -> {Bef1, {operator, Operator}} + end; + {Bef1, ">"=Operator} -> + try over_pid_port_or_ref(Bef1) + catch + %% not a pid, port or ref + throw:{error, _}=E -> throw(E); + throw:{soft_error, _Col} -> {Bef1, {operator, Operator}} + end; + {_Bef1, []} -> {stop}; + {Bef1, Operator} -> {Bef1, {operator, Operator}} + end; +extract_argument2(Bef0) -> + case read_operator(Bef0) of + {[$}|Bef1], []} -> over_map_record_or_tuple(Bef1); + {[$)|Bef1], []} -> over_parenthesis_or_call(Bef1); + {[$]|Bef1], []} -> over_list(Bef1); + {[$"|Bef2], []} -> {Bef3, _Quote} = over_to_opening_quote($", Bef2), + {Bef3, {string, _Quote}}; + {"dne "++Bef1, []} -> over_keyword_or_fun(Bef1); + {[$=,$:|_], []} -> {stop}; + {[$:|_], []} -> {stop}; + {"nehw" ++ _Bef1,[]} -> {stop}; + {_, []} -> extract_argument(Bef0); + {Bef1, Operator} -> + {Bef1, {operator, Operator}} + end. + +extract_argument(Bef0) -> + %% TODO: We probably need to be able to extract Terms with operators... + case edlin_expand:over_word(Bef0) of + {_Bef1, []} -> + case read_char(_Bef1) of + {_, []} -> {_Bef1, []}; + {Bef2, Char} -> {Bef2, {char, Char}} + end; + {Bef2, Var} -> + try list_to_integer(Var) of + _ -> %% there is an integer + case over_fun_function(Bef0) of + {Bef3, "fun " ++ _ModFunArr} -> {Bef3, {fun_, "fun "++_ModFunArr}}; + _ -> case over_number(Bef0) of + {Bef3, []} -> {Bef3, []}; %% how to deal with operators + {Bef3, Number} -> {Bef3, Number} + + end + end + catch + _:_ -> + case is_binding(Var) of + true -> {Bef2,{var, Var}}; + false -> case Bef2 of + [$#|_] -> throw({record}); + _ -> {Bef2, {atom, Var}} + end + end + end + end. +over_number(Bef) -> + case edlin_expand:over_word(Bef) of + {_, []} -> {Bef, []}; + {Bef2, Var} -> + try list_to_integer(Var) of + _ -> + {Bef6, {NumberType, Number}}=Res = case edlin_expand:over_word(Bef2) of + {[$.|Bef3],[]} -> %% float + {Bef4, Integer} = edlin_expand:over_word(Bef3), + {Bef4, {float, Integer ++ "." ++ Var}}; + {[$#|Bef3],[]} -> %% integer base + {Bef4, Base} = edlin_expand:over_word(Bef3), + {Bef4, {integer, Base ++ "#" ++ Var}}; + _ -> + {Bef2, {integer, Var}} + %% otherwise its an operation that can be very complicated we should read everything up to the closest CC + %% and return an {operation, Clause} + end, + case edlin_expand:over_word(Bef6) of + {[$-|Bef5], []} -> + case read_opening_char(Bef5) of + {_, []} -> Res; + _ -> {Bef5, {NumberType, "-" ++Number}} + end; + _ -> Res + end + catch + _:_ -> {Bef, []} + end + end. +read_char([C,$$|Line]) -> + {Line, [$$,C]}; +read_char([$$|Line]) -> + {Line, "$ "}; +read_char(Line) -> + {Line, []}. + +over_fun_function(Bef) -> + over_fun_function(Bef, []). +over_fun_function(Bef, Acc) -> + case edlin_expand:over_word(Bef) of + {[$/|Bef1], Arity} -> over_fun_function(Bef1, [$/|Arity]++Acc); + {[$:|Bef1], Fun} -> over_fun_function(Bef1, [$:|Fun]++Acc); + {" nuf"++Bef1, ModOrFun} -> over_fun_function(Bef1, "fun "++ModOrFun ++ Acc); + _ -> {Bef,Acc} + end. + +%% Extracts everything within the quote +over_to_opening_quote(Q, Bef) when Q == $'; Q == $" -> + over_to_opening_quote([Q], Bef, [Q]); +over_to_opening_quote(_, Bef) -> {Bef, []}. +over_to_opening_quote([], Bef, Word) -> {Bef, Word}; +over_to_opening_quote([Q|Stack], [Q|Bef], Word) -> + over_to_opening_quote(Stack, Bef, [Q| Word]); +over_to_opening_quote([Q|Stack], [Q,EC|Bef], Word) when EC=:=$\\; EC=:=$$ -> + over_to_opening_quote([Q|Stack], Bef, [EC,Q| Word]); +over_to_opening_quote([Stack], [C|Bef], Word) -> + over_to_opening_quote([Stack], Bef, [C| Word]); +over_to_opening_quote(_,_,Word) -> {lists:reverse(Word), []}. + +matching_paren($(,$)) -> true; +matching_paren($[,$]) -> true; +matching_paren(${,$}) -> true; +matching_paren($<,$>) -> true; +matching_paren(_,_) -> false. + +%% Extracts everything within the brackets +%% Recursively extracts nested bracket expressions. +over_to_opening_paren(CC, Bef) when CC == $); CC == $]; + CC == $}; CC == $> -> + over_to_opening_paren([CC], Bef, [CC]); +over_to_opening_paren(_, Bef) -> {Bef, []}. %% Not a closing parenthesis +over_to_opening_paren([], Bef, Word) -> {Bef, Word}; +over_to_opening_paren(_, [], Word) -> {lists:reverse(Word), []}; %% Not a closing parenthesis +over_to_opening_paren([CC|Stack], [CC,$$|Bef], Word) -> + over_to_opening_paren([CC|Stack], Bef, [$$,CC|Word]); +over_to_opening_paren([CC|Stack], [OC|Bef], Word) when OC==$(; OC==$[; OC==${; OC==$< -> + case matching_paren(OC, CC) of + true -> over_to_opening_paren(Stack, Bef, [OC|Word]); + false -> over_to_opening_paren([CC|Stack], Bef, [OC|Word]) + end; +over_to_opening_paren([CC|Stack], [CC|Bef], Word) -> %% Nested parenthesis of same type + over_to_opening_paren([CC,CC|Stack], Bef, [CC|Word]); +over_to_opening_paren(Stack, [Q,NEC|Bef], Word) when Q == $"; Q == $', NEC /= $$, NEC /= $\\ -> + %% Consume the whole quoted text, it may contain parenthesis which + %% would have confused us. + {Bef1, QuotedWord} = over_to_opening_quote(Q, Bef), + over_to_opening_paren(Stack, Bef1, QuotedWord ++ Word); +over_to_opening_paren(CC, [C|Bef], Word) -> over_to_opening_paren(CC, Bef, [C|Word]). + +%% Extract a whole keyword expression +%% Keyword<code>end +%% Function expects a string of erlang code in reverse, and extracts everything +%% including a keyword being one of if, fun, case, maybe, receiver (need to add all here) +%% Recursively extracts nested keyword expressions +%% Note: In the future we could autocomplete case expressions by looking at the +%% return type of the expression. +over_keyword_expression(Bef) -> + over_keyword_expression(Bef, []). +over_keyword_expression("dne"++Bef, Expr)-> + %% Nested expression + {Bef1, KWE}=over_keyword_expression(Bef), + over_keyword_expression(Bef1, KWE++"end"++Expr); +over_keyword_expression("fi"++Bef, Expr) -> {Bef, "if" ++ Expr}; +over_keyword_expression("nuf"++Bef, Expr) -> {Bef, "fun" ++ Expr}; +over_keyword_expression("yrt"++Bef, Expr) -> {Bef, "try" ++ Expr}; +over_keyword_expression("esac"++Bef, Expr) -> {Bef, "case" ++ Expr}; +over_keyword_expression("hctac"++Bef, Expr) -> + case over_keyword_expression(Bef, []) of + {Bef1, "try" ++ Expr1} -> {Bef1, "try" ++ Expr1 ++ "catch" ++ Expr}; + _ -> {Bef, "catch" ++ Expr} + end; +over_keyword_expression("nigeb"++Bef, Expr) -> {Bef, "begin" ++ Expr}; +over_keyword_expression("ebyam"++Bef, Expr) -> {Bef, "maybe" ++ Expr}; +over_keyword_expression("eviecer"++Bef, Expr) -> {Bef, "receive" ++ Expr}; +over_keyword_expression([], _) -> {no, [], []}; +over_keyword_expression([C|Bef], Expr) -> over_keyword_expression(Bef, [C|Expr]). + +odd_quotes(Q, [Q,C|Line], Acc) when C == $\\; C == $$ -> + odd_quotes(Q, Line, Acc); +odd_quotes(Q, [Q|Line], Acc) -> + odd_quotes(Q, Line, Acc+1); +odd_quotes(Q, [_|Line], Acc) -> + odd_quotes(Q, Line, Acc); +odd_quotes(_, [], Acc) -> Acc band 1 == 1. +odd_quotes(Q, Line) -> + odd_quotes(Q, Line, 0). + +over_module(Bef, Fun)-> + case edlin_expand:over_word(Bef) of + {[$:|Bef1], _} -> + edlin_expand:over_word(Bef1); + {[], _} -> {Bef, edlin_expand:shell_default_or_bif(Fun)}; + _ -> {Bef, edlin_expand:bif(Fun)} + end. + +%% Check that the given string starts with a capital letter, or an underscore +%% followed by an alphanumeric grapheme. +is_binding(Word) -> + Normalized = unicode:characters_to_nfc_list(Word), + nomatch =/= re:run(Normalized, + "^[_[:upper:]][[:alpha:]]*$", + [unicode, ucp]). diff --git a/lib/stdlib/src/edlin_expand.erl b/lib/stdlib/src/edlin_expand.erl index bc3de13750..f7ec9f4c6c 100644 --- a/lib/stdlib/src/edlin_expand.erl +++ b/lib/stdlib/src/edlin_expand.erl @@ -18,47 +18,784 @@ %% %CopyrightEnd% %% -module(edlin_expand). +%% a default expand function for edlin, expanding modules, functions +%% filepaths, variable binding, record names, function parameter values, +%% record fields and map keys and record field values. +-include_lib("kernel/include/eep48.hrl"). +-export([expand/1, expand/2, expand/3, format_matches/2, number_matches/1, get_exports/1, + shell_default_or_bif/1, bif/1, over_word/1]). +-export([is_type/3, match_arguments1/3]). +-record(shell_state,{ + bindings = [], + records = [], + functions = [] + }). -%% a default expand function for edlin, expanding modules and functions +-spec expand(Bef0) -> {Res, Completion, Matches} when + Bef0 :: string(), %% a line of erlang expressions in reverse + Res :: 'yes' | 'no', + Completion :: string(), + Matches :: [Element] | [Section], + Element :: {string(), [ElementOption]}, + ElementOption :: {ending, string()}, + Section :: #{title:=string(), elems:=Matches, options:=SectionOption}, + SectionOption :: {highlight_all} %% highlight the whole title + | {highlight, string()} %% highlight this part of the title + | {highlight_param, integer()} %% highlight this parameter + | {hide, title} %% hide the title + | {hide, result} %% hide the results + | {separator, string()}. %% specify another separator between title and result +expand(Bef0) -> + expand(Bef0, [{legacy_output, true}]). + +-spec expand(Bef0, Opts) -> {Res, Completion, Matches} when + Bef0 :: string(), %% a line of erlang expressions in reverse + Opts :: [Option], + Option :: {legacy_output, boolean()}, + Res :: 'yes' | 'no', + Completion :: string(), + Matches :: [Element] | [Section], + Element :: {string(), [ElementOption]}, + ElementOption :: {ending, string()}, + Section :: #{title:=string(), elems:=Matches, options:=SectionOption}, + SectionOption :: {highlight_all} %% highlight the whole title + | {highlight, string()} %% highlight this part of the title + | {highlight_param, integer()} %% highlight this parameter + | {hide, title} %% hide the title + | {hide, result} %% hide the results + | {separator, string()}. %% specify another separator between title and result +expand(Bef0, Opts) -> + ShellState = try + shell:get_state() + catch + _:_ -> + %% Running on a shell that does not support get_state() + #shell_state{bindings=[],records=[],functions=[]} + end, + expand(Bef0, Opts, ShellState). + +%% Only used for testing +expand(Bef0, Opts, #shell_state{bindings = Bs, records = RT, functions = FT}) -> + LegacyOutput = proplists:get_value(legacy_output, Opts, false), + {_Bef1, Word} = over_word(Bef0), + {Res, Expansion, Matches} = case edlin_context:get_context(Bef0) of + + {string} -> expand_string(Bef0); + + {binding} -> expand_binding(Word, Bs); + + {term} -> expand_module_function(Bef0, FT); + {term, _, {_, Unfinished}} -> expand_module_function(lists:reverse(Unfinished), FT); + {error, _Column} -> + {no, [], []}; + {function} -> expand_module_function(Bef0, FT); + {fun_} -> expand_module_function(Bef0, FT); + + {fun_, Mod} -> expand_function_name(Mod, Word, "/", FT); + + %% Complete with arity in a 'fun mod:fun/' expression + {fun_, Mod, Fun} -> + Arities = [integer_to_list(A) || A <- get_arities(Mod, Fun)], + match(Word, Arities, ""); + {new_fun, _ArgsString} -> {no, [], []}; + %% Suggest type of function parameter + %% Complete an unfinished list, tuple or map using type of function parameter + {function, Mod, Fun, Args, Unfinished, Nesting} -> + Mod2 = case Mod of + "user_defined" -> "shell_default"; + _ -> Mod + end, + FunExpansion = expand_function_type(Mod2, Fun, Args, Unfinished, Nesting, FT), + case Word of + [] -> FunExpansion; + _ -> + ModuleOrBifs = expand_helper(FT, module, Word, ":"), + Functions = case Args =/= [] andalso lists:last(Args) of + {atom, MaybeMod} -> expand_function_name(MaybeMod, Word, "", FT); + _ -> {no, [], []} + end, + fold_results([FunExpansion] ++ ModuleOrBifs ++ [Functions]) + end; + + %% Complete an unfinished key or suggest valid keys of a map binding + {map, Binding, Keys} -> expand_map(Word, Bs, Binding, Keys); + + {map_or_record} -> + {[$#|Bef2], _} = over_word(Bef0), + {_, Var} = over_word(Bef2), + case Bs of + [] -> expand_record(Word, RT); + _ -> + case proplists:get_value(list_to_atom(Var), Bs) of + undefined -> + expand_record(Word, RT); + Map when is_map(Map) -> {yes, "{", []}; + RecordTuple when is_tuple(RecordTuple), tuple_size(RecordTuple) > 0 -> + Atom = erlang:element(1, RecordTuple), + case (is_atom(Atom) andalso lists:keysearch(Atom, 1, RT)) of + {value, {Atom, _}} -> match(Word, [Atom], "{"); + _ -> {no, [], []} + end; + _ -> {no, [], []} + end + end; + + {record} -> expand_record(Word, RT); + + {record, Record, Fields, FieldToComplete, Args, Unfinished, Nestings} -> + RecordExpansion = expand_record_fields(FieldToComplete, Unfinished, Record, Fields, RT, Args, Nestings, FT), + case Word of + [] -> RecordExpansion; + _ -> + ModuleOrBifs = expand_helper(FT, module,Word,":"), + fold_results([RecordExpansion] ++ ModuleOrBifs) + end; + _ -> {no, [], []} + + end, + Matches1 = case {Res,number_matches(Matches)} of + {yes, 1} -> []; + _ -> Matches + end, + case LegacyOutput of + true -> {Res, Expansion, to_legacy_format(Matches1)}; + false -> {Res, Expansion, Matches1} + end. +expand_map(_, [], _, _) -> + {no, [], []}; +expand_map(Word, Bs, Binding, Keys) -> + case proplists:get_value(list_to_atom(Binding), Bs) of + Map when is_map(Map) -> + K1 = sets:from_list(maps:keys(Map)), + K2 = sets:subtract(K1, sets:from_list([list_to_atom(K) || K <- Keys])), + match(Word, sets:to_list(K2), "=>"); + _ -> {no, [], []} + end. + +over_word(Bef) -> + {Bef1,_,_} = over_white(Bef, [], 0), + {Bef2, Word, _} = edlin:over_word(Bef1, [], 0), + {Bef2, Word}. + + +expand_binding(Prefix, Bindings) -> + Alts = [strip_quotes(K) || {K,_} <- Bindings], + case match(Prefix, Alts, "") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"bindings", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_record(Prefix, RT) -> + Alts = [Name || {Name, _} <- RT], + case match(Prefix, Alts, "{") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"records", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_record_fields(FieldToComplete, Word, Record, Fields, RT, _Args, Nestings, FT) -> + Record2 = list_to_atom(Record), + FieldSet2 = sets:from_list([list_to_atom(F) || F <- Fields]), + FieldToComplete2 = list_to_atom(FieldToComplete), + Word1 = case Word of + {_, Word2} -> Word2; + [] -> [] + end, + case [RecordSpec || {Record3, RecordSpec} <- RT, Record2 =:= Record3] of + [RecordType|_] -> + case sets:is_element(FieldToComplete2, FieldSet2) of + true -> + expand_record_field_content(FieldToComplete2, RecordType, Word1, Nestings, FT); + false -> + expand_record_field_name(Record2, FieldSet2, RecordType, Word1) + end; + _ -> + {no, [], []} + end. + +expand_record_field_name(Record, Fields, RecordType, Word) -> + RecordFieldsList = extract_record_fields(Record, RecordType), + RecordFieldsSet = sets:from_list(RecordFieldsList), + RecordFields = sets:subtract(RecordFieldsSet, Fields), + Alts = sets:to_list(RecordFields), + case match(Word, Alts, "=") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"fields", elems=>Matches, options=>[highlight_all]}]} + end. --export([expand/1, format_matches/1]). +expand_record_field_content(Field, + {attribute, _, record, + {_Record, FieldTypes}}, Word, Nestings, FT) -> + FieldTypesFiltered = [Type1 || {typed_record_field, {record_field, _, {_,_, F}}, Type1} <- FieldTypes, F == Field] ++ + [Type1 || {typed_record_field, {record_field, _, {_,_, F}, _}, Type1} <- FieldTypes, F == Field], + case FieldTypesFiltered of + [] -> {no, [], []}; + [Type] -> + T = edlin_type_suggestion:type_tree(erlang, Type, Nestings, FT), + Types = edlin_type_suggestion:get_types([], T, Nestings), + case Nestings of + [] -> + Atoms = edlin_type_suggestion:get_atoms([], T, Nestings), + case {Word, match(Word, Atoms, ", ")} of + {[],{_Res,_Expansion,_}} -> {_Res, _Expansion, [#{title=>"types", elems=>Types, options=>[{hide, title}]}]}; + {_,{_Res,_Expansion,[]}=M} -> M; + {_,{Res,Expansion,Matches}} -> {Res, Expansion, [#{title=>"matches", elems=>Matches, options=>[highlight_all]}]} + end; + _ -> + expand_nesting_content(T, [], Nestings, #{title=>"types", elems=>Types, options=>[{hide, title}]}) + end + end. + +%% Check that the actual type on previous arguments +%% matches with the expected types +%% Since we are not doing any evaluations at this point we +%% don't know if a parenthesis, keyword, var, call or fun returns +%% a value with the wrong type. +match_arguments({function, {{parameters, Ps}, _}, Cs}, As) -> + match_arguments1(Ps, Cs, As); +match_arguments({{parameters, Ps}, _}, As) -> + match_arguments1(Ps, [], As). +match_arguments1(_,_,[]) -> true; +%% Just assume that it will evaluate to the correct type. +match_arguments1([_|Ps], Cs, [{parenthesis, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{operation, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{keyword, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{var, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{call, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{fun_, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([P|Ps], Cs, [{atom, [$'|_]=String}|As]) -> + case edlin_context:odd_quotes($', lists:reverse(String)) of + true -> false; % we know that the atom is unfinished, and thus cannot match any valid atom + _ -> case is_type(P, Cs, String) of + true -> match_arguments1(Ps, Cs, As); + false -> false + end + end; +match_arguments1([P|Ps], Cs, [{_, String}|As]) -> + case is_type(P, Cs, String) of + true -> match_arguments1(Ps, Cs, As); + false -> false + end. + +is_type(Type, Cs, String) -> + {ok, A, _} = erl_scan:string(String++"."), + Types = [T || T <- edlin_type_suggestion:get_types(Cs, Type, [], [no_print]) ], + try + {ok, Term} = erl_parse:parse_term(A), + case Term of + Atom when is_atom(Atom) -> + Atoms = edlin_type_suggestion:get_atoms(Cs, Type, []), + lists:member(to_list(Atom), Atoms) orelse + lists:member(atom_to_list(Atom), Atoms) orelse + find_type(Types, [atom, node, module, 'fun']); + Tuple when is_tuple(Tuple) -> find_type(Types, [tuple]); + Map when is_map(Map) -> find_type(Types, [map]); + Binary when is_binary(Binary) -> find_type(Types, [binary]); + Float when is_float(Float) -> find_type(Types, [float]); + Integer when is_integer(Integer) -> check_integer_type(Types, Integer); + List when is_list(List), length(List) > 0 -> + find_type(Types, [list, string, nonempty_list,maybe_improper_list, nonempty_improper_list]); + List when is_list(List) -> find_type(Types, [list, string, maybe_improper_list]) + end + catch + _:_ -> + %% Types not possible to deduce with erl_parse + % If string contains variables, erl_parse:parse_term will fail, but we + % consider them valid sooo.. lets replace them with the atom var + B = [(fun({var, Anno, _}) -> {atom, Anno, var}; (Token) -> Token end)(X) || X <- A], + try + {ok, Term2} = erl_parse:parse_term(B), + case Term2 of + Tuple2 when is_tuple(Tuple2) -> find_type(Types, [tuple]); + Map2 when is_map(Map2) -> find_type(Types, [map]); + Binary2 when is_binary(Binary2) -> find_type(Types, [binary]); + List2 when is_list(List2), length(List2) > 0 -> + find_type(Types, [list, string, nonempty_list,maybe_improper_list, nonempty_improper_list]); + List2 when is_list(List2) -> find_type(Types, [list, string, maybe_improper_list]) + end + catch + _:_ -> + case A of + [{'#',_},{var,_,'Port'},{'<',_},{float,_,_},{'>',_},{dot,_}] -> find_type(Types, [port]); + [{'#',_},{var,_,'Ref'},{'<',_},{float,_,_},{'.',_},{float,_,_},{'>',_},{dot,_}] -> find_type(Types, [reference]); + [{'fun',_},{'(',_} | _] -> find_type(Types, [parameters, function, 'fun']); + [{'#',_},{var,_,'Fun'},{'<',_},{atom,_,erl_eval},{'.',_},{float,_,_},{'>',_}] -> find_type(Types, [parameters, function, 'fun']); + [{'<', _}, {float, _, _}, {'.', _}, {integer, _, _}, {'>', _}, {dot, _}] -> find_type(Types, [pid]); + [{'#', _}, {atom, _, RecordName},{'{', _}| _] -> find_type(Types, [{record, RecordName}]); + _ -> false + end + end + end. + +find_type([],_) -> false; +find_type([any|_], _) -> true; % If we find any then every type is valid +find_type([{type, any, []}|_], _) -> true; +find_type([{{parameters, _},_}|Types], ValidTypes) -> + case lists:member(parameters, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{record, _}=Type|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{Type, _}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{type, Type, _}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{type, Type, _, any}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([_|Types], ValidTypes) -> find_type(Types, ValidTypes). + +in_range(_, []) -> false; +in_range(Integer, [{type, range, [{integer, Start}, {integer, End}]}|_]) when Start =< Integer, Integer =< End -> true; +in_range(Integer, [_|Types]) -> in_range(Integer, Types). + +check_integer_type(Types, Integer) when Integer == 0 -> find_type(Types, [integer, non_neg_integer, arity]) orelse in_range(Integer, Types); +check_integer_type(Types, Integer) when Integer < 0 -> find_type(Types, [integer, neg_integer]) orelse in_range(Integer, Types); +check_integer_type(Types, Integer) when Integer > 0 -> find_type(Types, [integer, non_neg_integer, pos_integer]) orelse in_range(Integer, Types). --import(lists, [reverse/1, prefix/2]). +add_to_last_nesting(Term, Nesting) -> + Last = lists:last(Nesting), + List = lists:droplast(Nesting), + case Last of + {tuple, Args, U} -> + List ++ [{tuple, Args ++ [Term], U}]; + {list, Args, U} -> + List ++ [{list, Args ++ [Term], U}]; + {map, F, Fs, Args, U} -> + List ++ [{map, F, Fs, Args ++ [Term], U}] + end. + +close_nesting(Nesting) -> + Last = lists:last(Nesting), + case Last of + {tuple, _Args, _} -> + "}"; + {list, _Args, _} -> + "]"; + {map, _F, _Fs, _Args, _} -> + "}" + end. +expand_function_parameter_type(Mod, MFA, FunType, Args, Unfinished, Nestings, FT) -> + TypeTree = edlin_type_suggestion:type_tree(Mod, FunType, Nestings, FT), + + {Parameters, Constraints1} = case TypeTree of + {function, {{parameters, Parameters1},_}, Constraints} -> + {Parameters1, Constraints}; + {{parameters, Parameters1},_}=_F -> + {Parameters1, []} + end, + case match_arguments(TypeTree, Args) of + false -> {no, [], []}; + true when Parameters == [] -> {yes, ")", []}; + true -> + Parameter = lists:nth(length(Args)+1, Parameters), + {T, _Name} = case Parameter of + Atom when is_atom(Atom) -> {Atom, atom_to_list(Atom)}; + {var, Name1}=T1 -> {T1, atom_to_list(Name1)}; + {ann_type, {var, Name1}, T1} -> {T1, atom_to_list(Name1)}; + T1 -> {T1, edlin_type_suggestion:print_type(T1, [], [{first_only, true}])} + end, + Ts = edlin_type_suggestion:get_types(Constraints1, T, Nestings), + Types = case Ts of + [] -> []; + _ -> + SectionTypes = [S || #{}=S <- Ts], + Types1 = case [E || {_, _}=E<-Ts] of + [] -> SectionTypes; + Elems -> + case SectionTypes of + [] -> Elems; + ST -> [#{title=>"simple types", elems=>Elems, options=>[{hide, title}]}|ST] + end + end, + + [#{title=>"types", elems=>(Types1), options=>[{hide, title}]}] + end, + case Nestings of + [] -> %% Expand function type + case Unfinished of + [] -> + case T of + Atom1 when is_atom(Atom1) -> + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + {Res, Expansion, Matches} = match([], [Atom1], CC), + case Matches of + [] -> {no, [], []}; + _ -> {Res, Expansion, [#{title=>MFA, elems=>[], options=>[{highlight_param, length(Args)+1}]}]} + end; + _ when Types == [] -> + {no, [], []}; + _ -> + {no, [], [#{title=>MFA, elems=>Types, options=>[{highlight_param, length(Args)+1}]}]} + end; + {_, Word} when is_atom(T) -> + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + {Res, Expansion, Matches} = match(Word, [T], CC), + case Matches of + [] -> {no, [], []}; + _ -> {Res, Expansion, [#{title=>MFA, elems=>[], options=>[{highlight_param, length(Args)+1}]}]} + end; + {_, Word} -> + {Res, Expansion, Matches} = begin + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + Atoms1 = edlin_type_suggestion:get_atoms(Constraints1, T, Nestings), + {Res1, Expansion1, Matches1} = match(Word, Atoms1, CC), + case Matches1 of + [] -> + case match_arguments(TypeTree, Args ++ [Unfinished]) of + false -> {Res1, Expansion1, Matches1}; + true -> + {yes, CC, [{CC, []}]} + end; + _ -> + {Res1, Expansion1, Matches1} + end + end, + Match1 = case Matches of + [] -> []; + _ -> Atoms = [#{title=>"atoms", elems=>Matches, options=>[{hide, title}]}], + [#{title=>MFA, elems=>Atoms, options=>[{highlight_param, length(Args)+1}]}] + end, + {Res, Expansion,Match1} + end; + _ -> %% Expand last nesting types + expand_nesting_content(T, Constraints1, Nestings, #{title=>MFA, elems=>Types, options=>[{highlight_param, length(Args)+1}]}) + end + end. +expand_nesting_content(T, Constraints, Nestings, Section) -> + {NestingType, UnfinishedNestingArg, NestingArgs} = case lists:last(Nestings) of + {tuple, NestingArgs1, Unfinished1} -> {tuple, Unfinished1, NestingArgs1}; + {list, NestingArgs1, Unfinished1} -> {list, Unfinished1, NestingArgs1}; + {map, _, _, NestingArgs1, Unfinished1} -> {map, Unfinished1, NestingArgs1} + end, + %% in the case of + %% erlang:system_info({allocator, ) + %% we have a tuple nesting with an atom + %% this should give us "allocator" in the nestingsargs, and empty unfinished part + %% but we also know that we have a nesting, if we expect something other than a tuple, we shouldnt print that function + %% lets call it NestingType + %% now when that is fixed, how do we filter {allocator_sizes, ...} and others + Types = [Ts || Ts <- edlin_type_suggestion:get_types(Constraints, T, lists:droplast(Nestings), [no_print]) ], + case UnfinishedNestingArg of + [] -> + case find_type(Types, [NestingType]) of + true -> + %% if we know had a tuple, {allocator_sizes, } will be allowed + %% probably get_arity will return none + Nestings2 = add_to_last_nesting({var, "Var"}, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ -> {no, [], [Section]} + end + end || NestingArity <- NestingArities]); + false -> {no, [], []} + end; + {_, Word} -> + Atoms1 = edlin_type_suggestion:get_atoms(Constraints, T, Nestings), + {Res1, Expansion1, Matches1} = match(Word, Atoms1, ""), + {Res, Expansion, Matches} = case Matches1 of + [] -> + Nestings2 = add_to_last_nesting(UnfinishedNestingArg, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ when NestingType =:= tuple -> + CC = case length(NestingArgs)+1 < NestingArity of + true -> ", "; + false -> close_nesting(Nestings) + end, + {yes, CC, [{CC, []}]}; + _ when NestingType =:= list -> + {no, [], [{", ", []}, {"]", []}]}; + _ when NestingType =:= map -> + {no, [], [{", ",[]},{"}", []}]}; + _ -> + {no, [], []} + end + end || NestingArity <- NestingArities]); + [{Word2,_}] -> + Nestings2 = add_to_last_nesting({atom, Word2}, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ when NestingType =:= tuple -> + CC = case length(NestingArgs)+1 < NestingArity of + true -> ", "; + false -> close_nesting(Nestings) + end, + {yes, Expansion1++CC, [{Word2, [{ending, CC}]}]}; + _ -> + {Res1, Expansion1, Matches1} + end + end || NestingArity <- NestingArities]); + _ -> {Res1, Expansion1, Matches1} + end, + Match1 = case Matches of + [] -> []; + _ -> Atoms = [#{title=>"atoms", elems=>Matches, options=>[{hide, title}]}], + [Section#{elems:=Atoms}] + end, + {Res, Expansion, Match1} + end. + +extract_record_fields(Record, {attribute,_,record,{Record, Fields}})-> + [X || X <- [extract_record_field(F) || F <- Fields], X /= []]; +extract_record_fields(_, _)-> error. +extract_record_field({typed_record_field, {_, _,{atom, _, Field}},_})-> + Field; +extract_record_field({typed_record_field, {_, _,{atom, _, Field}, _},_})-> + Field; +extract_record_field({record_field, _,{atom, _, Field},_})-> + Field; +extract_record_field({record_field, _,{atom, _, Field}})-> + Field; +extract_record_field(_) -> []. + +fold_results([]) -> {no, [], []}; +fold_results([R|Results]) -> + lists:foldl(fun fold_completion_result/2, R, Results). + +fold_completion_result({yes, Cmp1, Matches1}, {yes, Cmp2, Matches2}) -> + {_, Cmp} = longest_common_head([Cmp1,Cmp2]), + case Cmp of + [] -> {no, [], ordsets:union([Matches1,Matches2])}; + _ -> {yes, Cmp, ordsets:union([Matches1,Matches2])} + end; +fold_completion_result({yes, Cmp, Matches}, {no, [], []}) -> + {yes, Cmp, Matches}; +fold_completion_result({no, [], []},{yes, Cmp, Matches}) -> + {yes, Cmp, Matches}; +fold_completion_result({_, _, Matches1}, {_, [], Matches2}) -> + {no, [], ordsets:union([Matches1,Matches2])}; +fold_completion_result(A, B) -> + fold_completion_result(B,A). + +expand_function_type(ModStr, FunStr, Args, Unfinished, Nestings, FT) -> + Mod = list_to_atom(ModStr), + Fun = list_to_atom(FunStr), + MinArity = if Unfinished =:= [], length(Args) =:= 0 -> 0; + true -> length(Args)+1 + end, + case [A || A <- get_arities(ModStr, FunStr, FT), A >= MinArity] of + [] -> {no, [], []}; + Arities -> + {Res, Expansion, Matches} = fold_results([begin + FunTypes = edlin_type_suggestion:get_function_type(Mod, Fun, Arity, FT), + case FunTypes of + [] -> MFA = print_function_head(ModStr, FunStr, Arity), + case Unfinished of + [] -> {no, [], [#{title=>MFA, elems=>[], options=>[]}]}; + _ -> {no, [], []} + end; + _ -> + fold_results([begin + MFA = print_function_head(ModStr, FunStr, FunType, FT), + expand_function_parameter_type(Mod, MFA, FunType, Args, Unfinished, Nestings, FT) + end || FunType <- FunTypes]) + end + end || Arity <- Arities]), + case Matches of + [] -> {Res, Expansion, Matches}; + _ -> {Res, Expansion, [#{title=>"typespecs", elems=>Matches, options=>[highlight_all]}]} + end + end. -%% expand(CurrentBefore) -> -%% {yes, Expansion, Matches} | {no, Matches} +%% Behaves like zsh +%% filters all files starting with . unless Word starts with . +%% outputs / on end of folders +expand_filepath(PathPrefix, Word) -> + Path = case PathPrefix of + [$/|_] -> PathPrefix; + _ -> + {ok, Cwd} = file:get_cwd(), + Cwd ++ "/" ++ PathPrefix + end, + ShowHidden = case Word of + "." ++ _ -> true; + _ -> false + end, + Entries = case file:list_dir(Path) of + {ok, E} -> lists:map( + fun(X)-> + case filelib:is_dir(Path ++ "/" ++ X) of + true -> X ++ "/"; + false -> X + end + end, [".."|E]); + _ -> [] + end, + EntriesFiltered = [File || File <- Entries, + case File of + [$.|_] -> ShowHidden; + _ -> true + end], + case match(Word, EntriesFiltered, []) of + {yes, Cmp, [Match]} -> + case filelib:is_dir(Path ++ "/" ++ Word ++ Cmp) of + true -> {yes, Cmp, [Match]}; + false -> {yes, Cmp ++ "\"", [Match]} + end; + X -> X + end. + +shell(Fun) -> + case shell:local_func(list_to_atom(Fun)) of + true -> "shell"; + false -> "user_defined" + end. + +shell_default_or_bif(Fun) -> + case lists:member(list_to_atom(Fun), [E || {E,_}<-get_exports(shell_default)]) of + true -> "shell_default"; + _ -> bif(Fun) + end. +bif(Fun) -> + case lists:member(list_to_atom(Fun), [E || {E,A}<-get_exports(erlang), erl_internal:bif(E,A)]) of + true -> "erlang"; + _ -> shell(Fun) + end. + +expand_string(Bef0) -> + case over_filepath(Bef0, []) of + {_, Filepath} -> + {Path, File} = split_at_last_slash(Filepath), + expand_filepath(Path, File); + _ -> {no, [], []} + end. +%% Extract a whole filepath +%% Stops as soon as we hit a double quote (") +%% and returns everything it found before stopping. +%% assumes the string is not a filepath if it contains unescaped spaces +over_filepath([],_) -> none; +over_filepath([$", $\\|Bef1], Filepath) -> over_filepath(Bef1, [$" | Filepath]); +over_filepath([$"|Bef1], Filepath) -> {Bef1, Filepath}; +over_filepath([$\ ,$\\|Bef1], Filepath) -> over_filepath(Bef1, [$\ |Filepath]); +over_filepath([$\ |_], _) -> none; +over_filepath([C|Bef1], Filepath) -> + over_filepath(Bef1, [C|Filepath]). +split_at_last_slash(Filepath) -> + {File, Path} = lists:splitwith(fun(X)->X/=$/ end, lists:reverse(Filepath)), + {lists:reverse(Path), lists:reverse(File)}. + +print_function_head(ModStr, FunStr, Arity) -> + lists:flatten(ModStr ++ ":" ++ FunStr ++ "/" ++ integer_to_list(Arity)). +print_function_head(ModStr, FunStr, FunType, FT) -> + lists:flatten(print_function_head_from_type(ModStr, FunStr, FunType, FT)). + +print_function_head1(Mod, Fun, Par, _Ret) -> + Mod++":"++Fun++"("++lists:join(", ", + [case P of + Atom when is_atom(Atom) -> atom_to_list(Atom); + {var, V} -> atom_to_list(V); + {ann_type, {var, V}, _T} -> atom_to_list(V); + T -> edlin_type_suggestion:print_type(T, [], [{first_only, true}]) + end || {_N,P} <- lists:enumerate(Par)])++")". +print_function_head_from_type(Mod, Fun, FunType, FT) -> + case edlin_type_suggestion:type_tree(list_to_atom(Mod), FunType, [], FT) of + {function, {{parameters, Parameters},{return, Return}}, _} -> + print_function_head1(Mod, Fun, Parameters, Return); + {{parameters, Parameters},{return, Return}} -> + print_function_head1(Mod, Fun, Parameters, Return) + end. + +%% expand_module_function(CurrentBefore, FT) -> {yes, Expansion, Matches} | {no, [], Matches} %% Try to expand the word before as either a module name or a function %% name. We can handle white space around the seperating ':' but the %% function name must be on the same line. CurrentBefore is reversed %% and over_word/3 reverses the characters it finds. In certain cases -%% possible expansions are printed. +%% possible expansions are printed.´´´ %% -%% The function also handles expansion with "h(" for module and functions. -expand(Bef0) -> +%% The function also handles expansion with "h(" and "ht("" for module and functions. +expand_module_function(Bef0, FT) -> {Bef1,Word,_} = edlin:over_word(Bef0, [], 0), case over_white(Bef1, [], 0) of {[$,|Bef2],_White,_Nwh} -> - {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), - {Bef4,Mod,_Nm} = edlin:over_word(Bef3, [], 0), + {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), + {Bef4,Mod,_Nm} = edlin:over_word(Bef3, [], 0), case expand_function(Bef4) of help -> - expand_function_name(Mod, Word, ","); + expand_function_name(Mod, Word, ", ", FT); + help_type -> + expand_type_name(Mod, Word, ", "); _ -> - expand_module_name(Word, ",") + fold_results(expand_helper(FT, module, Word, ":")) end; {[$:|Bef2],_White,_Nwh} -> - {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), - {_,Mod,_Nm} = edlin:over_word(Bef3, [], 0), - expand_function_name(Mod, Word, "("); - {_,_,_} -> + {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), + {_,Mod,_Nm} = edlin:over_word(Bef3, [], 0), + expand_function_name(Mod, Word, "(", FT); + {[CC, N_Esc|_], _White, _Nwh} when (CC =:= $] orelse CC =:= $) orelse CC =:= $> orelse CC =:= $} + orelse CC =:= $" orelse CC =:= $'), + N_Esc =/= $$, N_Esc =/= $- -> + {no, [], []}; + {[], _, _} -> + case Word of + [] -> {no, [], []}; %fold_results([expand_shell_default(Word), expand_user_defined_functions(FT, Word)]); + _ -> fold_results(expand_helper(FT, all, Word, ":")) + end; + {_,_,_} -> + case Word of + [] -> {no, [], []}; + _ -> + TypeOfExpand = expand_function(Bef1), CompleteChar - = case expand_function(Bef1) of - help -> ","; + = case TypeOfExpand of + help -> ", "; + help_type -> ", "; _ -> ":" end, - expand_module_name(Word, CompleteChar) + fold_results(expand_helper(FT, TypeOfExpand, Word, CompleteChar)) + end end. - +expand_keyword(Word) -> + Keywords = ["begin", "case", "of", "receive", "after", "maybe", "try", "catch", "throw", "if", "fun", "when", "end"], + {Res, Expansion, Matches} = match(Word, Keywords, ""), + case Matches of + [] -> {no, [], []}; + [{Word, _}] -> {no, [], []}; %% exact match + _ -> {Res,Expansion,[#{title=>"keywords", elems=>Matches, options=>[highlight_all]}]} + end. +expand_helper(_, help, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar)]; +expand_helper(_, help_type, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar)]; +expand_helper(FT, all, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar), expand_bifs(Word), expand_shell_default(Word), + expand_user_defined_functions(FT, Word), expand_keyword(Word)]; +expand_helper(FT, _, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar), expand_bifs(Word), + expand_user_defined_functions(FT, Word), expand_keyword(Word)]. expand_function("("++Str) -> case edlin:over_word(Str, [], 0) of {_,"h",_} -> @@ -71,68 +808,157 @@ expand_function("("++Str) -> expand_function(_) -> module. +expand_bifs(Prefix) -> + Alts = [EA || {E,A}=EA <- get_exports(erlang), erl_internal:bif(E,A)], + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"bifs", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_shell_default(Prefix) -> + Alts = get_exports(shell_default) ++ shell:local_func(), + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"commands",elems=>Matches, options=>[highlight_all]}]} + end. + +expand_user_defined_functions(FT, Prefix) -> + Alts = [{Name, Arity}||{{function, {_, Name, Arity}}, _} <- FT], + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"user_defined", elems=>Matches, options=>[highlight_all]}]} + end. + expand_module_name("",_) -> {no, [], []}; -expand_module_name(Prefix,CompleteChar) -> - match(Prefix, [{list_to_atom(M),P} || {M,P,_} <- code:all_available()], CompleteChar). +expand_module_name(Prefix,CC) -> + Alts = [{list_to_atom(M),""} || {M,_,_} <- code:all_available()], + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"modules", elems=>Matches, options=>[highlight_all]}]} + end. -expand_function_name(ModStr, FuncPrefix, CompleteChar) -> +get_arities("shell_default"=ModStr, FuncStr, FT) -> + case [A || {{function, {_, Fun, A}}, _} <- FT, Fun =:= list_to_atom(FuncStr)] of + [] -> get_arities(ModStr, FuncStr); + Arities -> Arities + end; +get_arities(ModStr, FuncStr, _) -> + get_arities(ModStr, FuncStr). +get_arities(ModStr, FuncStr) -> case to_atom(ModStr) of - {ok, Mod} -> - Exports = - case erlang:module_loaded(Mod) of - true -> - Mod:module_info(exports); - false -> - case beam_lib:chunks(code:which(Mod), [exports]) of - {ok, {Mod, [{exports,E}]}} -> - E; - _ -> - {no, [], []} - end - end, - case Exports of + {ok, Mod} -> + Exports = get_exports(Mod), + lists:sort( + [A || {H, A} <- Exports, string:equal(FuncStr, flat_write(H))]); + error -> + {no, [], []} + end. + +get_exports(Mod) -> + case erlang:module_loaded(Mod) of + true -> + Mod:module_info(exports); + false -> + case beam_lib:chunks(code:which(Mod), [exports]) of + {ok, {Mod, [{exports,E}]}} -> + E; + _ -> + [] + end + end. + +expand_function_name(ModStr, FuncPrefix, CompleteChar, FT) -> + case to_atom(ModStr) of + {ok, Mod} -> + Extra = case Mod of + shell_default -> [{Name, Arity}||{{function, {_, Name, Arity}}, _} <- FT]; + _ -> [] + end, + Exports = get_exports(Mod) ++ Extra, + {Res, Expansion, Matches}=Result = match(FuncPrefix, Exports, CompleteChar), + case Matches of + [] -> Result; + _ -> {Res, Expansion, [#{title=>"functions", elems=>Matches, options=>[highlight_all]}]} + end; + error -> + {no, [], []} + end. + +get_module_types(Mod) -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + [{T, A} || {{type, T, A},_Anno,_Sig,_Doc,_Meta} <- Docs]; + _ -> {no, [], []} + end. + +expand_type_name(ModStr, TypePrefix, CompleteChar) -> + case to_atom(ModStr) of + {ok, Mod} -> + case get_module_types(Mod) of {no, [], []} -> {no, [], []}; - Exports -> - match(FuncPrefix, Exports, CompleteChar) + Types -> + {Res, Expansion, Matches}=Result = match(TypePrefix, Types, CompleteChar), + case Matches of + [] -> Result; + _ -> {Res, Expansion, [#{title=>"types", elems=>Matches, options=>[highlight_all]}]} + end end; - error -> - {no, [], []} + error -> + {no, [], []} end. -%% if it's a quoted atom, atom_to_list/1 will do the wrong thing. to_atom(Str) -> case erl_scan:string(Str) of - {ok, [{atom,_,A}], _} -> - {ok, A}; - _ -> - error + {ok, [{atom,_,A}], _} -> + {ok, A}; + _ -> + error end. +to_list(Atom) -> + io_lib:write_atom(Atom). + +strip_quotes(Atom) -> + [C || C<-atom_to_list(Atom), C/=$']. + +match_preprocess_alt({_,_}=Alt) -> Alt; +match_preprocess_alt(X) -> {X, ""}. + match(Prefix, Alts, Extra0) -> + Alts2 = [match_preprocess_alt(A) || A <- Alts], Len = string:length(Prefix), Matches = lists:sort( - [{S, A} || {H, A} <- Alts, - prefix(Prefix, S=flat_write(H))]), + [{S, A} || {H, A} <- Alts2, + lists:prefix(Prefix, S=flat_write(H))]), + Matches2 = lists:usort( + case Extra0 of + [] -> [{S,[]} || {S,_} <- Matches]; + _ -> [{S,[{ending, Extra0}]} || {S,_} <- Matches] + end), case longest_common_head([N || {N, _} <- Matches]) of - {partial, []} -> - {no, [], Matches}; % format_matches(Matches)}; - {partial, Str} -> + {partial, []} -> + {no, [], Matches2}; + {partial, Str} -> case string:slice(Str, Len) of - [] -> - {yes, [], Matches}; % format_matches(Matches)}; - Remain -> - {yes, Remain, []} - end; - {complete, Str} -> - Extra = case {Extra0,Matches} of - {"(",[{Str,0}]} -> "()"; - {_,_} -> Extra0 - end, - {yes, string:slice(Str, Len) ++ Extra, []}; - no -> - {no, [], []} + [] -> + {yes, [], Matches2}; + Remain -> + {yes, Remain, Matches2} + end; + {complete, Str} -> + Extra = case {Extra0,Matches} of + {"/",[{Str,N}]} when is_integer(N) -> "/"++integer_to_list(N); + {"(",[{Str,0}]} -> "()"; + {_,_} -> Extra0 + end, + {yes, string:slice(Str, Len) ++ Extra, ordsets:from_list(Matches2)}; + no -> + {no, [], []} end. flat_write(T) when is_atom(T) -> @@ -140,79 +966,198 @@ flat_write(T) when is_atom(T) -> flat_write(S) -> S. -%% Return the list of names L in multiple columns. -format_matches(L) -> - {S1, Dots} = format_col(lists:sort(L), []), - S = case Dots of - true -> - {_, Prefix} = longest_common_head(vals(L)), - PrefixLen = string:length(Prefix), - case PrefixLen =< 3 of - true -> S1; % Do not replace the prefix with "...". - false -> - LeadingDotsL = leading_dots(L, PrefixLen), - {S2, _} = format_col(lists:sort(LeadingDotsL), []), - S2 - end; - false -> S1 +special_sort1([C|A], B) when C == ${ ; C == $. ; C == $# -> + special_sort1(A, B); +special_sort1(A, [C|B]) when C == ${ ; C == $. ; C == $# -> + special_sort1(A,B); +special_sort1(A,B) -> + string:lowercase(A) =< string:lowercase(B). +special_sort(#{title:=A}, #{title:=B}) -> + special_sort1(A,B); +%% Sections and elemts should not be in the same list +special_sort(#{}, {}) -> + error; +special_sort({}, #{}) -> + error; +special_sort({A,_},{B,_}) -> + special_sort1(A,B); +special_sort(A,B) -> + special_sort1(A,B). + +to_legacy_format([]) -> []; +to_legacy_format([#{title:=Title}|Rest]) when Title =:= "commands"; Title =:= "bifs" -> + to_legacy_format(Rest); +to_legacy_format([#{title:=Title, elems:=Elems}|Rest]) + when Title =:= "modules"; Title =:= "functions"; Title =:= "bindings"; + Title =:= "user_defined", Title =:= "records"; Title =:= "fields"; + Title =:= "types"; Title =:= "atoms"; Title =:= "matches"; + Title =:= "keywords"; Title =:= "typespecs" -> + Elems1 = to_legacy_format(Elems), + Elems1 ++ to_legacy_format(Rest); +to_legacy_format([#{title:=Title, elems:=_Elems}|Rest]) -> + [Title] ++ to_legacy_format(Rest); +to_legacy_format([{Val, _}|Rest]) -> + [{Val, ""}] ++ to_legacy_format(Rest). + +format_matches([], _LineWidth) -> []; +format_matches([#{}|_]=FF, LineWidth) -> + %% Group function head that have the exact same Type suggestion + Groups = maps:groups_from_list( + fun(#{title:=Title, elems:=T, options:=Opts}) -> + Separator = proplists:get_value(separator, Opts, "\n"), + case lists:last(string:split(Title++Separator, "\n", all)) of + [] -> format_section_matches(T, LineWidth); + Chars -> %% we have chars that compete with the results on the first line + Len = length(Chars), + format_section_matches(T, LineWidth, Len) + end + end, + fun(F) -> + format_title(F, LineWidth) + end, FF), + S = lists:flatten( + [lists:join("", F)++Matches || + {Matches, F}<-lists:sort(fun({_,A},{_,B}) -> A =< B end, maps:to_list(Groups))]), + lists:flatten(string:trim(S, trailing)++"\n"); +format_matches(Elems, LineWidth) -> + S = format_section_matches1(Elems, LineWidth, 0), + lists:flatten(string:trim(S, trailing)++"\n"). +format_title(#{title:=MFA, options:=Options}, _LineWidth) -> + case proplists:get_value(hide, Options) of + title -> ""; + _ -> + Separator = proplists:get_value(separator, Options, "\n"), + HighlightAll = proplists:is_defined(highlight_all, Options), + case HighlightAll of + true -> "\033[;1;4m"++MFA++"\033[0m"++Separator; + _ -> + HighlightParam = proplists:get_value(highlight_param, Options, false), + + MFA2 = case HighlightParam of + false -> MFA; + _ -> + PreviousParams = HighlightParam -1, + TuplePattern = "(?:\\{[^\\}]+\\})", + AtomVarPattern = "(?:\\w+)", + TypePattern="(?:(?:"++AtomVarPattern++":)?(?:"++AtomVarPattern++"\\(\\))(?:\\s[><=]+\\s\\d+)?)", + SimplePatterns = "(?:"++TuplePattern++"|"++TypePattern++"|"++AtomVarPattern++")", + UnionPattern = "(?:"++SimplePatterns++"(?:\\s\\|\\s"++SimplePatterns++")*)", + FunPattern="(?:fun\\(\\(" ++ UnionPattern ++ "\\)\\s*->\\s*" ++ UnionPattern ++ "\\))", + ArgPattern3 = "(?:"++FunPattern++"|"++UnionPattern++")", + PrevArgs="(?:"++ArgPattern3++",\\s){"++integer_to_list(PreviousParams) ++ "}", + FunctionHeadStart="^([^\\(]+\\("++PrevArgs++")", %% \\1 + + HighlightArg="("++ArgPattern3++")", %\\2 + NextArgs="(?:,\\s"++ArgPattern3++")*", + FunctionHeadEnd="("++NextArgs++"\\)(?:.*))$", % \\3 + + re:replace(MFA, + FunctionHeadStart ++ HighlightArg ++ FunctionHeadEnd, + "\\1\033[;1;4m\\2\033[0m\\3", + [global, {return, list}, unicode]) + end, + Highlight = proplists:get_value(highlight, Options, false), + case Highlight of + false -> MFA2; + _ -> re:replace(MFA2, "(\\Q"++Highlight++"\\E)", "\033[;1;4m\\1\033[0m", [global, {return, list}, unicode]) + end ++ Separator + end + end; +format_title(_Elems, _LineWidth) -> + %% not a section, old interface + %% output empty list + "". + +format_section_matches(LS, LineWidth) -> format_section_matches(LS, LineWidth, 0). +format_section_matches([], _, _) -> "\n"; +format_section_matches([#{}|_]=FF, LineWidth, Acc) -> + Groups = maps:groups_from_list( + fun(#{title:=Title, elems:=T, options:=Opts}) -> + Separator = proplists:get_value(separator, Opts, "\n"), + case lists:last(string:split(Title++Separator, "\n", trailing)) of + [] -> format_section_matches(T, LineWidth); + Chars -> %% we have chars that compete with the results on the first line + Len = string:length(Chars), + format_section_matches(T, LineWidth, Len+Acc) + end end, - ["\n" | S]. + fun(F) -> + format_title(F, LineWidth) + end, FF), + lists:flatten( + [lists:join("", F)++Matches || + {Matches, F}<-lists:sort(fun({_,A},{_,B}) -> A =< B end, maps:to_list(Groups))]); +format_section_matches(Elems, LineWidth, Acc) -> + format_section_matches1(Elems, LineWidth, Acc). -format_col([], _) -> []; -format_col(L, Acc) -> - LL = 79, - format_col(L, field_width(L, LL), 0, Acc, LL, false). +format_section_matches1([], _, _) -> []; +format_section_matches1(LS, LineWidth, Len) -> + L = lists:usort(fun special_sort/2, ordsets:to_list(LS)), + Opt = case Len == 0 of + true -> []; + false -> [{title, Len}] + end, + S1 = format_col(Opt ++ L, field_width(Opt ++ L, LineWidth), Len, [], LineWidth, Opt), + S2 = lists:map( + fun(Line) -> + case string:length(Line) of + Len1 when Len1 > LineWidth -> + string:sub_string(Line, 1, LineWidth-4) ++ "...\n"; + _ -> Line + end + end,S1), + lists:flatten(string:trim(S2, trailing)++"\n"). -format_col(X, Width, Len, Acc, LL, Dots) when Width + Len > LL -> - format_col(X, Width, 0, ["\n" | Acc], LL, Dots); -format_col([A|T], Width, Len, Acc0, LL, Dots) -> +format_col(X, Width, Len, Acc, LL, Opt) when Width + Len > LL -> + format_col(X, Width, 0, ["\n" | Acc], LL, Opt); +format_col([{title,TitleLen}|T], Width, Len, Acc0, LL, Opt) -> + Acc = [io_lib:format("~-*ts", [Width-TitleLen, ""])|Acc0], + format_col(T, Width, Len+Width, Acc, LL, Opt); +format_col([A|T], Width, Len, Acc0, LL, _Opt) -> {H0, R} = format_val(A), - Hmax = LL - length(R), - {H, NewDots} = + Hmax = LL - string:length(R), + {H, _} = case string:length(H0) > Hmax of true -> {io_lib:format("~-*ts", [Hmax - 3, H0]) ++ "...", true}; - false -> {H0, Dots} + false -> {H0, false} end, Acc = [io_lib:format("~-*ts", [Width, H ++ R]) | Acc0], - format_col(T, Width, Len+Width, Acc, LL, NewDots); -format_col([], _, _, Acc, _LL, Dots) -> - {lists:reverse(Acc, "\n"), Dots}. + format_col(T, Width, Len+Width, Acc, LL, []); +format_col([], _, _, Acc, _LL, _Opt) -> + lists:reverse(Acc). +format_val({H, L}) when is_list(L) -> + {H, proplists:get_value(ending, L, "")}; format_val({H, I}) when is_integer(I) -> - %% If it's a tuple {string(), integer()}, we assume it's an - %% arity, and meant to be printed. - {H, "/" ++ integer_to_list(I)}; + {H, "/"++integer_to_list(I)}; format_val({H, _}) -> {H, ""}; format_val(H) -> {H, ""}. field_width(L, LL) -> field_width(L, 0, LL). - -field_width([{H,_}|T], W, LL) -> - case string:length(H) of +field_width([{title, Len}|T], W, LL) -> + case Len of L when L > W -> field_width(T, L, LL); _ -> field_width(T, W, LL) end; field_width([H|T], W, LL) -> - case string:length(H) of + {H1, Ending} = format_val(H), + case string:length(H1++Ending) of L when L > W -> field_width(T, L, LL); _ -> field_width(T, W, LL) end; -field_width([], W, LL) when W < LL - 3 -> +field_width([], W, LL) when W < LL -> W + 4; field_width([], _, LL) -> LL. -vals([]) -> []; -vals([{S, _}|L]) -> [S|vals(L)]; -vals([S|L]) -> [S|vals(L)]. - -leading_dots([], _Len) -> []; -leading_dots([{H, I}|L], Len) -> - [{"..." ++ string:slice(H, Len), I}|leading_dots(L, Len)]; -leading_dots([H|L], Len) -> - ["..." ++ string:slice(H, Len)|leading_dots(L, Len)]. +number_matches([#{ elems := Matches }|T]) -> + number_matches(Matches) + number_matches(T); +number_matches([_|T]) -> + 1 + number_matches(T); +number_matches([]) -> + 0. %% Strings are handled naively, but it should be OK here. longest_common_head([]) -> @@ -221,24 +1166,23 @@ longest_common_head(LL) -> longest_common_head(LL, []). longest_common_head([[]|_], L) -> - {partial, reverse(L)}; + {partial, lists:reverse(L)}; longest_common_head(LL, L) -> case same_head(LL) of - true -> - [[H|_]|_] = LL, - LL1 = all_tails(LL), - case all_nil(LL1) of - false -> - longest_common_head(LL1, [H|L]); - true -> - {complete, reverse([H|L])} - end; - false -> - {partial, reverse(L)} + true -> + [[H|_]|_] = LL, + LL1 = all_tails(LL), + case all_nil(LL1) of + false -> + longest_common_head(LL1, [H|L]); + true -> + {complete, lists:reverse([H|L])} + end; + false -> + {partial, lists:reverse(L)} end. same_head([[H|_]|T1]) -> same_head(H, T1). - same_head(H, [[H|_]|T]) -> same_head(H, T); same_head(_, []) -> true; same_head(_, _) -> false. diff --git a/lib/stdlib/src/edlin_type_suggestion.erl b/lib/stdlib/src/edlin_type_suggestion.erl new file mode 100644 index 0000000000..7a37f7ea6b --- /dev/null +++ b/lib/stdlib/src/edlin_type_suggestion.erl @@ -0,0 +1,487 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_type_suggestion). +-include_lib("kernel/include/eep48.hrl"). +-export([type_tree/4, get_arity/3, get_atoms/3, get_types/3, get_types/4, get_function_type/4, print_type/3]). + + +%% type_tree/4 returns a unwrapped and trimmed type specification containing +%% all the valid 'types' that are valid in each function parameter of a function or +%% each record field of a record. +%% +%% Translates the spec AST to a structure that resembles the AST but trimmed of unneeded data. +%% User types and remote types are fetched and embedded in the structure depending on requested +%% level of unnestling. +%% Unions are flattened. +%% Visited is used to prevent infinite loops when looking up a recursive / cyclic type +%% FT is a map of type {{type, Type}, {attribute,_,type,{_,TypeAST,_}}}) +type_tree(Mod, FunType, Nestings, FT) -> + %% TODO when FT is updated we would be getting incorrect results because of cache + %% we should probably store a "dirty bit" in the table to make this aware that + %% a new result should be calculated. Preferably only types that depend on the table + %% would be invalidated, but it may turn advanced. + %% TODO look this over to make sure we don't do unnecessary work, + case get({type_traverser, Mod, FunType, Nestings}) of + undefined -> Res = type_traverser_cache(Mod, FunType, #{}, length(Nestings)+1, FT), + put({type_traverser, Mod, FunType, Nestings}, Res), + Res; + Res -> Res + end. +type_traverser_cache(Mod, T, Visited, Level, FT) -> + case get({Mod, T, Level}) of + undefined -> + Res = type_traverser(Mod, T, Visited, Level, FT), + put({Mod, T, Level}, Res), + Res; + Res -> Res + end. + +type_traverser(Mod, {type, _, bounded_fun, [Fun, Constraints]}, Visited, Level, FT) -> + Cl = [type_traverser(Mod,X,Visited, Level, FT) || X <- Constraints], + F = type_traverser(Mod, Fun, Visited, Level, FT), + {function, F, Cl}; +type_traverser(Mod, {type, _, 'fun', [Product, Return]}, Visited, Level, FT) -> + P = type_traverser(Mod, Product, Visited, Level, FT), + R = type_traverser(Mod, Return, Visited, Level, FT), + {P, {return, R}}; +type_traverser(Mod, {type, _, product, Childs}, Visited, Level, FT) -> + Cl = [type_traverser(Mod, X, Visited, Level, FT) || X <- Childs], + {parameters, Cl}; +type_traverser(Mod, {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]}, Visited, Level, FT) -> + {constraint, type_traverser(Mod, Type1, Visited, Level, FT), type_traverser(Mod, Type2, Visited, Level, FT)}; +type_traverser(_, {var, _, Name}, _Visited, _Level, _FT) -> + {var, Name}; +type_traverser(_Mod,{type, _, map, any}, _Visited, _Level, _FT) -> + {type, map, []}; +type_traverser(Mod, {type, _, map, Params}, Visited, Level, FT) -> + {map, [type_traverser(Mod, X, Visited, Level-1, FT) || X <- Params]}; +type_traverser(Mod, {type, _, map_field_exact, [Type1, Type2]}, Visited, Level, FT) -> + {map_field_exact, type_traverser(Mod,Type1, Visited, Level, FT), type_traverser(Mod,Type2, Visited, Level, FT)}; +type_traverser(Mod, {type, _, map_field_assoc, [Type1, Type2]}, Visited, Level, FT) -> + {map_field_assoc, type_traverser(Mod,Type1, Visited, Level, FT), type_traverser(Mod,Type2, Visited, Level, FT)}; +type_traverser(_Mod, {atom, _, Atom}, _Visited, _Level, _FT) when is_atom(Atom) -> + Atom; +type_traverser(Mod, {op, _, Op, Type}, Visited, Level, FT) -> + {op, Op, type_traverser(Mod, Type, Visited, Level, FT)}; +type_traverser(Mod, {op, _, Op, Type1, Type2}, Visited, Level, FT) -> + {op, Op, type_traverser(Mod, Type1, Visited, Level, FT), type_traverser(Mod, Type2, Visited, Level, FT)}; +type_traverser(_Mod, {integer, _, Int}, _Visited, _Level, _FT) -> + {integer, Int}; +type_traverser(Mod, {type, _, list, [ChildType]}, Visited, Level, FT) -> + {list, type_traverser(Mod, ChildType, Visited, Level-1, FT)}; +type_traverser(_Mod, {type, _, tuple, any}, _Visited, _Level, _FT) -> + {type, tuple, []}; +type_traverser(Mod, {type, _, tuple, ChildTypes}, Visited, Level, FT) -> + {tuple, [type_traverser(Mod, X, Visited, Level-1, FT) || X <- ChildTypes]}; +type_traverser(Mod, {type, _, union, ChildTypes}, Visited, Level, FT) -> + Childs = [type_traverser(Mod, X, Visited, Level, FT) || X <- ChildTypes], + ChildsFiltered = [X || X <- Childs, X/=undefined], + {UnionChilds, NonUnionChilds} = lists:partition( + fun(X) -> + case X of + {union, _} -> true; + _ -> false + end + end, ChildsFiltered), + ChildsFlattened = lists:flatten([T || {union, T} <- UnionChilds]) ++ NonUnionChilds, + {union, ChildsFlattened}; +type_traverser(Mod, {ann_type,_,[T1,T2]}, Visited, Level, FT) -> + {ann_type, type_traverser(Mod, T1, Visited, Level, FT), type_traverser(Mod, T2, Visited, Level, FT)}; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, Level, FT) when 0 >= Level -> + %% when we have level 0, do not traverse the type further, just print it + case maps:is_key(strip_anno(T), Visited) of + false -> + {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, 0, FT) || P <- Params]}; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type,_,[{_,_,Mod},{_,_,Name}, Params]}=T, Visited, Level, FT) when 0 >= Level -> + case maps:is_key(strip_anno(T), Visited) of + false -> + {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, 0, FT) || P <- Params]}; + true -> {type, Mod, Name, []} + end; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, 1=Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> {user_type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params], type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT)} + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type,_,[{_,_,Mod},{_,_,Name}, Params]}=T, Visited, 1=Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> {user_type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params], type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT)} + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), Level}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT) + end, + put({strip_anno(T), Level}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type, _, [{_,_,Mod},{_,_,Name}, Params]}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), Level}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT) + end, + put({strip_anno(T), Level}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {type, _, record, [{atom, _, Record}]}, _Visited, _Level, _FT) -> + {record, Record}; +type_traverser(_, {type, _, Name, any}, _, _, _) -> + {type, Name, []}; +type_traverser(_, {type, _, term}, _, _, _) -> + {type, any, []}; +type_traverser(_, {type, _, Name}, _, _, _) -> + {type, Name, []}; +type_traverser(_, {type, _, term, _}, _, _, _) -> + {type, any, []}; +type_traverser(_, {type, _, Name, Params}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(erlang, Name, length(Params), FT) of + hidden -> {type, Name, [type_traverser(erlang, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(erlang, Type, Visited#{ strip_anno(T) => true}, Level, FT) + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Name, []} + end. + +strip_anno({A, _, B}) -> {A, B}; +strip_anno({A, _, B, C}) -> {A, B, C}. + +simplified_type(erlang, binary, 0) -> {type, undefined, binary, []}; +simplified_type(erlang, char, 0) -> {type, undefined, char, []}; +simplified_type(erlang, iolist, 0) -> {type, undefined, iolist, []}; +simplified_type(erlang, string, 0) -> {type, undefined, string, []}; +simplified_type(unicode, chardata, 0) -> {type, erlang, string, []}; +simplified_type(file, filename_all, 0) -> {type, erlang, string, []}; +simplified_type(file, filename, 0) -> {type, erlang, string, []}; +simplified_type(file, name_all, 0) -> {type, erlang, string, []}; +simplified_type(file, name, 0) -> {type, erlang, string, []}; +simplified_type(_Module, _TypeName, _Arity) -> none. + +lookup_type(Mod, Type, Arity, FT) -> + case simplified_type(Mod, Type, Arity) of + none -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + FnFunctions = + lists:filter(fun({{type, T, A},_Anno,_Sig,_Doc,_Meta}) -> + T =:= Type andalso A =:= Arity; + (_) -> + false + end, Docs), + case FnFunctions of + [] -> + case [TypeAST || {{type, Type2}, {attribute,_,type,{_,TypeAST,_}}} <- FT, Type2 =:= Type] of + [] -> hidden; %% can be an opaque type or missing type + [SingleTypeAST] -> SingleTypeAST + end; + [{_,_,_,_,#{signature := [{attribute,_,type,{_,TypeAST,_}}]}}] -> TypeAST + end; + _ -> + case [TypeAST || {{type, Type2}, {attribute,_,type,{_,TypeAST,_}}} <- FT, Type2 =:= Type] of + [] -> hidden; %% can be an opaque type or missing type + [SingleTypeAST] -> SingleTypeAST + end + end; + T -> T + end. +get_function_type(Mod, Fun, Arity, FT) -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + R = lists:flatten([FunTypes || + {{function, F, A},_Anno,_Sig,_Doc, #{ signature := [{attribute,_,spec,{_,FunTypes}}]}} <- Docs, + F =:= Fun, A =:= Arity]), + case {Mod, R} of + {shell_default, []} -> + lists:flatten([FunTypes || + {{function_type, {shell_default, F, A}},{attribute,_,spec,{_,FunTypes}}} <- FT, + F =:= Fun, A =:= Arity]); + _ -> R + end; + _ when Mod =:= shell_default -> + lists:flatten([FunTypes || {{function_type, {shell_default, F, A}},{attribute,_,spec,{_,FunTypes}}} <- FT, + F =:= Fun, A =:= Arity]); + _ -> [] + end. +get_arity(Constraints, Type, Nestings) -> + case get_arity1(Type, Constraints, Nestings) of + List when is_list(List) -> List; + Val -> [Val] + end. +get_arity1({var, _Var}=C, Constraints, Nestings) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_arity1(T, Constraints, Nestings); + _ -> none + end; +get_arity1({list, _T}, _Constraints, [{'list', _, _}]) -> + 99; %% Can be higher, but probably do not need completion for that +get_arity1({list, T}, Constraints, [{'list', _, _}|Nestings]) -> + get_arity1(T, Constraints, Nestings); +get_arity1({tuple, LT}, Constraints, [{'tuple', Args, _}]) when length(LT) >= length(Args) -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> length(LT); + false -> + none + end; +get_arity1({tuple, LT}, Constraints, [{'tuple', Args, _}|Nestings]) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> get_arity1(lists:nth(length(Args)+1, LT), Constraints, Nestings); + false -> none + end; +get_arity1({map, Types}, _Constraints, [{'map', _Keys, [], _, _}]) -> + length(Types); +get_arity1({map, Types}, _Constraints, [{'map', _Keys, _Key, _, _}]) -> + length(Types); +get_arity1({map, Types}, Constraints, [{'map', Keys, [], _, _}|Nestings]) -> + lists:flatten([get_arity1(T, Constraints, Nestings) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_arity1({map, Types}, Constraints, [{'map', _Keys, Key, _, _}|Nestings]) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> none; + [Type] -> get_arity1(Type, Constraints, Nestings) + end; +get_arity1({map_field_assoc, K, _V}, C, Nestings) -> + get_arity1(K, C, Nestings); +get_arity1({map_field_exact, K, _V}, C, Nestings) -> + get_arity1(K, C, Nestings); +get_arity1({union, Types}, Constraints, Nestings) -> + Arities = [get_arity1(T, Constraints, Nestings) || T <- Types], + [X || X <- lists:flatten(Arities), X/=none]; +get_arity1({ann_type, _Var, Type}, Constraints, Nestings) -> + get_arity1(Type, Constraints, Nestings); +get_arity1({user_type, _, _, _, Type}, Constraints, Nestings) -> + get_arity1(Type, Constraints, Nestings); +get_arity1(_, _, _) -> + none. + +%% get_atoms returns the valid atoms in the current context as a list +get_atoms(Constraints, Type, Nestings) -> + case get_atoms1(Type, Constraints, Nestings) of + List when is_list(List) -> [io_lib:write_atom(Atom) || Atom <- List]; + Atom when is_atom(Atom) -> [io_lib:write_atom(Atom)] + end. +get_atoms1({var, _Var}=C, Constraints, Nestings) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_atoms1(T, Constraints, Nestings); + _ -> [] + end; +get_atoms1({list, T}, Constraints, [{'list', _, _}|Nestings]) -> + get_atoms1(T, Constraints, Nestings); +get_atoms1({tuple, LT}, Constraints, [{'tuple', Args, _}|Nestings]) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> get_atoms1(lists:nth(length(Args)+1, LT), Constraints, Nestings); + false -> [] + end; +get_atoms1({map, Types}, Constraints, [{'map', Keys, [], _, _}|Nestings]) -> + lists:flatten([get_atoms1(T, Constraints, Nestings) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_atoms1({map, Types}, Constraints, [{'map', _Keys, Key, _, _}|Nestings]) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> []; + [Type] -> get_atoms1(Type, Constraints, Nestings) + end; +get_atoms1({map_field_assoc, K, _V}, C, Nestings) -> + get_atoms1(K, C, Nestings); +get_atoms1({map_field_exact, K, _V}, C, Nestings) -> + get_atoms1(K, C, Nestings); +get_atoms1( {union, Types}, Constraints, Nestings) -> + Atoms = [get_atoms1(T, Constraints, Nestings) || T <- Types], + [X || X <- lists:flatten(Atoms), X/=[]]; +get_atoms1(Atom, _Constraints, []) when is_atom(Atom) -> + Atom; +get_atoms1({user_type, _, _, _, Type}, Constraints, Nestings) -> + get_atoms1(Type, Constraints, Nestings); +get_atoms1(_, _, _) -> + []. + +get_types(Constraints, T, Nestings) -> + get_types(Constraints, T, Nestings,[]). +get_types(Constraints, T, Nestings, Options) -> + MaxUserTypeExpansions = 1, + case get_types1(T, Constraints, Nestings, MaxUserTypeExpansions, Options) of + [] -> []; + [_|_]=Types -> [Type || Type <- Types, Type /= []]; + Type -> [Type] + end. +get_types1({var, _Var}=C, Constraints, Nestings, MaxUserTypeExpansions, Options) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_types1(T, Constraints, Nestings, MaxUserTypeExpansions, Options); + _ -> [] + end; +get_types1({union, Types}, Cs, Nestings, MaxUserTypeExpansions, Options) -> + lists:flatten([get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options) || T <- Types]); + +get_types1({list, T}, Cs, [{list, _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options); +get_types1({tuple, LT}, Cs, [{tuple, Args, _}|Nestings], MaxUserTypeExpansions, Options) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Cs, Args) of + true -> get_types1(lists:nth(length(Args)+1, LT), Cs, Nestings, MaxUserTypeExpansions, Options); + false -> [] + end; +get_types1({'map', Types}, Cs, [{'map', Keys, [], _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + lists:flatten([get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_types1({'map', Types}, Cs, [{'map', _, Key, _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> []; + [Type] -> get_types1(Type, Cs, Nestings, MaxUserTypeExpansions, Options) + end; +get_types1({user_type, _Mod, _Name, _Params, Type}, Cs, Nestings, MaxUserTypeExpansions, [no_print]=Options) when MaxUserTypeExpansions > 0 -> + lists:flatten([get_types1(Type, Cs, Nestings, MaxUserTypeExpansions-1, Options)]); +get_types1({user_type, _, _, _, Type}, Cs, Nestings, 0, [no_print]=Options) -> + get_types1(Type, Cs, Nestings, 0, Options); +get_types1({ann_type, _Var, T}, Cs, Nestings, MaxUserTypeExpansions, [no_print]) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, [no_print]); +get_types1({ann_type, _Var, _T}=Type, Cs, [], _MaxUserTypeExpansions, []) -> + {print_type(Type, Cs), ""}; +get_types1({ann_type, _Var, T}, Cs, Nestings, MaxUserTypeExpansions, []) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, []); +get_types1(Type, _Cs, [], _, [no_print]) -> + Type; +get_types1({user_type, Mod, Name, Params, Type}, Cs, Nestings, MaxUserTypeExpansions, []) when MaxUserTypeExpansions > 0 -> + Title = print_type({type, Mod, Name, Params}, Cs, []), + Elems = lists:flatten([get_types1(Type, Cs, Nestings, MaxUserTypeExpansions-1, [])]), + #{title=>Title, elems=>Elems, options=>[{separator, " :: "}, {highlight_all}]}; +get_types1({user_type, _, _, _, Type}, Cs, Nestings, 0, []) -> + get_types1(Type, Cs, Nestings, 0, []); +get_types1(Type, Cs, [],_, []) -> + {print_type(Type, Cs), ""}; +get_types1(_, _, _, _, _) -> []. + +get_constraint(Type, Constraints) -> + case [ X || {constraint, T, _}=X <- Constraints, T == Type] of + [C|_] -> C; + [] -> [] + end. + +print_type(Type, Constraints) -> + lists:flatten(print_type(Type, Constraints, [], [])). +print_type(Type, Constraints, Options) -> + lists:flatten(print_type(Type, Constraints, [], Options)). +print_type({var, Name}=Var, Constraints, Visited, Options) -> + case lists:member(Var, Visited) of + true -> atom_to_list(Name); + false -> + case get_constraint(Var, Constraints) of + {constraint, _, T2} -> print_type(T2, Constraints, [Var| Visited], Options); + _ -> atom_to_list(Name) + end + end; +print_type(Atom, _Cs, _V, _) when is_atom(Atom) -> io_lib:write_atom(Atom); +print_type({{parameters, Ps}, {return, R}}, Cs, V, Options) -> + "fun(("++lists:join(", ", [print_type(X, Cs, V, Options) || X <- Ps]) ++ ") -> " ++ print_type(R, Cs, V, Options) ++ ")"; +print_type({list, Type}, Cs, V, Options)-> + "[" ++ print_type(Type, Cs, V, Options) ++ "]"; +print_type({tuple, Types}, Cs, V, Options) when is_list(Types) -> + Types1 = [print_type(X, Cs, V, Options) || X <- Types], + case Types1 of + [] -> "{}"; + _ -> "{"++ lists:nth(1, Types1) ++ ", ...}" + end; +print_type({ann_type, Var, Type}, Cs, V, Options) -> + print_type(Var, Cs, V, Options) ++ " :: " ++ print_type(Type, Cs, V, Options); +print_type({map, Types}, Cs, V, Options) -> + Types1 = [print_type(X, Cs, V, Options) || X <- Types], + "#{"++lists:join(", ", Types1) ++ "}"; +print_type({map_field_assoc, Type1, Type2}, Cs, V, Options) -> + print_type(Type1, Cs, V, Options) ++ "=>" ++ print_type(Type2, Cs, V, Options); +print_type({map_field_exact, Type1, Type2}, Cs, V, Options) -> + print_type(Type1, Cs, V, Options) ++ ":=" ++ print_type(Type2, Cs, V, Options); +print_type({integer, Int}, _Cs, _V, _) -> + integer_to_list(Int); +print_type({op, Op, Type}, Cs, V, Options) -> + "op ("++atom_to_list(Op)++" "++print_type(Type, Cs, V, Options)++")"; +print_type({op, Op, Type1, Type2}, Cs, V, Options) -> + "op ("++print_type(Type1, Cs, V, Options)++" "++atom_to_list(Op)++" "++print_type(Type2, Cs, V, Options)++")"; +print_type({record, Record}, _Cs, _V, _) -> + "#" ++ atom_to_list(Record); +print_type({type, range, [{integer, Int1},{integer, Int2}]}, _Cs, _V, _) -> + integer_to_list(Int1) ++ ".." ++ integer_to_list(Int2); +print_type({type, non_neg_integer, []}, _Cs, _V, _) -> + "integer() >= 0"; +print_type({type, neg_integer, []}, _Cs, _V, _) -> + "integer() < 0"; +print_type({type, pos_integer, []}, _Cs, _V, _) -> + "integer() > 0"; +print_type({type, Name, []}, _Cs, _V, _) -> + atom_to_list(Name)++"()"; +print_type({type, Name, Params}, _Cs, _V, _) -> + atom_to_list(Name) ++ "(" ++ lists:join(", ",[ extract_param(P) || P <- Params]) ++ ")"; +print_type({union, Types}, Cs, V, Options) -> + lists:join(" | ", [print_type(X, Cs, V, Options) || X <- Types]); +print_type({type, Mod, Name, Params}, _Cs, _V, _) -> + atom_to_list(Mod) ++ ":" ++ atom_to_list(Name) ++ + "(" ++ lists:join(", ", [extract_param(P) || P <- Params]) ++ ")"; +print_type({user_type, Mod, Name, Params, Type}, Cs, V, Options) -> + First = proplists:get_value(first_only, Options, false), + case First of + true -> print_type({type, Mod, Name, Params}, Cs, V, Options); + _ -> print_type({type, Mod, Name, Params}, Cs, V, Options) ++ " :: " ++ print_type(Type, Cs, V, Options) + end; +print_type(_,_,_,_) -> atom_to_list(unknown). + + +extract_param({var, Var}) -> + atom_to_list(Var); +extract_param({integer, Value}) -> + io_lib:format("~p",[Value]); +extract_param({type, Type,_}) -> + io_lib:format("~p", [Type]); +extract_param(T)-> + print_type(T, []). diff --git a/lib/stdlib/src/erl_expand_records.erl b/lib/stdlib/src/erl_expand_records.erl index 83f21670c2..d15460c0ba 100644 --- a/lib/stdlib/src/erl_expand_records.erl +++ b/lib/stdlib/src/erl_expand_records.erl @@ -692,24 +692,20 @@ record_wildcard_init([]) -> none. record_update(R, Name, Fs, Us0, St0) -> Anno = element(2, R), {Pre,Us,St1} = record_exprs(Us0, St0), - Nf = length(Fs), %# of record fields - Nu = length(Us), %# of update fields - Nc = Nf - Nu, %# of copy fields %% We need a new variable for the record expression %% to guarantee that it is only evaluated once. {Var,St2} = new_var(Anno, St1), + %% Honor the `strict_record_updates` option needed by `dialyzer`, otherwise + %% expand everything to chains of `setelement/3` as that's far more + %% efficient in the JIT. StrictUpdates = strict_record_updates(St2#exprec.compile), - - %% Try to be intelligent about which method of updating record to use. {Update,St} = if - Nu =:= 0 -> - record_match(Var, Name, Anno, Fs, Us, St2); - Nu =< Nc, not StrictUpdates -> %Few fields updated + not StrictUpdates, Us =/= [] -> {record_setel(Var, Name, Fs, Us), St2}; - true -> %The wide area inbetween + true -> record_match(Var, Name, Anno, Fs, Us, St2) end, {{block,Anno,Pre ++ [{match,Anno,Var,R},Update]},St}. diff --git a/lib/stdlib/src/erl_features.erl b/lib/stdlib/src/erl_features.erl index 625a9d7952..ac2ecd3c37 100644 --- a/lib/stdlib/src/erl_features.erl +++ b/lib/stdlib/src/erl_features.erl @@ -18,6 +18,7 @@ %% %CopyrightEnd% %% -module(erl_features). +-feature(maybe_expr, enable). -export([all/0, configurable/0, @@ -371,7 +372,7 @@ init_features() -> end, FOps = lists:filtermap(F, FeatureOps), {Features, _, _} = collect_features(FOps), - {Enabled, Keywords} = + {Enabled0, Keywords} = lists:foldl(fun(Ftr, {Ftrs, Keys}) -> case lists:member(Ftr, Ftrs) of true -> @@ -385,11 +386,15 @@ init_features() -> Features), %% Save state + Enabled = lists:uniq(Enabled0 ++ permanently_enabled_in_erts()), enabled_features(Enabled), set_keywords(Keywords), persistent_term:put({?MODULE, init_done}, true), ok. +permanently_enabled_in_erts() -> + [maybe_expr]. + init_specs() -> Specs = case os:getenv("OTP_TEST_FEATURES") of "true" -> test_features(); @@ -428,25 +433,18 @@ set_keywords(Words) -> %% {not_allowed, <list of not enabled features>}. -spec load_allowed(binary()) -> ok | {not_allowed, [feature()]}. load_allowed(Binary) -> - case erts_internal:beamfile_chunk(Binary, "Meta") of - undefined -> - ok; - Meta -> - MetaData = erlang:binary_to_term(Meta), - case proplists:get_value(enabled_features, MetaData) of - undefined -> - ok; - Used -> - Enabled = enabled(), - case lists:filter(fun(UFtr) -> - not lists:member(UFtr, Enabled) - end, - Used) of - [] -> ok; - NotEnabled -> - {not_allowed, NotEnabled} - end - end + maybe + Meta = erts_internal:beamfile_chunk(Binary, "Meta"), + true ?= Meta =/= undefined, + MetaData = binary_to_term(Meta), + Used = proplists:get_value(enabled_features, MetaData, []), + Enabled = enabled(), + NotEnabled = [UFtr || UFtr <- Used, + not lists:member(UFtr, Enabled)], + [_|_] ?= NotEnabled, + {not_allowed, NotEnabled} + else + _ -> ok end. %% Return features used by module or beam file diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 84cbab7dfd..da0b6c67b8 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -21,6 +21,7 @@ %% Do necessary checking of Erlang code. -module(erl_lint). +-feature(maybe_expr, enable). -export([module/1,module/2,module/3,format_error/1]). -export([exprs/2,exprs_opt/3,used_vars/2]). % Used from erl_eval.erl. @@ -436,12 +437,8 @@ format_error({undefined_type, {TypeName, Arity}}) -> io_lib:format("type ~tw~s undefined", [TypeName, gen_type_paren(Arity)]); format_error({unused_type, {TypeName, Arity}}) -> io_lib:format("type ~tw~s is unused", [TypeName, gen_type_paren(Arity)]); -format_error({new_builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a new builtin type; " - "its (re)definition is allowed only until the next release", - [TypeName, gen_type_paren(Arity)]); -format_error({builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a builtin type; it cannot be redefined", +format_error({redefine_builtin_type, {TypeName, Arity}}) -> + io_lib:format("local redefinition of built-in type: ~w~s", [TypeName, gen_type_paren(Arity)]); format_error({renamed_type, OldName, NewName}) -> io_lib:format("type ~w() is now called ~w(); " @@ -674,7 +671,10 @@ start(File, Opts) -> true, Opts)}, {keyword_warning, bool_option(warn_keywords, nowarn_keywords, - false, Opts)} + false, Opts)}, + {redefined_builtin_type, + bool_option(warn_redefined_builtin_type, nowarn_redefined_builtin_type, + true, Opts)} ], Enabled1 = [Category || {Category,true} <- Enabled0], Enabled = ordsets:from_list(Enabled1), @@ -2084,7 +2084,8 @@ bit_size_check(Anno, all, #bittype{type=Type}, St) -> binary -> {all,St}; _ -> {unknown,add_error(Anno, illegal_bitsize, St)} end; -bit_size_check(Anno, Size, #bittype{type=Type,unit=Unit}, St) -> +bit_size_check(Anno, Size, #bittype{type=Type,unit=Unit}, St) + when is_integer(Size), is_integer(Unit) -> Sz = Unit * Size, %Total number of bits! St2 = elemtype_check(Anno, Type, Sz, St), {Sz,St2}. @@ -2974,17 +2975,13 @@ type_def(Attr, Anno, TypeName, ProtoType, Args, St0) -> not member(no_auto_import_types, St0#lint.compile) of true -> case is_obsolete_builtin_type(TypePair) of - true -> StoreType(St0); + true -> + StoreType(St0); false -> - case is_newly_introduced_builtin_type(TypePair) of - %% allow some types just for bootstrapping - true -> - Warn = {new_builtin_type, TypePair}, - St1 = add_warning(Anno, Warn, St0), - StoreType(St1); - false -> - add_error(Anno, {builtin_type, TypePair}, St0) - end + %% Starting from OTP 26, redefining built-in types + %% is allowed. + St1 = StoreType(St0), + warn_redefined_builtin_type(Anno, TypePair, St1) end; false -> case is_map_key(TypePair, TypeDefs) of @@ -3004,12 +3001,29 @@ type_def(Attr, Anno, TypeName, ProtoType, Args, St0) -> end end. +warn_redefined_builtin_type(Anno, TypePair, #lint{compile=Opts}=St) -> + case is_warn_enabled(redefined_builtin_type, St) of + true -> + NoWarn = [Type || + {nowarn_redefined_builtin_type, Type0} <- Opts, + Type <- lists:flatten([Type0])], + case lists:member(TypePair, NoWarn) of + true -> + St; + false -> + Warn = {redefine_builtin_type, TypePair}, + add_warning(Anno, Warn, St) + end; + false -> + St + end. + is_underspecified({type,_,term,[]}, 0) -> true; is_underspecified({type,_,any,[]}, 0) -> true; is_underspecified(_ProtType, _Arity) -> false. check_type(Types, St) -> - {SeenVars, St1} = check_type(Types, maps:new(), St), + {SeenVars, St1} = check_type_1(Types, maps:new(), St), maps:fold(fun(Var, {seen_once, Anno}, AccSt) -> case atom_to_list(Var) of "_"++_ -> AccSt; @@ -3019,24 +3033,39 @@ check_type(Types, St) -> AccSt end, St1, SeenVars). -check_type({ann_type, _A, [_Var, Type]}, SeenVars, St) -> - check_type(Type, SeenVars, St); -check_type({remote_type, A, [{atom, _, Mod}, {atom, _, Name}, Args]}, +check_type_1({type, Anno, TypeName, Args}=Type, SeenVars, #lint{types=Types}=St) -> + TypePair = {TypeName, + if + is_list(Args) -> length(Args); + true -> 0 + end}, + case is_map_key(TypePair, Types) of + true -> + check_type_2(Type, SeenVars, used_type(TypePair, Anno, St)); + false -> + check_type_2(Type, SeenVars, St) + end; +check_type_1(Types, SeenVars, St) -> + check_type_2(Types, SeenVars, St). + +check_type_2({ann_type, _A, [_Var, Type]}, SeenVars, St) -> + check_type_1(Type, SeenVars, St); +check_type_2({remote_type, A, [{atom, _, Mod}, {atom, _, Name}, Args]}, SeenVars, St00) -> St0 = check_module_name(Mod, A, St00), St = deprecated_type(A, Mod, Name, Args, St0), CurrentMod = St#lint.module, case Mod =:= CurrentMod of - true -> check_type({user_type, A, Name, Args}, SeenVars, St); + true -> check_type_2({user_type, A, Name, Args}, SeenVars, St); false -> lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) + check_type_1(T, AccSeenVars, AccSt) end, {SeenVars, St}, Args) end; -check_type({integer, _A, _}, SeenVars, St) -> {SeenVars, St}; -check_type({atom, _A, _}, SeenVars, St) -> {SeenVars, St}; -check_type({var, _A, '_'}, SeenVars, St) -> {SeenVars, St}; -check_type({var, A, Name}, SeenVars, St) -> +check_type_2({integer, _A, _}, SeenVars, St) -> {SeenVars, St}; +check_type_2({atom, _A, _}, SeenVars, St) -> {SeenVars, St}; +check_type_2({var, _A, '_'}, SeenVars, St) -> {SeenVars, St}; +check_type_2({var, A, Name}, SeenVars, St) -> NewSeenVars = case maps:find(Name, SeenVars) of {ok, {seen_once, _}} -> maps:put(Name, seen_multiple, SeenVars); @@ -3044,34 +3073,34 @@ check_type({var, A, Name}, SeenVars, St) -> error -> maps:put(Name, {seen_once, A}, SeenVars) end, {NewSeenVars, St}; -check_type({type, A, bool, []}, SeenVars, St) -> +check_type_2({type, A, bool, []}, SeenVars, St) -> {SeenVars, add_warning(A, {renamed_type, bool, boolean}, St)}; -check_type({type, A, 'fun', [Dom, Range]}, SeenVars, St) -> +check_type_2({type, A, 'fun', [Dom, Range]}, SeenVars, St) -> St1 = case Dom of {type, _, product, _} -> St; {type, _, any} -> St; _ -> add_error(A, {type_syntax, 'fun'}, St) end, - check_type({type, nowarn(), product, [Dom, Range]}, SeenVars, St1); -check_type({type, A, range, [From, To]}, SeenVars, St) -> + check_type_2({type, nowarn(), product, [Dom, Range]}, SeenVars, St1); +check_type_2({type, A, range, [From, To]}, SeenVars, St) -> St1 = case {erl_eval:partial_eval(From), erl_eval:partial_eval(To)} of {{integer, _, X}, {integer, _, Y}} when X < Y -> St; _ -> add_error(A, {type_syntax, range}, St) end, {SeenVars, St1}; -check_type({type, _A, map, any}, SeenVars, St) -> +check_type_2({type, _A, map, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, _A, map, Pairs}, SeenVars, St) -> +check_type_2({type, _A, map, Pairs}, SeenVars, St) -> lists:foldl(fun(Pair, {AccSeenVars, AccSt}) -> - check_type(Pair, AccSeenVars, AccSt) + check_type_2(Pair, AccSeenVars, AccSt) end, {SeenVars, St}, Pairs); -check_type({type, _A, map_field_assoc, [Dom, Range]}, SeenVars, St) -> - check_type({type, nowarn(), product, [Dom, Range]}, SeenVars, St); -check_type({type, _A, tuple, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, _A, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, A, binary, [Base, Unit]}, SeenVars, St) -> +check_type_2({type, _A, map_field_assoc, [Dom, Range]}, SeenVars, St) -> + check_type_2({type, nowarn(), product, [Dom, Range]}, SeenVars, St); +check_type_2({type, _A, tuple, any}, SeenVars, St) -> {SeenVars, St}; +check_type_2({type, _A, any}, SeenVars, St) -> {SeenVars, St}; +check_type_2({type, A, binary, [Base, Unit]}, SeenVars, St) -> St1 = case {erl_eval:partial_eval(Base), erl_eval:partial_eval(Unit)} of {{integer, _, BaseVal}, @@ -3079,20 +3108,20 @@ check_type({type, A, binary, [Base, Unit]}, SeenVars, St) -> _ -> add_error(A, {type_syntax, binary}, St) end, {SeenVars, St1}; -check_type({type, A, record, [Name|Fields]}, SeenVars, St) -> +check_type_2({type, A, record, [Name|Fields]}, SeenVars, St) -> case Name of {atom, _, Atom} -> St1 = used_record(Atom, St), check_record_types(A, Atom, Fields, SeenVars, St1); _ -> {SeenVars, add_error(A, {type_syntax, record}, St)} end; -check_type({type, _A, Tag, Args}, SeenVars, St) when Tag =:= product; +check_type_2({type, _A, Tag, Args}, SeenVars, St) when Tag =:= product; Tag =:= union; Tag =:= tuple -> lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) + check_type_1(T, AccSeenVars, AccSt) end, {SeenVars, St}, Args); -check_type({type, Anno, TypeName, Args}, SeenVars, St) -> +check_type_2({type, Anno, TypeName, Args}, SeenVars, St) -> #lint{module = Module, types=Types} = St, Arity = length(Args), TypePair = {TypeName, Arity}, @@ -3108,20 +3137,21 @@ check_type({type, Anno, TypeName, Args}, SeenVars, St) -> Tag = deprecated_builtin_type, W = {Tag, TypePair, Replacement, Rel}, add_warning(Anno, W, St) - end; - _ -> St - end, - check_type({type, nowarn(), product, Args}, SeenVars, St1); -check_type({user_type, A, TypeName, Args}, SeenVars, St) -> + end; + _ -> + St + end, + check_type_2({type, nowarn(), product, Args}, SeenVars, St1); +check_type_2({user_type, A, TypeName, Args}, SeenVars, St) -> Arity = length(Args), TypePair = {TypeName, Arity}, St1 = used_type(TypePair, A, St), lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) + check_type_1(T, AccSeenVars, AccSt) end, {SeenVars, St1}, Args); -check_type([{typed_record_field,Field,_T}|_], SeenVars, St) -> +check_type_2([{typed_record_field,Field,_T}|_], SeenVars, St) -> {SeenVars, add_error(element(2, Field), old_abstract_code, St)}; -check_type(I, SeenVars, St) -> +check_type_2(I, SeenVars, St) -> case erl_eval:partial_eval(I) of {integer,_A,_Integer} -> {SeenVars, St}; _Other -> @@ -3156,7 +3186,7 @@ check_record_types([{type, _, field_type, [{atom, Anno, FName}, Type]}|Left], false -> St1 end, %% Check Type - {NewSeenVars, St3} = check_type(Type, SeenVars, St2), + {NewSeenVars, St3} = check_type_2(Type, SeenVars, St2), NewSeenFields = ordsets:add_element(FName, SeenFields), check_record_types(Left, Name, DefFields, NewSeenVars, St3, NewSeenFields); check_record_types([], _Name, _DefFields, SeenVars, St, _SeenFields) -> @@ -3172,8 +3202,6 @@ used_type(TypePair, Anno, #lint{usage = Usage, file = File} = St) -> is_default_type({Name, NumberOfTypeVariables}) -> erl_internal:is_type(Name, NumberOfTypeVariables). -is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. - is_obsolete_builtin_type(TypePair) -> obsolete_builtin_type(TypePair) =/= no. @@ -4240,19 +4268,18 @@ keyword_warning(Anno, Atom, St) -> %% Add warning for bad calls to io:fwrite/format functions. format_function(DefAnno, M, F, As, St) -> - case is_format_function(M, F) of - true -> - case St#lint.warn_format of - Lev when Lev > 0 -> - case check_format_1(As) of - {warn,Level,Fmt,Fas} when Level =< Lev -> - add_warning(DefAnno, {format_error,{Fmt,Fas}}, St); - {warn,Level,Anno,Fmt,Fas} when Level =< Lev -> - add_warning(Anno, {format_error,{Fmt,Fas}}, St); - _ -> St - end; - _Lev -> St - end; + maybe + true ?= is_format_function(M, F), + Lev = St#lint.warn_format, + true ?= Lev > 0, + case check_format_1(As) of + {warn,Level,Fmt,Fas} when Level =< Lev -> + add_warning(DefAnno, {format_error,{Fmt,Fas}}, St); + {warn,Level,Anno,Fmt,Fas} when Level =< Lev -> + add_warning(Anno, {format_error,{Fmt,Fas}}, St); + _ -> St + end + else false -> St end. @@ -4371,9 +4398,11 @@ extract_sequences(Fmt, Need0) -> end end. -extract_sequence(1, [$-,C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(1, [$-,C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(1, Fmt, Need); -extract_sequence(1, [C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(1, [C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(1, Fmt, Need); extract_sequence(1, [$-,$*|Fmt], Need) -> extract_sequence(2, Fmt, [int|Need]); @@ -4382,7 +4411,8 @@ extract_sequence(1, [$*|Fmt], Need) -> extract_sequence(1, Fmt, Need) -> extract_sequence(2, Fmt, Need); -extract_sequence(2, [$.,C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(2, [$.,C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(2, Fmt, Need); extract_sequence(2, [$.,$*|Fmt], Need) -> extract_sequence(3, Fmt, [int|Need]); @@ -4436,7 +4466,8 @@ extract_sequence(5, [C|Fmt], Need0) -> end; extract_sequence(_, [], _Need) -> {error,"truncated"}. -extract_sequence_digits(Fld, [C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence_digits(Fld, [C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(Fld, Fmt, Need); extract_sequence_digits(Fld, Fmt, Need) -> extract_sequence(Fld+1, Fmt, Need). diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index f2e9d2d7b9..e7e2582c03 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -264,15 +264,15 @@ string_thing(_) -> "string". -define(WHITE_SPACE(C), is_integer(C) andalso (C >= $\000 andalso C =< $\s orelse C >= $\200 andalso C =< $\240)). --define(DIGIT(C), C >= $0 andalso C =< $9). --define(CHAR(C), is_integer(C), C >= 0). +-define(DIGIT(C), (is_integer(C) andalso $0 =< C andalso C =< $9)). +-define(CHAR(C), (is_integer(C) andalso 0 =< C andalso C < 16#110000)). -define(UNICODE(C), - is_integer(C) andalso + (is_integer(C) andalso (C >= 0 andalso C < 16#D800 orelse C > 16#DFFF andalso C < 16#FFFE orelse - C > 16#FFFF andalso C =< 16#10FFFF)). + C > 16#FFFF andalso C =< 16#10FFFF))). --define(UNI255(C), C >= 0, C =< 16#ff). +-define(UNI255(C), (is_integer(C) andalso 0 =< C andalso C =< 16#ff)). options(Opts0) when is_list(Opts0) -> Opts = lists:foldr(fun expand_opt/2, [], Opts0), @@ -363,7 +363,7 @@ string1(Cs, St, Line, Col, Toks) -> Error end. -scan(Cs, St, Line, Col, Toks, _) -> +scan(Cs, #erl_scan{}=St, Line, Col, Toks, _) -> scan1(Cs, St, Line, Col, Toks). scan1([$\s|Cs], St, Line, Col, Toks) when St#erl_scan.ws -> @@ -374,10 +374,6 @@ scan1([$\n|Cs], St, Line, Col, Toks) when St#erl_scan.ws -> scan_newline(Cs, St, Line, Col, Toks); scan1([$\n|Cs], St, Line, Col, Toks) -> skip_white_space(Cs, St, Line+1, new_column(Col, 1), Toks, 0); -scan1([C|Cs], St, Line, Col, Toks) when C >= $A, C =< $Z -> - scan_variable(Cs, St, Line, Col, Toks, [C]); -scan1([C|Cs], St, Line, Col, Toks) when C >= $a, C =< $z -> - scan_atom(Cs, St, Line, Col, Toks, [C]); %% Optimization: some very common punctuation characters: scan1([$,|Cs], St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ",", ',', 1); @@ -397,11 +393,17 @@ scan1([$;|Cs], St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ";", ';', 1); scan1([$_=C|Cs], St, Line, Col, Toks) -> scan_variable(Cs, St, Line, Col, Toks, [C]); -%% More punctuation characters below. scan1([$\%|Cs], St, Line, Col, Toks) when not St#erl_scan.comment -> skip_comment(Cs, St, Line, Col, Toks, 1); scan1([$\%=C|Cs], St, Line, Col, Toks) -> scan_comment(Cs, St, Line, Col, Toks, [C]); +%% More punctuation characters below. +scan1([C|_], _St, _Line, _Col0, _Toks) when not ?CHAR(C) -> + error({not_character,C}); +scan1([C|Cs], St, Line, Col, Toks) when C >= $A, C =< $Z -> + scan_variable(Cs, St, Line, Col, Toks, [C]); +scan1([C|Cs], St, Line, Col, Toks) when C >= $a, C =< $z -> + scan_atom(Cs, St, Line, Col, Toks, [C]); scan1([C|Cs], St, Line, Col, Toks) when ?DIGIT(C) -> scan_number(Cs, St, Line, Col, Toks, [C], no_underscore); scan1("..."++Cs, St, Line, Col, Toks) -> @@ -557,39 +559,49 @@ scan1([]=Cs, _St, Line, Col, Toks) -> scan1(eof=Cs, _St, Line, Col, Toks) -> {ok,Toks,Cs,Line,Col}. +scan_atom_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_atom(Cs, St, Line, Col, Toks, Ncs). + scan_atom(Cs0, St, Line, Col, Toks, Ncs0) -> case scan_name(Cs0, Ncs0) of {more,Ncs} -> - {more,{[],Col,Toks,Line,Ncs,fun scan_atom/6}}; + {more,{[],Col,Toks,Line,Ncs,fun scan_atom_fun/6}}; {Wcs,Cs} -> - case catch list_to_atom(Wcs) of - Name when is_atom(Name) -> + try list_to_atom(Wcs) of + Name -> case (St#erl_scan.resword_fun)(Name) of true -> tok2(Cs, St, Line, Col, Toks, Wcs, Name); false -> tok3(Cs, St, Line, Col, Toks, atom, Wcs, Name) - end; - _Error -> + end + catch + _:_ -> Ncol = incr_column(Col, length(Wcs)), scan_error({illegal,atom}, Line, Col, Line, Ncol, Cs) end end. +scan_variable_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_variable(Cs, St, Line, Col, Toks, Ncs). + scan_variable(Cs0, St, Line, Col, Toks, Ncs0) -> case scan_name(Cs0, Ncs0) of {more,Ncs} -> - {more,{[],Col,Toks,Line,Ncs,fun scan_variable/6}}; + {more,{[],Col,Toks,Line,Ncs,fun scan_variable_fun/6}}; {Wcs,Cs} -> - case catch list_to_atom(Wcs) of - Name when is_atom(Name) -> - tok3(Cs, St, Line, Col, Toks, var, Wcs, Name); - _Error -> + try list_to_atom(Wcs) of + Name -> + tok3(Cs, St, Line, Col, Toks, var, Wcs, Name) + catch + _:_ -> Ncol = incr_column(Col, length(Wcs)), scan_error({illegal,var}, Line, Col, Line, Ncol, Cs) end end. +scan_name([C|_]=Cs, Ncs) when not ?CHAR(C) -> + {lists:reverse(Ncs),Cs}; scan_name([C|Cs], Ncs) when C >= $a, C =< $z -> scan_name(Cs, [C|Ncs]); scan_name([C|Cs], Ncs) when C >= $A, C =< $Z -> @@ -663,20 +675,31 @@ scan_newline([], _St, Line, Col, Toks) -> scan_newline(Cs, St, Line, Col, Toks) -> scan_nl_white_space(Cs, St, Line, Col, Toks, "\n"). +scan_nl_spcs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N) -> + scan_nl_spcs(Cs, St, Line, Col, Toks, N). + scan_nl_spcs([$\s|Cs], St, Line, Col, Toks, N) when N < 17 -> scan_nl_spcs(Cs, St, Line, Col, Toks, N+1); scan_nl_spcs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_nl_spcs/6}}; + {more,{Cs,Col,Toks,Line,N,fun scan_nl_spcs_fun/6}}; scan_nl_spcs(Cs, St, Line, Col, Toks, N) -> newline_end(Cs, St, Line, Col, Toks, N, nl_spcs(N)). +scan_nl_tabs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N) -> + scan_nl_tabs(Cs, St, Line, Col, Toks, N). + scan_nl_tabs([$\t|Cs], St, Line, Col, Toks, N) when N < 11 -> scan_nl_tabs(Cs, St, Line, Col, Toks, N+1); scan_nl_tabs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_nl_tabs/6}}; + {more,{Cs,Col,Toks,Line,N,fun scan_nl_tabs_fun/6}}; scan_nl_tabs(Cs, St, Line, Col, Toks, N) -> newline_end(Cs, St, Line, Col, Toks, N, nl_tabs(N)). +scan_nl_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_nl_white_space(Cs, St, Line, Col, Toks, Ncs). + %% Note: returning {more,Cont} is meaningless here; one could just as %% well return several tokens. But since tokens() scans up to a full %% stop anyway, nothing is gained by not collecting all white spaces. @@ -689,10 +712,11 @@ scan_nl_white_space([$\n|Cs], St, Line, Col, Toks, Ncs0) -> Anno = anno(Line, Col, St, ?STR(white_space, St, Ncs)), Token = {white_space,Anno,Ncs}, scan_newline(Cs, St, Line+1, new_column(Col, length(Ncs)), [Token|Toks]); -scan_nl_white_space([C|Cs], St, Line, Col, Toks, Ncs) when ?WHITE_SPACE(C) -> +scan_nl_white_space([C|Cs], St, Line, Col, Toks, Ncs) + when ?WHITE_SPACE(C) -> scan_nl_white_space(Cs, St, Line, Col, Toks, [C|Ncs]); scan_nl_white_space([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_nl_white_space/6}}; + {more,{Cs,Col,Toks,Line,Ncs,fun scan_nl_white_space_fun/6}}; scan_nl_white_space(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Ncs) -> Anno = anno(Line), @@ -706,40 +730,54 @@ scan_nl_white_space(Cs, St, Line, Col, Toks, Ncs0) -> newline_end(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _N, Ncs) -> scan1(Cs, St, Line+1, Col, [{white_space,anno(Line),Ncs}|Toks]); -newline_end(Cs, St, Line, Col, Toks, N, Ncs) -> +newline_end(Cs, #erl_scan{}=St, Line, Col, Toks, N, Ncs) -> Anno = anno(Line, Col, St, ?STR(white_space, St, Ncs)), scan1(Cs, St, Line+1, new_column(Col, N), [{white_space,Anno,Ncs}|Toks]). +scan_spcs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N), N >= 1 -> + scan_spcs(Cs, St, Line, Col, Toks, N). + scan_spcs([$\s|Cs], St, Line, Col, Toks, N) when N < 16 -> scan_spcs(Cs, St, Line, Col, Toks, N+1); scan_spcs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_spcs/6}}; + {more,{Cs,Col,Toks,Line,N,fun scan_spcs_fun/6}}; scan_spcs(Cs, St, Line, Col, Toks, N) -> white_space_end(Cs, St, Line, Col, Toks, N, spcs(N)). +scan_tabs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N), N >= 1 -> + scan_tabs(Cs, St, Line, Col, Toks, N). + scan_tabs([$\t|Cs], St, Line, Col, Toks, N) when N < 10 -> scan_tabs(Cs, St, Line, Col, Toks, N+1); scan_tabs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_tabs/6}}; + {more,{Cs,Col,Toks,Line,N,fun scan_tabs_fun/6}}; scan_tabs(Cs, St, Line, Col, Toks, N) -> white_space_end(Cs, St, Line, Col, Toks, N, tabs(N)). +skip_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) -> + skip_white_space(Cs, St, Line, Col, Toks, N). + skip_white_space([$\n|Cs], St, Line, Col, Toks, _N) -> skip_white_space(Cs, St, Line+1, new_column(Col, 1), Toks, 0); skip_white_space([C|Cs], St, Line, Col, Toks, N) when ?WHITE_SPACE(C) -> skip_white_space(Cs, St, Line, Col, Toks, N+1); skip_white_space([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun skip_white_space/6}}; + {more,{Cs,Col,Toks,Line,N,fun skip_white_space_fun/6}}; skip_white_space(Cs, St, Line, Col, Toks, N) -> scan1(Cs, St, Line, incr_column(Col, N), Toks). +scan_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_white_space(Cs, St, Line, Col, Toks, Ncs). + %% Maybe \t and \s should break the loop. scan_white_space([$\n|_]=Cs, St, Line, Col, Toks, Ncs) -> white_space_end(Cs, St, Line, Col, Toks, length(Ncs), lists:reverse(Ncs)); scan_white_space([C|Cs], St, Line, Col, Toks, Ncs) when ?WHITE_SPACE(C) -> scan_white_space(Cs, St, Line, Col, Toks, [C|Ncs]); scan_white_space([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_white_space/6}}; + {more,{Cs,Col,Toks,Line,Ncs,fun scan_white_space_fun/6}}; scan_white_space(Cs, St, Line, Col, Toks, Ncs) -> white_space_end(Cs, St, Line, Col, Toks, length(Ncs), lists:reverse(Ncs)). @@ -778,7 +816,7 @@ scan_char([], _St, Line, Col, Toks) -> scan_char(eof, _St, Line, Col, _Toks) -> scan_error(char, Line, Col, Line, incr_column(Col, 1), eof). -scan_string(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> +scan_string(Cs, #erl_scan{}=St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> case scan_string0(Cs, St, Line, Col, $\", Str, Wcs) of %" {more,Ncs,Nline,Ncol,Nstr,Nwcs} -> State = {Nwcs,Nstr,Line0,Col0}, @@ -793,7 +831,7 @@ scan_string(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> scan1(Ncs, St, Nline, Ncol, [{string,Anno,Nwcs}|Toks]) end. -scan_qatom(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> +scan_qatom(Cs, #erl_scan{}=St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> case scan_string0(Cs, St, Line, Col, $\', Str, Wcs) of %' {more,Ncs,Nline,Ncol,Nstr,Nwcs} -> State = {Nwcs,Nstr,Line0,Col0}, @@ -803,12 +841,13 @@ scan_qatom(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> {error,Nline,Ncol,Nwcs,Ncs} -> Estr = string:slice(Nwcs, 0, 16), % Expanded escape chars. scan_error({string,$\',Estr}, Line0, Col0, Nline, Ncol, Ncs); %' - {Ncs,Nline,Ncol,Nstr,Nwcs} -> - case catch list_to_atom(Nwcs) of + {Ncs,Nline,Ncol,Nstr,Nwcs} -> + try list_to_atom(Nwcs) of A when is_atom(A) -> Anno = anno(Line0, Col0, St, ?STR(atom, St, Nstr)), - scan1(Ncs, St, Nline, Ncol, [{atom,Anno,A}|Toks]); - _ -> + scan1(Ncs, St, Nline, Ncol, [{atom,Anno,A}|Toks]) + catch + _:_ -> scan_error({illegal,atom}, Line0, Col0, Nline, Ncol, Ncs) end end. @@ -884,10 +923,11 @@ scan_string1([]=Cs, Line, Col, _Q, Str, Wcs) -> scan_string1(eof, Line, Col, _Q, _Str, Wcs) -> {error,Line,Col,lists:reverse(Wcs),eof}. --define(OCT(C), C >= $0, C =< $7). --define(HEX(C), C >= $0 andalso C =< $9 orelse - C >= $A andalso C =< $F orelse - C >= $a andalso C =< $f). +-define(OCT(C), (is_integer(C) andalso $0 =< C andalso C =< $7)). +-define(HEX(C), (is_integer(C) andalso + (C >= $0 andalso C =< $9 orelse + C >= $A andalso C =< $F orelse + C >= $a andalso C =< $f))). %% \<1-3> octal digits scan_escape([O1,O2,O3|Cs], Col) when ?OCT(O1), ?OCT(O2), ?OCT(O3) -> @@ -939,26 +979,31 @@ scan_escape([], _Col) -> scan_escape(eof, Col) -> {eof,Col}. -scan_hex([C|Cs], no_col=Col, Wcs) when ?HEX(C) -> - scan_hex(Cs, Col, [C|Wcs]); scan_hex([C|Cs], Col, Wcs) when ?HEX(C) -> - scan_hex(Cs, Col+1, [C|Wcs]); + scan_hex(Cs, incr_column(Col, 1), [C|Wcs]); scan_hex(Cs, Col, Wcs) -> - scan_esc_end(Cs, Col, Wcs, 16, "x{"). + scan_hex_end(Cs, Col, Wcs, "x{"). -scan_esc_end([$}|Cs], Col, Wcs0, B, Str0) -> +scan_hex_end([$}|Cs], Col, [], _Str) -> + %% Empty escape sequence. + {error,Cs,{illegal,character},incr_column(Col, 1)}; +scan_hex_end([$}|Cs], Col, Wcs0, Str0) -> Wcs = lists:reverse(Wcs0), - case catch erlang:list_to_integer(Wcs, B) of + try list_to_integer(Wcs, 16) of Val when ?UNICODE(Val) -> {Val,Str0++Wcs++[$}],Cs,incr_column(Col, 1)}; - _ -> + _Val -> + {error,Cs,{illegal,character},incr_column(Col, 1)} + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. {error,Cs,{illegal,character},incr_column(Col, 1)} end; -scan_esc_end([], _Col, _Wcs, _B, _Str0) -> +scan_hex_end([], _Col, _Wcs, _Str0) -> more; -scan_esc_end(eof, Col, _Wcs, _B, _Str0) -> +scan_hex_end(eof, Col, _Wcs, _Str0) -> {eof,Col}; -scan_esc_end(Cs, Col, _Wcs, _B, _Str0) -> +scan_hex_end(Cs, Col, _Wcs, _Str0) -> {error,Cs,{illegal,character},Col}. escape_char($n) -> $\n; % \n = LF @@ -972,7 +1017,7 @@ escape_char($s) -> $\s; % \s = SPC escape_char($d) -> $\d; % \d = DEL escape_char(C) -> C. -scan_number(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +scan_number(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_number(Cs, St, Line, Col, Toks, Ncs, Us). scan_number([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -988,22 +1033,28 @@ scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs, Us) -> {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_integer(remove_digit_separators(Ncs, Us)) of - B when B >= 2, B =< 1+$Z-$A+10 -> + try list_to_integer(remove_digit_separators(Ncs, Us)) of + B when is_integer(B), 2 =< B, B =< 1+$Z-$A+10 -> Bcs = Ncs++[$#], scan_based_int(Cs, St, Line, Col, Toks, B, [], Bcs, no_underscore); - B -> + B when is_integer(B) -> Len = length(Ncs), scan_error({base,B}, Line, Col, Line, incr_column(Col, Len), Cs0) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. + scan_error({illegal,base}, Line, Col, Line, Col, Cs0) end; scan_number([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; scan_number(Cs, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_integer(remove_digit_separators(Ncs, Us)) of - N when is_integer(N) -> - tok3(Cs, St, Line, Col, Toks, integer, Ncs, N); - _ -> + try list_to_integer(remove_digit_separators(Ncs, Us), 10) of + N -> + tok3(Cs, St, Line, Col, Toks, integer, Ncs, N) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. Ncol = incr_column(Col, length(Ncs)), scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs) end. @@ -1014,11 +1065,14 @@ remove_digit_separators(Number, with_underscore) -> [C || C <- Number, C =/= $_]. -define(BASED_DIGIT(C, B), - ((?DIGIT(C) andalso C < $0 + B) - orelse (C >= $A andalso B > 10 andalso C < $A + B - 10) - orelse (C >= $a andalso B > 10 andalso C < $a + B - 10))). - -scan_based_int(Cs, St, Line, Col, Toks, {B,NCs,BCs,Us}) -> + (is_integer(C) + andalso + ((?DIGIT(C) andalso C < $0 + B) + orelse (C >= $A andalso B > 10 andalso C < $A + B - 10) + orelse (C >= $a andalso B > 10 andalso C < $a + B - 10)))). + +scan_based_int(Cs, #erl_scan{}=St, Line, Col, Toks, {B,NCs,BCs,Us}) + when is_integer(B), 2 =< B, B =< 1+$Z-$A+10 -> scan_based_int(Cs, St, Line, Col, Toks, B, NCs, BCs, Us). scan_based_int([C|Cs], St, Line, Col, Toks, B, Ncs, Bcs, Us) when @@ -1032,18 +1086,25 @@ scan_based_int([$_]=Cs, _St, Line, Col, Toks, B, NCs, BCs, Us) -> {more,{Cs,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; scan_based_int([]=Cs, _St, Line, Col, Toks, B, NCs, BCs, Us) -> {more,{Cs,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; -scan_based_int(Cs, St, Line, Col, Toks, B, Ncs0, Bcs, Us) -> +scan_based_int(Cs, _St, Line, Col, _Toks, _B, [], Bcs, _Us) -> + %% No actual digits following the base. + Len = length(Bcs), + Ncol = incr_column(Col, Len), + scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs); +scan_based_int(Cs, St, Line, Col, Toks, B, Ncs0, [_|_]=Bcs, Us) -> Ncs = lists:reverse(Ncs0), - case catch erlang:list_to_integer(remove_digit_separators(Ncs, Us), B) of - N when is_integer(N) -> - tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N); - _ -> + try list_to_integer(remove_digit_separators(Ncs, Us), B) of + N -> + tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. Len = length(Bcs)+length(Ncs), Ncol = incr_column(Col, Len), scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs) end. -scan_fraction(Cs, St, Line, Col, Toks, {Ncs,Us}) -> +scan_fraction(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs,Us}) -> scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us). scan_fraction([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -1060,7 +1121,7 @@ scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us) -> float_end(Cs, St, Line, Col, Toks, Ncs, Us). -scan_exponent_sign(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +scan_exponent_sign(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us). scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs, Us) when @@ -1071,7 +1132,7 @@ scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us) -> scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us). -scan_exponent(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +scan_exponent(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us). scan_exponent([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -1088,14 +1149,18 @@ scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us) -> float_end(Cs, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_float(remove_digit_separators(Ncs, Us)) of - F when is_float(F) -> - tok3(Cs, St, Line, Col, Toks, float, Ncs, F); - _ -> + try list_to_float(remove_digit_separators(Ncs, Us)) of + F -> + tok3(Cs, St, Line, Col, Toks, float, Ncs, F) + catch + _:_ -> Ncol = incr_column(Col, length(Ncs)), scan_error({illegal,float}, Line, Col, Line, Ncol, Cs) end. +skip_comment_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) -> + skip_comment(Cs, St, Line, Col, Toks, N). + skip_comment([C|Cs], St, Line, Col, Toks, N) when C =/= $\n, ?CHAR(C) -> case ?UNICODE(C) of true -> @@ -1105,11 +1170,15 @@ skip_comment([C|Cs], St, Line, Col, Toks, N) when C =/= $\n, ?CHAR(C) -> scan_error({illegal,character}, Line, Col, Line, Ncol, Cs) end; skip_comment([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun skip_comment/6}}; + {more,{Cs,Col,Toks,Line,N,fun skip_comment_fun/6}}; skip_comment(Cs, St, Line, Col, Toks, N) -> scan1(Cs, St, Line, incr_column(Col, N), Toks). -scan_comment([C|Cs], St, Line, Col, Toks, Ncs) when C =/= $\n, ?CHAR(C) -> +scan_comment_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_comment(Cs, St, Line, Col, Toks, Ncs). + +scan_comment([C|Cs], St, Line, Col, Toks, Ncs) + when C =/= $\n, ?CHAR(C) -> case ?UNICODE(C) of true -> scan_comment(Cs, St, Line, Col, Toks, [C|Ncs]); @@ -1118,33 +1187,33 @@ scan_comment([C|Cs], St, Line, Col, Toks, Ncs) when C =/= $\n, ?CHAR(C) -> scan_error({illegal,character}, Line, Col, Line, Ncol, Cs) end; scan_comment([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_comment/6}}; + {more,{Cs,Col,Toks,Line,Ncs,fun scan_comment_fun/6}}; scan_comment(Cs, St, Line, Col, Toks, Ncs0) -> Ncs = lists:reverse(Ncs0), tok3(Cs, St, Line, Col, Toks, comment, Ncs, Ncs). tok2(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _Wcs, P) -> scan1(Cs, St, Line, Col, [{P,anno(Line)}|Toks]); -tok2(Cs, St, Line, Col, Toks, Wcs, P) -> +tok2(Cs, #erl_scan{}=St, Line, Col, Toks, Wcs, P) -> Anno = anno(Line, Col, St, ?STR(P, St, Wcs)), scan1(Cs, St, Line, incr_column(Col, length(Wcs)), [{P,Anno}|Toks]). tok2(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _Wcs, P, _N) -> scan1(Cs, St, Line, Col, [{P,anno(Line)}|Toks]); -tok2(Cs, St, Line, Col, Toks, Wcs, P, N) -> +tok2(Cs, #erl_scan{}=St, Line, Col, Toks, Wcs, P, N) -> Anno = anno(Line, Col, St, ?STR(P,St,Wcs)), scan1(Cs, St, Line, incr_column(Col, N), [{P,Anno}|Toks]). tok3(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Item, _S, Sym) -> scan1(Cs, St, Line, Col, [{Item,anno(Line),Sym}|Toks]); -tok3(Cs, St, Line, Col, Toks, Item, String, Sym) -> +tok3(Cs, #erl_scan{}=St, Line, Col, Toks, Item, String, Sym) -> Token = {Item,anno(Line, Col, St, ?STR(Item, St, String)),Sym}, scan1(Cs, St, Line, incr_column(Col, length(String)), [Token|Toks]). tok3(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Item, _String, Sym, _Length) -> scan1(Cs, St, Line, Col, [{Item,anno(Line),Sym}|Toks]); -tok3(Cs, St, Line, Col, Toks, Item, String, Sym, Length) -> +tok3(Cs, #erl_scan{}=St, Line, Col, Toks, Item, String, Sym, Length) -> Token = {Item,anno(Line, Col, St, ?STR(Item, St, String)),Sym}, scan1(Cs, St, Line, incr_column(Col, Length), [Token|Toks]). diff --git a/lib/stdlib/src/erl_stdlib_errors.erl b/lib/stdlib/src/erl_stdlib_errors.erl index eeccb77db1..36af07e1ac 100644 --- a/lib/stdlib/src/erl_stdlib_errors.erl +++ b/lib/stdlib/src/erl_stdlib_errors.erl @@ -641,6 +641,9 @@ format_ets_error(lookup_element, [_,_,Pos]=Args, Cause) -> [TabCause, "", PosCause] end end; +format_ets_error(lookup_element, [Tab, Key, Pos, _Default], Cause) -> + % The default argument cannot cause an error. + format_ets_error(lookup_element, [Tab, Key, Pos], Cause); format_ets_error(match, [_], _Cause) -> [bad_continuation]; format_ets_error(match, [_,_,_]=Args, Cause) -> diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 979a75b231..3545c8a186 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -70,7 +70,7 @@ -export([all/0, delete/1, delete/2, delete_all_objects/1, delete_object/2, first/1, give_away/3, info/1, info/2, insert/2, insert_new/2, is_compiled_ms/1, last/1, lookup/2, - lookup_element/3, match/1, match/2, match/3, match_object/1, + lookup_element/3, lookup_element/4, match/1, match/2, match/3, match_object/1, match_object/2, match_object/3, match_spec_compile/1, match_spec_run_r/3, member/2, new/2, next/2, prev/2, rename/2, safe_fixtable/2, select/1, select/2, select/3, @@ -232,6 +232,16 @@ lookup(_, _) -> lookup_element(_, _, _) -> erlang:nif_error(undef). +-spec lookup_element(Table, Key, Pos, Default) -> Elem when + Table :: table(), + Key :: term(), + Pos :: pos_integer(), + Default :: term(), + Elem :: term() | [term()]. + +lookup_element(_, _, _, _) -> + erlang:nif_error(undef). + -spec match(Table, Pattern) -> [Match] when Table :: table(), Pattern :: match_pattern(), diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 8bf4e97b9f..fc8e8110a0 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -78,8 +78,10 @@ -include_lib("kernel/include/file.hrl"). --define(IS_DRIVELETTER(Letter),(((Letter >= $A) andalso (Letter =< $Z)) orelse - ((Letter >= $a) andalso (Letter =< $z)))). +-define(IS_DRIVELETTER(Letter), + (is_integer(Letter) + andalso (($A =< Letter andalso Letter =< $Z) + orelse ($a =< Letter andalso Letter =< $z)))). %% Converts a relative filename to an absolute filename %% or the filename itself if it already is an absolute filename @@ -333,8 +335,7 @@ dirname([$/|Rest], Dir, File, Seps) -> dirname([DirSep|Rest], Dir, File, {DirSep,_}=Seps) when is_integer(DirSep) -> dirname(Rest, File++Dir, [$/], Seps); dirname([Dl,DrvSep|Rest], [], [], {_,DrvSep}=Seps) - when is_integer(DrvSep), ((($a =< Dl) and (Dl =< $z)) or - (($A =< Dl) and (Dl =< $Z))) -> + when is_integer(DrvSep), ?IS_DRIVELETTER(Dl) -> dirname(Rest, [DrvSep,Dl], [], Seps); dirname([Char|Rest], Dir, File, Seps) when is_integer(Char) -> dirname(Rest, Dir, [Char|File], Seps); @@ -757,7 +758,8 @@ win32_split([X, $\\|Rest]) when is_integer(X) -> win32_split([X, $/|Rest]); win32_split([X, Y, $\\|Rest]) when is_integer(X), is_integer(Y) -> win32_split([X, Y, $/|Rest]); -win32_split([UcLetter, $:|Rest]) when UcLetter >= $A, UcLetter =< $Z -> +win32_split([UcLetter, $:|Rest]) + when is_integer(UcLetter), $A =< UcLetter, UcLetter =< $Z -> win32_split([UcLetter+$a-$A, $:|Rest]); win32_split([Letter, $:, $/|Rest]) -> split(Rest, [], [[Letter, $:, $/]], win32); diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index 8dda0d4ee0..93131d1b24 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -259,13 +259,13 @@ is_member_1(_, nil) -> Set1 :: set(Element), Set2 :: set(Element). -insert(Key, {S, T}) -> +insert(Key, {S, T}) when is_integer(S), S >= 0 -> S1 = S + 1, {S1, insert_1(Key, T, ?pow(S1, ?p))}. insert_1(Key, {Key1, Smaller, Bigger}, S) when Key < Key1 -> case insert_1(Key, Smaller, ?div2(S)) of - {T1, H1, S1} when is_integer(H1) -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, T1, Bigger}, {H2, S2} = count(Bigger), H = ?mul2(erlang:max(H1, H2)), @@ -282,7 +282,7 @@ insert_1(Key, {Key1, Smaller, Bigger}, S) when Key < Key1 -> end; insert_1(Key, {Key1, Smaller, Bigger}, S) when Key > Key1 -> case insert_1(Key, Bigger, ?div2(S)) of - {T1, H1, S1} when is_integer(H1) -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, Smaller, T1}, {H2, S2} = count(Smaller), H = ?mul2(erlang:max(H1, H2)), @@ -317,7 +317,7 @@ count(nil) -> Set1 :: set(Element), Set2 :: set(Element). -balance({S, T}) -> +balance({S, T}) when is_integer(S), S >= 0 -> {S, balance(T, S)}. balance(T, S) -> @@ -550,9 +550,9 @@ next([]) -> Set2 :: set(Element), Set3 :: set(Element). -union({N1, T1}, {N2, T2}) when N2 < N1 -> +union({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2), N2 < N1 -> union(to_list_1(T2), N2, T1, N1); -union({N1, T1}, {N2, T2}) -> +union({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2) -> union(to_list_1(T1), N1, T2, N2). %% We avoid the expensive mathematical computations if there is little @@ -633,7 +633,7 @@ push([X | Xs], As) -> push([], As) -> As. -balance_revlist(L, S) -> +balance_revlist(L, S) when is_integer(S) -> {T, _} = balance_revlist_1(L, S), T. @@ -670,9 +670,9 @@ union_list(S, []) -> S. Set2 :: set(Element), Set3 :: set(Element). -intersection({N1, T1}, {N2, T2}) when N2 < N1 -> +intersection({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2), N2 < N1 -> intersection(to_list_1(T2), N2, T1, N1); -intersection({N1, T1}, {N2, T2}) -> +intersection({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2) -> intersection(to_list_1(T1), N1, T2, N2). intersection(L, _N1, T2, N2) when N2 < 10 -> @@ -770,7 +770,8 @@ subtract(S1, S2) -> Set2 :: set(Element), Set3 :: set(Element). -difference({N1, T1}, {N2, T2}) -> +difference({N1, T1}, {N2, T2}) when is_integer(N1), N1 >= 0, + is_integer(N2), N2 >= 0 -> difference(to_list_1(T1), N1, T2, N2). difference(L, N1, T2, N2) when N2 < 10 -> @@ -820,7 +821,8 @@ difference_2(Xs, [], As, S) -> Set1 :: set(Element), Set2 :: set(Element). -is_subset({N1, T1}, {N2, T2}) -> +is_subset({N1, T1}, {N2, T2}) when is_integer(N1), N1 >= 0, + is_integer(N2), N2 >= 0 -> is_subset(to_list_1(T1), N1, T2, N2). is_subset(L, _N1, T2, N2) when N2 < 10 -> diff --git a/lib/stdlib/src/gb_trees.erl b/lib/stdlib/src/gb_trees.erl index c0cdde012e..2d5d013a25 100644 --- a/lib/stdlib/src/gb_trees.erl +++ b/lib/stdlib/src/gb_trees.erl @@ -279,7 +279,7 @@ insert(Key, Val, {S, T}) when is_integer(S) -> insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key < Key1 -> case insert_1(Key, Value, Smaller, ?div2(S)) of - {T1, H1, S1} -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, V, T1, Bigger}, {H2, S2} = count(Bigger), H = ?mul2(erlang:max(H1, H2)), @@ -296,7 +296,7 @@ insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key < Key1 -> end; insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key > Key1 -> case insert_1(Key, Value, Bigger, ?div2(S)) of - {T1, H1, S1} -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, V, Smaller, T1}, {H2, S2} = count(Smaller), H = ?mul2(erlang:max(H1, H2)), @@ -349,7 +349,7 @@ count(nil) -> Tree1 :: tree(Key, Value), Tree2 :: tree(Key, Value). -balance({S, T}) -> +balance({S, T}) when is_integer(S), S >= 0 -> {S, balance(T, S)}. balance(T, S) -> diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index af5e04f78a..d920aa9fcd 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -147,6 +147,22 @@ ( (X) =:= infinity orelse ( is_integer(X) andalso (X) >= 0 ) ) ). +-record(callback_cache,{module :: module(), + handle_call :: fun((Request :: term(), From :: from(), State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), timeout() | hibernate | {continue, term()}} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}), + handle_cast :: fun((Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), NewState :: term()}), + handle_info :: fun((Info :: timeout | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), NewState :: term()})}). %%%========================================================================= %%% API %%%========================================================================= @@ -783,7 +799,8 @@ enter_loop(Mod, Options, State, ServerName, TimeoutOrHibernate) Parent = gen:get_parent(), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - loop(Parent, Name, State, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug); + CbCache = create_callback_cache(Mod), + loop(Parent, Name, State, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug); %% enter_loop(Mod, Options, State, ServerName, {continue, _}=Continue) when is_atom(Mod), is_list(Options) -> @@ -791,7 +808,8 @@ enter_loop(Mod, Options, State, ServerName, {continue, _}=Continue) Parent = gen:get_parent(), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - loop(Parent, Name, State, Mod, Continue, HibernateAfterTimeout, Debug). + CbCache = create_callback_cache(Mod), + loop(Parent, Name, State, CbCache, Continue, HibernateAfterTimeout, Debug). %%%======================================================================== %%% Gen-callback functions @@ -810,19 +828,19 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) -> Name = gen:name(Name0), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - + CbCache = create_callback_cache(Mod), case init_it(Mod, Args) of {ok, {ok, State}} -> proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug); + loop(Parent, Name, State, CbCache, infinity, HibernateAfterTimeout, Debug); {ok, {ok, State, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug); + loop(Parent, Name, State, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug); {ok, {ok, State, {continue, _}=Continue}} -> proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, Continue, HibernateAfterTimeout, Debug); + loop(Parent, Name, State, CbCache, Continue, HibernateAfterTimeout, Debug); {ok, {stop, Reason}} -> %% For consistency, we must make sure that the %% registered name (if any) is unregistered before @@ -848,10 +866,10 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) -> end. init_it(Mod, Args) -> try - {ok, Mod:init(Args)} + {ok, Mod:init(Args)} catch - throw:R -> {ok, R}; - Class:R:S -> {'EXIT', Class, R, S} + throw:R -> {ok, R}; + Class:R:S -> {'EXIT', Class, R, S} end. %%%======================================================================== @@ -861,58 +879,68 @@ init_it(Mod, Args) -> %%% The MAIN loop. %%% --------------------------------------------------- -loop(Parent, Name, State, Mod, {continue, Continue} = Msg, HibernateAfterTimeout, Debug) -> - Reply = try_dispatch(Mod, handle_continue, Continue, State), +loop(Parent, Name, State, CbCache, {continue, Continue} = Msg, HibernateAfterTimeout, Debug) -> + Reply = try_handle_continue(CbCache, Continue, State), case Debug of - [] -> - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, - HibernateAfterTimeout, State); - _ -> - Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, Msg), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, - HibernateAfterTimeout, State, Debug1) + [] -> + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, + HibernateAfterTimeout, State); + _ -> + Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, Msg), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, + HibernateAfterTimeout, State, Debug1) end; -loop(Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug) -> +loop(Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug) -> + Mod = CbCache#callback_cache.module, proc_lib:hibernate(?MODULE,wake_hib,[Parent, Name, State, Mod, HibernateAfterTimeout, Debug]); -loop(Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug) -> - receive - Msg -> - decode_msg(Msg, Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug, false) - after HibernateAfterTimeout -> - loop(Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug) - end; +loop(Parent, Name, State, CbCache, infinity, HibernateAfterTimeout, Debug) -> + receive + Msg -> + decode_msg(Msg, Parent, Name, State, CbCache, infinity, HibernateAfterTimeout, Debug, false) + after HibernateAfterTimeout -> + loop(Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug) + end; -loop(Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug) -> +loop(Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug) -> Msg = receive - Input -> - Input - after Time -> - timeout - end, - decode_msg(Msg, Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug, false). + Input -> + Input + after Time -> + timeout + end, + decode_msg(Msg, Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug, false). + +-spec create_callback_cache(module()) -> #callback_cache{}. +create_callback_cache(Mod) -> + #callback_cache{module = Mod, + handle_call = fun Mod:handle_call/3, + handle_cast = fun Mod:handle_cast/2, + handle_info = fun Mod:handle_info/2}. wake_hib(Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> Msg = receive - Input -> - Input - end, - decode_msg(Msg, Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug, true). + Input -> + Input + end, + CbCache = create_callback_cache(Mod), + decode_msg(Msg, Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug, true). -decode_msg(Msg, Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug, Hib) -> +decode_msg(Msg, Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug, Hib) -> case Msg of - {system, From, Req} -> - sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, - [Name, State, Mod, Time, HibernateAfterTimeout], Hib); - {'EXIT', Parent, Reason} -> - terminate(Reason, ?STACKTRACE(), Name, undefined, Msg, Mod, State, Debug); - _Msg when Debug =:= [] -> - handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout); - _Msg -> - Debug1 = sys:handle_debug(Debug, fun print_event/3, - Name, {in, Msg}), - handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout, Debug1) + {system, From, Req} -> + sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, + [Name, State, CbCache, Time, HibernateAfterTimeout], Hib); + {'EXIT', Parent, Reason} -> + #callback_cache{module = Mod} = CbCache, + terminate(Reason, ?STACKTRACE(), Name, undefined, Msg, Mod, State, Debug); + _Msg when Debug =:= [] -> + handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout); + _Msg -> + Debug1 = sys:handle_debug(Debug, fun print_event/3, + Name, {in, Msg}), + handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug1) end. %%% --------------------------------------------------- @@ -1113,60 +1141,80 @@ start_monitor(Node, Name) when is_atom(Node), is_atom(Name) -> %% stacktraces. %% --------------------------------------------------- -try_dispatch({'$gen_cast', Msg}, Mod, State) -> - try_dispatch(Mod, handle_cast, Msg, State); -try_dispatch(Info, Mod, State) -> - try_dispatch(Mod, handle_info, Info, State). +try_dispatch({'$gen_cast', Msg}, CbCache, State) -> + try_handle_cast(CbCache, Msg, State); +try_dispatch(Info, CbCache, State) -> + try_handle_info(CbCache, Info, State). + +try_handle_continue(#callback_cache{module = Mod}, Msg, State) -> + try + {ok, Mod:handle_continue(Msg, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end. -try_dispatch(Mod, Func, Msg, State) -> +try_handle_info(#callback_cache{module = Mod, handle_info = HandleInfo}, Msg, State) -> try - {ok, Mod:Func(Msg, State)} + {ok, HandleInfo(Msg, State)} catch - throw:R -> - {ok, R}; - error:undef = R:Stacktrace when Func == handle_info -> + throw:R -> + {ok, R}; + error:undef = R:Stacktrace -> case erlang:function_exported(Mod, handle_info, 2) of false -> ?LOG_WARNING( - #{label=>{gen_server,no_handle_info}, - module=>Mod, - message=>Msg}, - #{domain=>[otp], - report_cb=>fun gen_server:format_log/2, - error_logger=> - #{tag=>warning_msg, - report_cb=>fun gen_server:format_log/1}}), + #{label=>{gen_server,no_handle_info}, + module=>Mod, + message=>Msg}, + #{domain=>[otp], + report_cb=>fun gen_server:format_log/2, + error_logger=> + #{tag=>warning_msg, + report_cb=>fun gen_server:format_log/1}}), {ok, {noreply, State}}; true -> {'EXIT', error, R, Stacktrace} end; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end. + +try_handle_cast(#callback_cache{handle_cast = HandleCast}, Msg, State) -> + try + {ok, HandleCast(Msg, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. -try_handle_call(Mod, Msg, From, State) -> +try_handle_call(#callback_cache{handle_call = HandleCall}, Msg, From, State) -> try - {ok, Mod:handle_call(Msg, From, State)} + {ok, HandleCall(Msg, From, State)} catch - throw:R -> - {ok, R}; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_terminate(Mod, Reason, State) -> case erlang:function_exported(Mod, terminate, 2) of - true -> - try - {ok, Mod:terminate(Reason, State)} - catch - throw:R -> - {ok, R}; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} - end; - false -> - {ok, ok} + true -> + try + {ok, Mod:terminate(Reason, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end; + false -> + {ok, ok} end. @@ -1174,69 +1222,72 @@ try_terminate(Mod, Reason, State) -> %%% Message handling functions %%% --------------------------------------------------- -handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, HibernateAfterTimeout) -> - Result = try_handle_call(Mod, Msg, From, State), +handle_msg({'$gen_call', From, Msg}, Parent, Name, State, CbCache, HibernateAfterTimeout) -> + Result = try_handle_call(CbCache, Msg, From, State), case Result of {ok, {reply, Reply, NState}} -> reply(From, Reply), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, []); {ok, {reply, Reply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> reply(From, Reply), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, []); {ok, {reply, Reply, NState, {continue, _}=Continue}} -> reply(From, Reply), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, []); {ok, {stop, Reason, Reply, NState}} -> try + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, []) after reply(From, Reply) end; - Other -> handle_common_reply(Other, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State) + Other -> handle_common_reply(Other, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State) end; -handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout) -> - Reply = try_dispatch(Msg, Mod, State), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, HibernateAfterTimeout, State). +handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout) -> + Reply = try_dispatch(Msg, CbCache, State), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, HibernateAfterTimeout, State). -handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> - Result = try_handle_call(Mod, Msg, From, State), +handle_msg({'$gen_call', From, Msg}, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug) -> + Result = try_handle_call(CbCache, Msg, From, State), case Result of {ok, {reply, Reply, NState}} -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, Debug1); {ok, {reply, Reply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); {ok, {reply, Reply, NState, {continue, _}=Continue}} -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, Debug1); {ok, {stop, Reason, Reply, NState}} -> try + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, Debug) after _ = reply(Name, From, Reply, NState, Debug) end; Other -> - handle_common_reply(Other, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State, Debug) + handle_common_reply(Other, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State, Debug) end; -handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> - Reply = try_dispatch(Msg, Mod, State), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, HibernateAfterTimeout, State, Debug). +handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug) -> + Reply = try_dispatch(Msg, CbCache, State), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, HibernateAfterTimeout, State, Debug). -handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State) -> +handle_common_reply(Reply, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State) -> + Mod = CbCache#callback_cache.module, case Reply of {ok, {noreply, NState}} -> - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, []); {ok, {noreply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, []); {ok, {noreply, NState, {continue, _}=Continue}} -> - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, []); {ok, {stop, Reason, NState}} -> terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, []); {'EXIT', Class, Reason, Stacktrace} -> @@ -1245,20 +1296,21 @@ handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, terminate({bad_return_value, BadReply}, ?STACKTRACE(), Name, From, Msg, Mod, State, []) end. -handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State, Debug) -> +handle_common_reply(Reply, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State, Debug) -> + Mod = CbCache#callback_cache.module, case Reply of {ok, {noreply, NState}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, Debug1); {ok, {noreply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); {ok, {noreply, NState, {continue, _}=Continue}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, Debug1); {ok, {stop, Reason, NState}} -> terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, Debug); {'EXIT', Class, Reason, Stacktrace} -> @@ -1270,32 +1322,34 @@ handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, reply(Name, From, Reply, State, Debug) -> reply(From, Reply), sys:handle_debug(Debug, fun print_event/3, Name, - {out, Reply, From, State} ). + {out, Reply, From, State} ). %%----------------------------------------------------------------- %% Callback functions for system messages handling. %%----------------------------------------------------------------- -system_continue(Parent, Debug, [Name, State, Mod, Time, HibernateAfterTimeout]) -> - loop(Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug). +system_continue(Parent, Debug, [Name, State, CbCache, Time, HibernateAfterTimeout]) -> + loop(Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug). -spec system_terminate(_, _, _, [_]) -> no_return(). -system_terminate(Reason, _Parent, Debug, [Name, State, Mod, _Time, _HibernateAfterTimeout]) -> +system_terminate(Reason, _Parent, Debug, [Name, State, CbCache, _Time, _HibernateAfterTimeout]) -> + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, undefined, [], Mod, State, Debug). -system_code_change([Name, State, Mod, Time, HibernateAfterTimeout], _Module, OldVsn, Extra) -> +system_code_change([Name, State, CbCache, Time, HibernateAfterTimeout], _Module, OldVsn, Extra) -> + Mod = CbCache#callback_cache.module, case catch Mod:code_change(OldVsn, State, Extra) of - {ok, NewState} -> {ok, [Name, NewState, Mod, Time, HibernateAfterTimeout]}; - Else -> Else + {ok, NewState} -> {ok, [Name, NewState, CbCache, Time, HibernateAfterTimeout]}; + Else -> Else end. system_get_state([_Name, State, _Mod, _Time, _HibernateAfterTimeout]) -> {ok, State}. -system_replace_state(StateFun, [Name, State, Mod, Time, HibernateAfterTimeout]) -> +system_replace_state(StateFun, [Name, State, CbCache, Time, HibernateAfterTimeout]) -> NState = StateFun(State), - {ok, NState, [Name, NState, Mod, Time, HibernateAfterTimeout]}. + {ok, NState, [Name, NState, CbCache, Time, HibernateAfterTimeout]}. %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees @@ -1659,7 +1713,8 @@ mod(_) -> "t". %% Status information %%----------------------------------------------------------------- format_status(Opt, StatusData) -> - [PDict, SysState, Parent, Debug, [Name, State, Mod, _Time, _HibernateAfterTimeout]] = StatusData, + [PDict, SysState, Parent, Debug, [Name, State, CbCache, _Time, _HibernateAfterTimeout]] = StatusData, + Mod = CbCache#callback_cache.module, Header = gen:format_status_header("Status for generic server", Name), Status = case gen:format_status(Mod, Opt, #{ state => State, log => sys:get_log(Debug) }, diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index 18f6ef3dde..a6f4069314 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -209,19 +209,22 @@ get_password(Io) -> -type encoding() :: 'latin1' | 'unicode' | 'utf8' | 'utf16' | 'utf32' | {'utf16', 'big' | 'little'} | {'utf32','big' | 'little'}. --type expand_fun() :: fun((term()) -> {'yes'|'no', string(), [string(), ...]}). +-type expand_fun() :: fun((string()) -> {'yes'|'no', string(), list()}). -type opt_pair() :: {'binary', boolean()} | {'echo', boolean()} | {'expand_fun', expand_fun()} - | {'encoding', encoding()}. + | {'encoding', encoding()} + | {atom(), term()}. +-type get_opt_pair() :: opt_pair() + | {'terminal', boolean()}. --spec getopts() -> [opt_pair()] | {'error', Reason} when +-spec getopts() -> [get_opt_pair()] | {'error', Reason} when Reason :: term(). getopts() -> getopts(default_input()). --spec getopts(IoDevice) -> [opt_pair()] | {'error', Reason} when +-spec getopts(IoDevice) -> [get_opt_pair()] | {'error', Reason} when IoDevice :: device(), Reason :: term(). diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index e2823b70f2..9f7e05a740 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -350,9 +350,9 @@ write(Term, Options) when is_list(Options) -> if Depth =:= 0; CharsLimit =:= 0 -> "..."; - CharsLimit < 0 -> + is_integer(CharsLimit), CharsLimit < 0, is_integer(Depth) -> write1(Term, Depth, Encoding); - CharsLimit > 0 -> + is_integer(CharsLimit), CharsLimit > 0 -> RecDefFun = fun(_, _) -> no end, If = io_lib_pretty:intermediate (Term, Depth, CharsLimit, RecDefFun, Encoding, _Str=false), @@ -439,7 +439,7 @@ write_binary(B, D) when is_integer(D) -> {S, _} = write_binary(B, D, -1), S. -write_binary(B, D, T) -> +write_binary(B, D, T) when is_integer(T) -> {S, Rest} = write_binary_body(B, D, tsub(T, 4), []), {[$<,$<,lists:reverse(S),$>,$>], Rest}. @@ -509,15 +509,16 @@ quote_atom(Atom, Cs0) -> true -> true; false -> case Cs0 of - [C|Cs] when C >= $a, C =< $z -> + [C|Cs] when is_integer(C), C >= $a, C =< $z -> not name_chars(Cs); - [C|Cs] when C >= $ß, C =< $ÿ, C =/= $÷ -> + [C|Cs] when is_integer(C), C >= $ß, C =< $ÿ, C =/= $÷ -> not name_chars(Cs); - _ -> true + [C|_] when is_integer(C) -> true; + [] -> true end end. -name_chars([C|Cs]) -> +name_chars([C|Cs]) when is_integer(C) -> case name_char(C) of true -> name_chars(Cs); false -> false @@ -580,7 +581,7 @@ write_string_as_latin1(S, Q) -> write_string1(_,[], Q) -> [Q]; -write_string1(Enc,[C|Cs], Q) -> +write_string1(Enc,[C|Cs], Q) when is_integer(C) -> string_char(Enc,C, Q, write_string1(Enc,Cs, Q)). string_char(_,Q, Q, Tail) -> [$\\,Q|Tail]; %Must check these first! @@ -803,7 +804,7 @@ collect_chars(Tag, Data, N) -> collect_chars(Tag, Data, latin1, N). %% Now we are aware of encoding... -collect_chars(start, Data, unicode, N) when is_binary(Data) -> +collect_chars(start, Data, unicode, N) when is_binary(Data), is_integer(N) -> {Size,Npos} = count_and_find_utf8(Data,N), if Size > N -> {B1,B2} = split_binary(Data, Npos), @@ -813,7 +814,7 @@ collect_chars(start, Data, unicode, N) when is_binary(Data) -> true -> {stop,Data,eof} end; -collect_chars(start, Data, latin1, N) when is_binary(Data) -> +collect_chars(start, Data, latin1, N) when is_binary(Data), is_integer(N) -> Size = byte_size(Data), if Size > N -> {B1,B2} = split_binary(Data, N), @@ -823,13 +824,13 @@ collect_chars(start, Data, latin1, N) when is_binary(Data) -> true -> {stop,Data,eof} end; -collect_chars(start,Data,_,N) when is_list(Data) -> +collect_chars(start,Data,_,N) when is_list(Data), is_integer(N) -> collect_chars_list([], N, Data); collect_chars(start, eof, _,_) -> {stop,eof,eof}; collect_chars({binary,Stack,_N}, eof, _,_) -> {stop,binrev(Stack),eof}; -collect_chars({binary,Stack,N}, Data,unicode, _) -> +collect_chars({binary,Stack,N}, Data,unicode, _) when is_integer(N) -> {Size,Npos} = count_and_find_utf8(Data,N), if Size > N -> {B1,B2} = split_binary(Data, Npos), @@ -839,7 +840,7 @@ collect_chars({binary,Stack,N}, Data,unicode, _) -> true -> {stop,binrev(Stack, [Data]),eof} end; -collect_chars({binary,Stack,N}, Data,latin1, _) -> +collect_chars({binary,Stack,N}, Data,latin1, _) when is_integer(N) -> Size = byte_size(Data), if Size > N -> {B1,B2} = split_binary(Data, N), @@ -849,7 +850,7 @@ collect_chars({binary,Stack,N}, Data,latin1, _) -> true -> {stop,binrev(Stack, [Data]),eof} end; -collect_chars({list,Stack,N}, Data, _,_) -> +collect_chars({list,Stack,N}, Data, _,_) when is_integer(N) -> collect_chars_list(Stack, N, Data); %% collect_chars(Continuation, MoreChars, Count) @@ -857,9 +858,9 @@ collect_chars({list,Stack,N}, Data, _,_) -> %% {done,Result,RestChars} %% {more,Continuation} -collect_chars([], Chars, _, N) -> +collect_chars([], Chars, _, N) when is_integer(N) -> collect_chars1(N, Chars, []); -collect_chars({Left,Sofar}, Chars, _, _N) -> +collect_chars({Left,Sofar}, Chars, _, _N) when is_integer(Left) -> collect_chars1(Left, Chars, Sofar). collect_chars1(N, Chars, Stack) when N =< 0 -> @@ -991,13 +992,13 @@ binrev(L) -> binrev(L, T) -> list_to_binary(lists:reverse(L, T)). --spec limit_term(term(), non_neg_integer()) -> term(). +-spec limit_term(term(), depth()) -> term(). %% The intention is to mimic the depth limitation of io_lib:write() %% and io_lib_pretty:print(). The leaves ('...') should never be %% seen when printed with the same depth. Bitstrings are never %% truncated, which is OK as long as they are not sent to other nodes. -limit_term(Term, Depth) -> +limit_term(Term, Depth) when is_integer(Depth), Depth >= -1 -> try test_limit(Term, Depth) of ok -> Term catch diff --git a/lib/stdlib/src/peer.erl b/lib/stdlib/src/peer.erl index ecf33b139b..331e0706b4 100644 --- a/lib/stdlib/src/peer.erl +++ b/lib/stdlib/src/peer.erl @@ -123,9 +123,12 @@ %% saving exit reason in the state %% crash: when peer terminates, origin process %% terminates with underlying reason - exec => exec(), %% path to executable, or SSH/Docker support connection => connection(), %% alternative connection specification + exec => exec(), %% path to executable, or SSH/Docker support + detached => boolean(), %% if the node should be start in detached mode args => [string()], %% additional command line parameters to append + post_process_args => + fun(([string()]) -> [string()]),%% fix the arguments env => [{string(), string()}], %% additional environment variables wait_boot => wait_boot(), %% default is synchronous start with 15 sec timeout shutdown => close | %% close supervision channel @@ -291,14 +294,20 @@ init([Notify, Options]) -> Env = maps:get(env, Options, []), + PostProcessArgs = maps:get(post_process_args, Options, fun(As) -> As end), + FinalArgs = PostProcessArgs(Args), + %% close port if running detached Conn = case maps:find(connection, Options) of {ok, standard_io} -> %% Cannot detach a peer that uses stdio. Request exit_status. - open_port({spawn_executable, Exec}, [{args, Args}, {env, Env}, hide, binary, exit_status]); + open_port({spawn_executable, Exec}, + [{args, FinalArgs}, {env, Env}, hide, + binary, exit_status, stderr_to_stdout]); _ -> - Port = open_port({spawn_executable, Exec}, [{args, Args}, {env, Env}, hide, binary]), + Port = open_port({spawn_executable, Exec}, + [{args, FinalArgs}, {env, Env}, hide, binary]), %% peer can close the port before we get here which will cause %% port_close to throw. Catch this and ignore. catch erlang:port_close(Port), @@ -339,6 +348,22 @@ handle_call({call, M, F, A}, From, origin_to_peer(tcp, Socket, {call, Seq, M, F, A}), {noreply, State#peer_state{outstanding = Out#{Seq => From}, seq = Seq + 1}}; +handle_call({starting, Node}, _From, #peer_state{ options = Options } = State) -> + case maps:find(shutdown, Options) of + {ok, {Timeout, MainCoverNode}} when is_integer(Timeout), + is_atom(MainCoverNode) -> + %% The node was started using test_server:start_peer/2 with cover enabled + %% so we should start cover on the starting node. + Modules = erpc:call(MainCoverNode,cover,modules,[]), + Sticky = [ begin erpc:call(Node, code, unstick_mod, [M]), M end + || M <- Modules, erpc:call(Node, code, is_sticky,[M])], + _ = erpc:call(MainCoverNode, cover, start, [Node]), + _ = [erpc:call(Node, code,stick_mod,[M]) || M <- Sticky], + ok; + _ -> + ok + end, + {reply, ok, State}; handle_call(get_node, _From, #peer_state{node = Node} = State) -> {reply, Node, State}; @@ -538,8 +563,9 @@ verify_args(Options) -> [error({invalid_arg, Arg}) || Arg <- Args, not io_lib:char_list(Arg)], %% alternative connection must be requested for non-distributed node, %% or a distributed node when origin is not alive - is_map_key(connection, Options) orelse - (is_map_key(name, Options) andalso erlang:is_alive()) orelse error(not_alive), + is_map_key(connection, Options) + orelse + (is_map_key(name, Options) andalso erlang:is_alive()) orelse error(not_alive), %% exec must be a string, or a tuple of string(), [string()] case maps:find(exec, Options) of {ok, {Exec, Strs}} -> @@ -579,8 +605,13 @@ verify_args(Options) -> ok; {ok, Err2} -> error({shutdown, Err2}) + end, + case maps:find(detached, Options) of + {ok, false} when map_get(connection, Options) =:= standard_io -> + error({detached, cannot_detach_with_standard_io}); + _ -> + ok end. - make_notify_ref(infinity) -> {self(), make_ref()}; @@ -796,6 +827,14 @@ command_line(Listen, Options) -> NameArg = name_arg(maps:find(name, Options), maps:find(host, Options), maps:find(longnames, Options)), %% additional command line args CmdOpts = maps:get(args, Options, []), + + %% If we should detach from the node. We use -detached to tell erl to detach + %% and -peer_detached to tell peer:start that we are detached. + DetachArgs = case maps:get(detached, Options, true) of + true -> ["-detached","-peer_detached"]; + false -> [] + end, + %% start command StartCmd = case Listen of @@ -803,14 +842,14 @@ command_line(Listen, Options) -> ["-user", atom_to_list(?MODULE)]; undefined -> Self = base64:encode_to_string(term_to_binary(self())), - ["-detached", "-noinput", "-user", atom_to_list(?MODULE), "-origin", Self]; + DetachArgs ++ ["-user", atom_to_list(?MODULE), "-origin", Self]; {Ips, Port} -> IpStr = lists:concat(lists:join(",", [inet:ntoa(Ip) || Ip <- Ips])), - ["-detached", "-noinput", "-user", atom_to_list(?MODULE), "-origin", IpStr, integer_to_list(Port)] + DetachArgs ++ ["-user", atom_to_list(?MODULE), "-origin", IpStr, integer_to_list(Port)] end, %% build command line {Exec, PreArgs} = exec(Options), - {Exec, PreArgs ++ NameArg ++ StartCmd ++ CmdOpts}. + {Exec, PreArgs ++ NameArg ++ CmdOpts ++ StartCmd}. exec(#{exec := Prog}) when is_list(Prog) -> {Prog, []}; @@ -1017,23 +1056,44 @@ start_peer_channel_handler() -> {ok, [[IpStr, PortString]]} -> %% enter this clause when -origin IpList Port is specified in the command line. Port = list_to_integer(PortString), - Ips = [begin {ok, Addr} = inet:parse_address(Ip), Addr end || Ip <- string:lexemes(IpStr, ",")], - spawn(fun () -> tcp_init(Ips, Port) end); + Ips = [begin {ok, Addr} = inet:parse_address(Ip), Addr end || + Ip <- string:lexemes(IpStr, ",")], + TCPConnection = spawn(fun () -> tcp_init(Ips, Port) end), + _ = case init:get_argument(peer_detached) of + {ok, _} -> + _ = register(user, TCPConnection); + error -> + _= user_sup:init( + [Flag || Flag <- init:get_arguments(), Flag =/= {user,["peer"]}]) + end, + TCPConnection; {ok, [[Base64EncProc]]} -> %% No alternative connection, but have "-origin Base64EncProc" OriginProcess = binary_to_term(base64:decode(Base64EncProc)), - %% setup 'user' process, I/O redirection: ask controlling process - %% who is the group leader. - GroupLeader = gen_server:call(OriginProcess, group_leader), - RelayPid = spawn(fun () -> relay(GroupLeader) end), - register(user, RelayPid), - spawn( - fun () -> - link(RelayPid), - MRef = monitor(process, OriginProcess), - notify_when_started(dist, OriginProcess), - origin_link(MRef, OriginProcess) - end); + OriginLink = spawn( + fun () -> + MRef = monitor(process, OriginProcess), + notify_when_started(dist, OriginProcess), + origin_link(MRef, OriginProcess) + end), + ok = gen_server:call(OriginProcess, {starting, node()}), + _ = case init:get_argument(peer_detached) of + {ok, _} -> + %% We are detached, so setup 'user' process, I/O redirection: + %% ask controlling process who is the group leader. + GroupLeader = gen_server:call(OriginProcess, group_leader), + RelayPid = spawn(fun () -> + link(OriginLink), + relay(GroupLeader) + end), + _ = register(user, RelayPid); + error -> + %% We are not detached, so after we spawn the link process we + %% start the terminal as normal but without the -user peer flag. + _ = user_sup:init( + [Flag || Flag <- init:get_arguments(), Flag =/= {user,["peer"]}]) + end, + OriginLink; error -> %% no -origin specified, meaning that standard I/O is used for alternative spawn(fun io_server/0) @@ -1071,7 +1131,6 @@ io_server() -> tcp_init(IpList, Port) -> try Sock = loop_connect(IpList, Port), - register(user, self()), erlang:group_leader(self(), self()), notify_when_started(tcp, Sock), io_server_loop(tcp, Sock, #{}, #{}, undefined) diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 5f8d3a4c59..e2f0278627 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -1502,8 +1502,7 @@ dummy_seed({A1, A2, A3}) -> -type mwc59_state() :: 1..?MWC59_P-1. -spec mwc59(CX0 :: mwc59_state()) -> CX1 :: mwc59_state(). -mwc59(CX0) -> % when is_integer(CX0), 1 =< CX0, CX0 < ?MWC59_P -> - CX = ?MASK(59, CX0), +mwc59(CX) when is_integer(CX), 1 =< CX, CX < ?MWC59_P -> C = CX bsr ?MWC59_B, X = ?MASK(?MWC59_B, CX), ?MWC59_A * X + C. @@ -1521,18 +1520,17 @@ mwc59(CX0) -> % when is_integer(CX0), 1 =< CX0, CX0 < ?MWC59_P -> %%% mwc59(CX1, N - 1). -spec mwc59_value32(CX :: mwc59_state()) -> V :: 0..?MASK(32). -mwc59_value32(CX1) -> % when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> +mwc59_value32(CX1) when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> CX = ?MASK(32, CX1), CX bxor ?BSL(32, CX, ?MWC59_XS). -spec mwc59_value(CX :: mwc59_state()) -> V :: 0..?MASK(59). -mwc59_value(CX1) -> % when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> - CX = ?MASK(59, CX1), +mwc59_value(CX) when is_integer(CX), 1 =< CX, CX < ?MWC59_P -> CX2 = CX bxor ?BSL(59, CX, ?MWC59_XS1), CX2 bxor ?BSL(59, CX2, ?MWC59_XS2). -spec mwc59_float(CX :: mwc59_state()) -> V :: float(). -mwc59_float(CX1) -> +mwc59_float(CX1) when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> CX = ?MASK(53, CX1), CX2 = CX bxor ?BSL(53, CX, ?MWC59_XS1), CX3 = CX2 bxor ?BSL(53, CX2, ?MWC59_XS2), diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 863bbeb652..3a3eca8f44 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -31,6 +31,8 @@ | bsr_anycrlf | bsr_unicode | no_start_optimize | ucp | never_utf. +-type replace_fun() :: fun((binary(), [binary()]) -> iodata() | unicode:charlist()). + %%% BIFs -export([internal_run/4]). @@ -353,7 +355,7 @@ compile_split(_,_) -> -spec replace(Subject, RE, Replacement) -> iodata() | unicode:charlist() when Subject :: iodata() | unicode:charlist(), RE :: mp() | iodata(), - Replacement :: iodata() | unicode:charlist(). + Replacement :: iodata() | unicode:charlist() | replace_fun(). replace(Subject,RE,Replacement) -> try @@ -366,7 +368,7 @@ replace(Subject,RE,Replacement) -> -spec replace(Subject, RE, Replacement, Options) -> iodata() | unicode:charlist() when Subject :: iodata() | unicode:charlist(), RE :: mp() | iodata() | unicode:charlist(), - Replacement :: iodata() | unicode:charlist(), + Replacement :: iodata() | unicode:charlist() | replace_fun(), Options :: [Option], Option :: anchored | global | notbol | noteol | notempty | notempty_atstart @@ -380,11 +382,11 @@ replace(Subject,RE,Replacement) -> replace(Subject,RE,Replacement,Options) -> try - {NewOpt,Convert} = process_repl_params(Options,iodata), - Unicode = check_for_unicode(RE, Options), - FlatSubject = to_binary(Subject, Unicode), - FlatReplacement = to_binary(Replacement, Unicode), - IoList = do_replace(FlatSubject,Subject,RE,FlatReplacement,NewOpt), + {NewOpt,Convert} = process_repl_params(Options,iodata), + Unicode = check_for_unicode(RE, Options), + FlatSubject = to_binary(Subject, Unicode), + Replacement1 = normalize_replacement(Replacement, Unicode), + IoList = do_replace(FlatSubject,Subject,RE,Replacement1,NewOpt), case Convert of iodata -> IoList; @@ -412,6 +414,10 @@ replace(Subject,RE,Replacement,Options) -> badarg_with_info([Subject,RE,Replacement,Options]) end. +normalize_replacement(Replacement, _Unicode) when is_function(Replacement, 2) -> + Replacement; +normalize_replacement(Replacement, Unicode) -> + to_binary(Replacement, Unicode). do_replace(FlatSubject,Subject,RE,Replacement,Options) -> case re:run(FlatSubject,RE,Options) of @@ -512,7 +518,9 @@ precomp_repl(<<X,Rest/binary>>) -> [<<X,BHead/binary>> | T0]; Other -> [<<X>> | Other] - end. + end; +precomp_repl(Repl) when is_function(Repl) -> + Repl. @@ -540,6 +548,16 @@ do_mlist(Whole,Subject,Pos,Repl,[[{MPos,Count} | Sub] | Tail]) [NewData | do_mlist(Whole,Rest,Pos+EatLength,Repl,Tail)]. +do_replace(Subject, Repl, SubExprs0) when is_function(Repl) -> + All = binary:part(Subject, hd(SubExprs0)), + SubExprs1 = + [ if + Pos >= 0, Len > 0 -> + binary:part(Subject, Pos, Len); + true -> + <<>> + end || {Pos, Len} <- tl(SubExprs0) ], + Repl(All, SubExprs1); do_replace(_,[Bin],_) when is_binary(Bin) -> Bin; do_replace(Subject,Repl,SubExprs0) -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 7de78758b0..c8610b23ee 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -20,10 +20,13 @@ -module(shell). -export([start/0, start/1, start/2, server/1, server/2, history/1, results/1]). --export([whereis_evaluator/0, whereis_evaluator/1]). +-export([get_state/0, get_function/2]). -export([start_restricted/1, stop_restricted/0]). --export([local_allowed/3, non_local_allowed/3]). +-export([local_func/0, local_func/1, local_allowed/3, non_local_allowed/3]). -export([catch_exception/1, prompt_func/1, strings/1]). +-export([start_interactive/0, start_interactive/1]). +-export([read_and_add_records/5]). +-export([whereis/0]). -define(LINEMAX, 30). -define(CHAR_MAX, 60). @@ -36,7 +39,11 @@ -define(RECORDS, shell_records). -define(MAXSIZE_HEAPBINARY, 64). - +-record(shell_state,{ + bindings = [], + records = [], + functions = [] + }). %% When used as the fallback restricted shell callback module... local_allowed(q,[],State) -> {true,State}; @@ -48,6 +55,24 @@ non_local_allowed({init,stop},[],State) -> non_local_allowed(_,_,State) -> {false,State}. +-spec start_interactive() -> ok | {error, already_started}. +start_interactive() -> + user_drv:start_shell(). +-spec start_interactive(noshell | mfa()) -> + ok | {error, already_started}; + ({remote, string()}) -> + ok | {error, already_started | noconnection}; + ({node(), mfa()} | {remote, string(), mfa()}) -> + ok | {error, already_started | noconnection | badfile | nofile | on_load_failure}. +start_interactive({Node, {M, F, A}}) -> + user_drv:start_shell(#{ initial_shell => {Node, M, F ,A} }); +start_interactive(InitialShell) -> + user_drv:start_shell(#{ initial_shell => InitialShell }). + +-spec whereis() -> pid() | undefined. +whereis() -> + group:whereis_shell(). + -spec start() -> pid(). start() -> @@ -60,62 +85,16 @@ start(NoCtrlG) -> start(NoCtrlG, StartSync) -> _ = code:ensure_loaded(user_default), - spawn(fun() -> server(NoCtrlG, StartSync) end). - -%% Find the pid of the current evaluator process. --spec whereis_evaluator() -> 'undefined' | pid(). - -whereis_evaluator() -> - %% locate top group leader, always registered as user - %% can be implemented by group (normally) or user - %% (if oldshell or noshell) - case whereis(user) of - undefined -> - undefined; - User -> - %% get user_drv pid from group, or shell pid from user - case group:interfaces(User) of - [] -> % old- or noshell - case user:interfaces(User) of - [] -> - undefined; - [{shell,Shell}] -> - whereis_evaluator(Shell) - end; - [{user_drv,UserDrv}] -> - %% get current group pid from user_drv - case user_drv:interfaces(UserDrv) of - [] -> - undefined; - [{current_group,Group}] -> - %% get shell pid from group - GrIfs = group:interfaces(Group), - case lists:keyfind(shell, 1, GrIfs) of - {shell, Shell} -> - whereis_evaluator(Shell); - false -> - undefined - end - end - end - end. - --spec whereis_evaluator(pid()) -> 'undefined' | pid(). - -whereis_evaluator(Shell) -> - case process_info(Shell, dictionary) of - {dictionary,Dict} -> - case lists:keyfind(evaluator, 1, Dict) of - {_, Eval} when is_pid(Eval) -> - Eval; - _ -> - undefined - end; - _ -> - undefined - end. - -%% Call this function to start a user restricted shell + Ancestors = [self() | case get('$ancestors') of + undefined -> []; + Anc -> Anc + end], + spawn(fun() -> + put('$ancestors', Ancestors), + server(NoCtrlG, StartSync) + end). + +%% Call this function to start a user restricted shell %% from a normal shell session. -spec start_restricted(Module) -> {'error', Reason} when Module :: module(), @@ -123,16 +102,16 @@ whereis_evaluator(Shell) -> start_restricted(RShMod) when is_atom(RShMod) -> case code:ensure_loaded(RShMod) of - {module,RShMod} -> - application:set_env(stdlib, restricted_shell, RShMod), + {module,RShMod} -> + application:set_env(stdlib, restricted_shell, RShMod), exit(restricted_shell_started); - {error,What} = Error -> - error_logger:error_report( - lists:flatten( - io_lib:fwrite( - "Restricted shell module ~w not found: ~tp\n", - [RShMod,What]))), - Error + {error,What} = Error -> + error_logger:error_report( + lists:flatten( + io_lib:fwrite( + "Restricted shell module ~w not found: ~tp\n", + [RShMod,What]))), + Error end. -spec stop_restricted() -> no_return(). @@ -152,27 +131,27 @@ server(NoCtrlG, StartSync) -> %%% We subscribe with init to get a notification of when. %%% In older releases we didn't syncronize the shell with init, but let it -%%% start in parallell with other system processes. This was bad since +%%% start in parallell with other system processes. This was bad since %%% accessing the shell too early could interfere with the boot procedure. %%% Still, by means of a flag, we make it possible to start the shell the -%%% old way (for backwards compatibility reasons). This should however not +%%% old way (for backwards compatibility reasons). This should however not %%% be used unless for very special reasons necessary. -spec server(boolean()) -> 'terminated'. server(StartSync) -> case init:get_argument(async_shell_start) of - {ok,_} -> - ok; % no sync with init - _ when not StartSync -> - ok; - _ -> - case init:notify_when_started(self()) of - started -> - ok; - _ -> - init:wait_until_started() - end + {ok,_} -> + ok; % no sync with init + _ when not StartSync -> + ok; + _ -> + case init:notify_when_started(self()) of + started -> + ok; + _ -> + init:wait_until_started() + end end, %% Our spawner has fixed the process groups. Bs = erl_eval:new_bindings(), @@ -183,57 +162,71 @@ server(StartSync) -> RT = ets:new(?RECORDS, [public,ordered_set]), _ = initiate_records(Bs, RT), process_flag(trap_exit, true), + %% Store function definitions and types in an ets table. + FT = ets:new(user_functions, [public,ordered_set]), %% Check if we're in user restricted mode. - RShErr = - case application:get_env(stdlib, restricted_shell) of - {ok,RShMod} when is_atom(RShMod) -> - io:fwrite(<<"Restricted ">>, []), - case code:ensure_loaded(RShMod) of - {module,RShMod} -> - undefined; - {error,What} -> - {RShMod,What} - end; + RShErr = + case application:get_env(stdlib, restricted_shell) of + {ok,RShMod} when is_atom(RShMod) -> + io:fwrite(<<"Restricted ">>, []), + case code:ensure_loaded(RShMod) of + {module,RShMod} -> + undefined; + {error,What} -> + {RShMod,What} + end; {ok, Term} -> {Term,not_an_atom}; - undefined -> - undefined - end, - - case get(no_control_g) of - true -> - io:fwrite(<<"Eshell V~s\n">>, [erlang:system_info(version)]); - _undefined_or_false -> - io:fwrite(<<"Eshell V~s (abort with ^G)\n">>, - [erlang:system_info(version)]) + undefined -> + undefined + end, + + JCL = + case get(no_control_g) of + true -> " (type help(). for help)"; + _ -> " (press Ctrl+G to abort, type help(). for help)" + end, + DefaultSessionSlogan = + io_lib:format(<<"Eshell V~s">>, [erlang:system_info(version)]), + SessionSlogan = + case application:get_env(stdlib, shell_session_slogan, DefaultSessionSlogan) of + SloganFun when is_function(SloganFun, 0) -> + SloganFun(); + Slogan -> + Slogan + end, + try + io:fwrite("~ts~ts\n",[unicode:characters_to_list(SessionSlogan),JCL]) + catch _:_ -> + io:fwrite("Warning! The slogan \"~p\" could not be printed.\n",[SessionSlogan]) end, erase(no_control_g), case RShErr of - undefined -> + undefined -> ok; - {RShMod2,What2} -> + {RShMod2,What2} -> io:fwrite( - ("Warning! Restricted shell module ~w not found: ~tp.\n" - "Only the commands q() and init:stop() will be allowed!\n"), + ("Warning! Restricted shell module ~w not found: ~tp.\n" + "Only the commands q() and init:stop() will be allowed!\n"), [RShMod2,What2]), application:set_env(stdlib, restricted_shell, ?MODULE) end, {History,Results} = check_and_get_history_and_results(), - server_loop(0, start_eval(Bs, RT, []), Bs, RT, [], History, Results). + server_loop(0, start_eval(Bs, RT, FT, []), Bs, RT, FT, [], History, Results). -server_loop(N0, Eval_0, Bs00, RT, Ds00, History0, Results0) -> +server_loop(N0, Eval_0, Bs00, RT, FT, Ds00, History0, Results0) -> N = N0 + 1, - {Eval_1,Bs0,Ds0,Prompt} = prompt(N, Eval_0, Bs00, RT, Ds00), - {Res,Eval0} = get_command(Prompt, Eval_1, Bs0, RT, Ds0), + {Eval_1,Bs0,Ds0,Prompt} = prompt(N, Eval_0, Bs00, RT, FT, Ds00), + {Res,Eval0} = get_command(Prompt, Eval_1, Bs0, RT, FT, Ds0), - case Res of - {ok,Es0} -> + case Res of + {ok,Es0} -> case expand_hist(Es0, N) of {ok,Es} -> - {V,Eval,Bs,Ds} = shell_cmd(Es, Eval0, Bs0, RT, Ds0, cmd), + {V,Eval,Bs,Ds} = shell_cmd(Es, Eval0, Bs0, RT, FT, Ds0, cmd), {History,Results} = check_and_get_history_and_results(), add_cmd(N, Es, V), HB1 = del_cmd(command, N - History, N - History0, false), @@ -247,42 +240,83 @@ server_loop(N0, Eval_0, Bs00, RT, Ds00, History0, Results0) -> true -> ok end, - server_loop(N, Eval, Bs, RT, Ds, History, Results); + server_loop(N, Eval, Bs, RT, FT, Ds, History, Results); {error,E} -> fwrite_severity(benign, <<"~ts">>, [E]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0) + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0) end; - {error,{Location,Mod,What}} -> + {error,{Location,Mod,What}} -> fwrite_severity(benign, <<"~s: ~ts">>, [pos(Location), Mod:format_error(What)]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0); - {error,terminated} -> %Io process terminated - exit(Eval0, kill), - terminated; - {error,interrupted} -> %Io process interrupted us - exit(Eval0, kill), - {_,Eval,_,_} = shell_rep(Eval0, Bs0, RT, Ds0), - server_loop(N0, Eval, Bs0, RT, Ds0, History0, Results0); - {error,tokens} -> %Most probably character > 255 - fwrite_severity(benign, <<"~w: Invalid tokens.">>, + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0); + {error,terminated} -> %Io process terminated + exit(Eval0, kill), + terminated; + {error,interrupted} -> %Io process interrupted us + exit(Eval0, kill), + {_,Eval,_,_} = shell_rep(Eval0, Bs0, RT, FT, Ds0), + server_loop(N0, Eval, Bs0, RT, FT, Ds0, History0, Results0); + {error,tokens} -> %Most probably character > 255 + fwrite_severity(benign, <<"~w: Invalid tokens.">>, [N]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0); - eof -> + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0); + eof -> fwrite_severity(fatal, <<"Terminating erlang (~w)">>, [node()]), - halt() + halt() end. -get_command(Prompt, Eval, Bs, RT, Ds) -> +get_command(Prompt, Eval, Bs, RT, FT, Ds) -> + Ancestors = [self() | get('$ancestors')], ResWordFun = fun erl_scan:reserved_word/1, Parse = fun() -> + put('$ancestors', Ancestors), exit( case io:scan_erl_exprs(group_leader(), Prompt, {1,1}, [text,{reserved_word_fun,ResWordFun}]) of {ok,Toks,_EndPos} -> - erl_eval:extended_parse_exprs(Toks); + %% NOTE: we can handle function definitions, records and soon type declarations + %% but this cannot be handled by the function which only expects erl_parse:abstract_expressions() + %% for now just pattern match against those types and pass the string to shell local func. + case Toks of + [{'-', _}, {atom, _, Atom}|_] -> + SpecialCase = fun(LocalFunc) -> + FakeLine = begin + case erl_parse:parse_form(Toks) of + {ok, Def} -> lists:flatten(erl_pp:form(Def)); + E -> + exit(E) + end + end, + {done, {ok, FakeResult, _}, _} = erl_scan:tokens( + [], atom_to_list(LocalFunc) ++ "(\""++FakeLine++"\").\n", + {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]), + erl_eval:extended_parse_exprs(FakeResult) + end, + case Atom of + record -> SpecialCase(rd); + spec -> SpecialCase(ft); + type -> SpecialCase(td) + end; + [{atom, _, FunName}, {'(', _}|_] -> + case erl_parse:parse_form(Toks) of + {ok, FunDef} -> + case {edlin_expand:shell_default_or_bif(atom_to_list(FunName)), shell:local_func(FunName)} of + {"user_defined", false} -> + FakeLine =reconstruct(FunDef, FunName), + {done, {ok, FakeResult, _}, _} = erl_scan:tokens( + [], "fd("++ atom_to_list(FunName) ++ ", " ++ FakeLine ++ ").\n", + {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]), + erl_eval:extended_parse_exprs(FakeResult); + _ -> erl_eval:extended_parse_exprs(Toks) + end; + _ -> erl_eval:extended_parse_exprs(Toks) + end; + _ -> + erl_eval:extended_parse_exprs(Toks) + end; {eof,_EndPos} -> eof; {error,ErrorInfo,_EndPos} -> @@ -297,30 +331,62 @@ get_command(Prompt, Eval, Bs, RT, Ds) -> Else -> Else end - ) + ) end, Pid = spawn_link(Parse), - get_command1(Pid, Eval, Bs, RT, Ds). - -get_command1(Pid, Eval, Bs, RT, Ds) -> + get_command1(Pid, Eval, Bs, RT, FT, Ds). + +reconstruct(Fun, Name) -> + lists:flatten(erl_pp:expr(reconstruct1(Fun, Name))). +reconstruct1({function, Anno, Name, Arity, Clauses}, Name) -> + {named_fun, Anno, 'RecursiveFuncVar', reconstruct1(Clauses, Name, Arity)}. +reconstruct1([{call, Anno, {atom, Anno1, Name}, Args}|Body], Name, Arity) when length(Args) =:= Arity -> + [{call, Anno, {var, Anno1, 'RecursiveFuncVar'}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; +reconstruct1([{call, Anno, {atom, Anno1, Name}, Args}|Body], Name, Arity) -> % arity not the same + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Name}}, reconstruct1(Args, Name, Arity)}| + reconstruct1(Body, Name, Arity)]; +reconstruct1([{call, Anno, {atom, Anno1, Fun}, Args}|Body], Name, Arity) -> % Name not the same + case {edlin_expand:shell_default_or_bif(atom_to_list(Fun)), shell:local_func(Fun)} of + {"user_defined", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| + reconstruct1(Body, Name, Arity)]; + {"shell_default", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; + {"erlang", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, erlang}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; + {_, true} -> + [{call, Anno, {atom, Anno1, Fun}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)] + end; +reconstruct1([E|Body], Name, Arity) when is_tuple(E) -> + [list_to_tuple(reconstruct1(tuple_to_list(E), Name, Arity))|reconstruct1(Body, Name, Arity)]; +reconstruct1([E|Body], Name, Arity) when is_list(E) -> + [reconstruct1(E, Name, Arity)|reconstruct1(Body, Name, Arity)]; +reconstruct1([E|Body], Name, Arity) -> + [E|reconstruct1(Body, Name, Arity)]; +reconstruct1([], _, _) -> []. + +get_command1(Pid, Eval, Bs, RT, FT, Ds) -> receive - {'EXIT', Pid, Res} -> - {Res, Eval}; - {'EXIT', Eval, {Reason,Stacktrace}} -> + {shell_state, From} -> + From ! {shell_state, Bs, RT, FT}, + get_command1(Pid, Eval, Bs, RT, FT, Ds); + {'EXIT', Pid, Res} -> + {Res, Eval}; + {'EXIT', Eval, {Reason,Stacktrace}} -> report_exception(error, {Reason,Stacktrace}, RT), - get_command1(Pid, start_eval(Bs, RT, Ds), Bs, RT, Ds); - {'EXIT', Eval, Reason} -> + get_command1(Pid, start_eval(Bs, RT, FT, Ds), Bs, RT, FT, Ds); + {'EXIT', Eval, Reason} -> report_exception(error, {Reason,[]}, RT), - get_command1(Pid, start_eval(Bs, RT, Ds), Bs, RT, Ds) + get_command1(Pid, start_eval(Bs, RT, FT, Ds), Bs, RT, FT, Ds) end. -prompt(N, Eval0, Bs0, RT, Ds0) -> +prompt(N, Eval0, Bs0, RT, FT, Ds0) -> case get_prompt_func() of {M,F} -> A = erl_anno:new(1), L = {cons,A,{tuple,A,[{atom,A,history},{integer,A,N}]},{nil,A}}, C = {call,A,{remote,A,{atom,A,M},{atom,A,F}},[L]}, - {V,Eval,Bs,Ds} = shell_cmd([C], Eval0, Bs0, RT, Ds0, pmt), + {V,Eval,Bs,Ds} = shell_cmd([C], Eval0, Bs0, RT, FT, Ds0, pmt), {Eval,Bs,Ds,case V of {pmt,Val} -> Val; @@ -352,8 +418,8 @@ default_prompt(N) -> %% Don't bother flattening the list irrespective of what the %% I/O-protocol states. case is_alive() of - true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); - false -> io_lib:format(<<"~w> ">>, [N]) + true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); + false -> io_lib:format(<<"~w> ">>, [N]) end. %% expand_hist(Expressions, CommandNumber) @@ -392,7 +458,7 @@ expand_expr({record_field,A,R,Name,F}, C) -> {record_field,A,expand_expr(R, C),Name,expand_expr(F, C)}; expand_expr({record,A,R,Name,Ups}, C) -> {record,A,expand_expr(R, C),Name,expand_fields(Ups, C)}; -expand_expr({record_field,A,R,F}, C) -> %This is really illegal! +expand_expr({record_field,A,R,F}, C) -> %This is really illegal! {record_field,A,expand_expr(R, C),expand_expr(F, C)}; expand_expr({block,A,Es}, C) -> {block,A,expand_exprs(Es, C)}; @@ -410,17 +476,17 @@ expand_expr({'receive',A,Cs,To,ToEs}, C) -> expand_expr({call,A,{atom,_,e},[N]}, C) -> case get_cmd(N, C) of {undefined,_,_} -> - no_command(N); - {[Ce],_V,_CommandN} -> - Ce; - {Ces,_V,_CommandN} when is_list(Ces) -> - {block,A,Ces} + no_command(N); + {[Ce],_V,_CommandN} -> + Ce; + {Ces,_V,_CommandN} when is_list(Ces) -> + {block,A,Ces} end; expand_expr({call,CA,{atom,VA,v},[N]}, C) -> case get_cmd(N, C) of {_,undefined,_} -> - no_command(N); - {Ces,_V,CommandN} when is_list(Ces) -> + no_command(N); + {Ces,_V,CommandN} when is_list(Ces) -> {call,CA,{atom,VA,v},[{integer,VA,CommandN}]} end; expand_expr({call,A,F,Args}, C) -> @@ -444,7 +510,8 @@ expand_expr({clause,A,H,G,B}, C) -> {clause,A,H, G, expand_exprs(B, C)}; expand_expr({bin,A,Fs}, C) -> {bin,A,expand_bin_elements(Fs, C)}; -expand_expr(E, _C) -> % Constants. + %expand_expr({'-'}) +expand_expr(E, _C) -> % Constants. E. expand_cs([{clause,A,P,G,B}|Cs], C) -> @@ -488,9 +555,9 @@ getc(N) -> get_cmd(Num, C) -> case catch erl_eval:expr(Num, erl_eval:new_bindings()) of - {value,N,_} when N < 0 -> getc(C+N); - {value,N,_} -> getc(N); - _Other -> {undefined,undefined,undefined} + {value,N,_} when N < 0 -> getc(C+N); + {value,N,_} -> getc(N); + _Other -> {undefined,undefined,undefined} end. del_cmd(_Type, N, N0, HasBin) when N < N0 -> @@ -521,61 +588,81 @@ has_bin(T, I) -> has_bin(element(I, T)), has_bin(T, I - 1). +get_state() -> + whereis() ! {shell_state, self()}, + receive + {shell_state, Bs, RT, FT} -> + #shell_state{bindings = Bs, records = ets:tab2list(RT), functions = ets:tab2list(FT)} + end. + +get_function(Func, Arity) -> + {shell_state, _Bs, _RT, FT} = get_state(), + try + {value, {_, Fun}} = lists:keysearch({function, {shell_default,Func,Arity}}, 1, FT), + Fun + catch _:_ -> + undefined + end. + %% shell_cmd(Sequence, Evaluator, Bindings, RecordTable, Dictionary, What) %% shell_rep(Evaluator, Bindings, RecordTable, Dictionary) -> -%% {Value,Evaluator,Bindings,Dictionary} +%% {Value,Evaluator,Bindings,Dictionary} %% Send a command to the evaluator and wait for the reply. Start a new %% evaluator if necessary. %% What = pmt | cmd. When evaluating a prompt ('pmt') the evaluated value %% must not be displayed, and it has to be returned. -shell_cmd(Es, Eval, Bs, RT, Ds, W) -> +shell_cmd(Es, Eval, Bs, RT, FT, Ds, W) -> Eval ! {shell_cmd,self(),{eval,Es}, W}, - shell_rep(Eval, Bs, RT, Ds). + shell_rep(Eval, Bs, RT, FT, Ds). -shell_rep(Ev, Bs0, RT, Ds0) -> +shell_rep(Ev, Bs0, RT, FT, Ds0) -> receive - {shell_rep,Ev,{value,V,Bs,Ds}} -> - {V,Ev,Bs,Ds}; + {shell_rep,Ev,{value,V,Bs,Ds}} -> + {V,Ev,Bs,Ds}; {shell_rep,Ev,{command_error,{Location,M,Error}}} -> fwrite_severity(benign, <<"~s: ~ts">>, [pos(Location), M:format_error(Error)]), {{'EXIT',Error},Ev,Bs0,Ds0}; {shell_req,Ev,{get_cmd,N}} -> - Ev ! {shell_rep,self(),getc(N)}, - shell_rep(Ev, Bs0, RT, Ds0); - {shell_req,Ev,get_cmd} -> - Ev ! {shell_rep,self(),get()}, - shell_rep(Ev, Bs0, RT, Ds0); - {shell_req,Ev,exit} -> - Ev ! {shell_rep,self(),exit}, - exit(normal); - {shell_req,Ev,{update_dict,Ds}} -> % Update dictionary - Ev ! {shell_rep,self(),ok}, - shell_rep(Ev, Bs0, RT, Ds); + Ev ! {shell_rep,self(),getc(N)}, + shell_rep(Ev, Bs0, RT, FT, Ds0); + {shell_req,Ev,get_cmd} -> + Ev ! {shell_rep,self(),get()}, + shell_rep(Ev, Bs0, RT, FT, Ds0); + {shell_req,Ev,exit} -> + Ev ! {shell_rep,self(),exit}, + exit(normal); + {shell_req,Ev,{update_dict,Ds}} -> % Update dictionary + Ev ! {shell_rep,self(),ok}, + shell_rep(Ev, Bs0, RT, FT, Ds); + {shell_state, From} -> + From ! {shell_state, Bs0, RT, FT}, + shell_rep(Ev, Bs0, RT, FT, Ds0); {ev_exit,{Ev,Class,Reason0}} -> % It has exited unnaturally receive {'EXIT',Ev,normal} -> ok end, - report_exception(Class, Reason0, RT), + report_exception(Class, Reason0, RT), Reason = nocatch(Class, Reason0), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; {ev_caught,{Ev,Class,Reason0}} -> % catch_exception is in effect - report_exception(Class, benign, Reason0, RT), + report_exception(Class, benign, Reason0, RT), Reason = nocatch(Class, Reason0), {{'EXIT',Reason},Ev,Bs0,Ds0}; - {'EXIT',_Id,interrupt} -> % Someone interrupted us - exit(Ev, kill), - shell_rep(Ev, Bs0, RT, Ds0); + {'EXIT',_Id,interrupt} -> % Someone interrupted us + exit(Ev, kill), + shell_rep(Ev, Bs0, RT, FT, Ds0); {'EXIT',Ev,{Reason,Stacktrace}} -> report_exception(exit, {Reason,Stacktrace}, RT), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; {'EXIT',Ev,Reason} -> report_exception(exit, {Reason,[]}, RT), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; - {'EXIT',_Id,R} -> - exit(Ev, R), - exit(R); - _Other -> % Ignore everything else - shell_rep(Ev, Bs0, RT, Ds0) + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; + {'EXIT',_Id,R} -> + exit(Ev, R), + exit(R); + _Other -> % Ignore everything else + io:format("Throwing ~p~n", [_Other]), + shell_rep(Ev, Bs0, RT, FT, Ds0) end. nocatch(throw, {Term,Stack}) -> @@ -599,9 +686,13 @@ report_exception(Class, Severity, {Reason,Stacktrace}, RT) -> {put_chars, unicode, Str}, nl]). -start_eval(Bs, RT, Ds) -> +start_eval(Bs, RT, FT, Ds) -> Self = self(), - Eval = spawn_link(fun() -> evaluator(Self, Bs, RT, Ds) end), + Ancestors = [self() | get('$ancestors')], + Eval = spawn_link(fun() -> + put('$ancestors', Ancestors), + evaluator(Self, Bs, RT, FT, Ds) + end), put(evaluator, Eval), Eval. @@ -609,45 +700,45 @@ start_eval(Bs, RT, Ds) -> %% Evaluate expressions from the shell. Use the "old" variable bindings %% and dictionary. -evaluator(Shell, Bs, RT, Ds) -> +evaluator(Shell, Bs, RT, FT, Ds) -> init_dict(Ds), case application:get_env(stdlib, restricted_shell) of - undefined -> - eval_loop(Shell, Bs, RT); - {ok,RShMod} -> - case get(restricted_shell_state) of - undefined -> put(restricted_shell_state, []); - _ -> ok - end, - put(restricted_expr_state, []), - restricted_eval_loop(Shell, Bs, RT, RShMod) + undefined -> + eval_loop(Shell, Bs, RT, FT); + {ok,RShMod} -> + case get(restricted_shell_state) of + undefined -> put(restricted_shell_state, []); + _ -> ok + end, + put(restricted_expr_state, []), + restricted_eval_loop(Shell, Bs, RT, FT, RShMod) end. -eval_loop(Shell, Bs0, RT) -> +eval_loop(Shell, Bs0, RT, FT) -> receive - {shell_cmd,Shell,{eval,Es},W} -> - Ef = {value, + {shell_cmd,Shell,{eval,Es},W} -> + Ef = {value, fun(MForFun, As) -> apply_fun(MForFun, As, Shell) end}, - Lf = local_func_handler(Shell, RT, Ef), + Lf = local_func_handler(Shell, RT, FT, Ef), Bs = eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W), - eval_loop(Shell, Bs, RT) + eval_loop(Shell, Bs, RT, FT) end. -restricted_eval_loop(Shell, Bs0, RT, RShMod) -> +restricted_eval_loop(Shell, Bs0, RT, FT, RShMod) -> receive - {shell_cmd,Shell,{eval,Es}, W} -> - {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT), + {shell_cmd,Shell,{eval,Es}, W} -> + {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT, FT), put(restricted_expr_state, []), Bs = eval_exprs(Es, Shell, Bs0, RT, {eval,LFH}, {value,NLFH}, W), - restricted_eval_loop(Shell, Bs, RT, RShMod) + restricted_eval_loop(Shell, Bs, RT, FT, RShMod) end. eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W) -> - try + try {R,Bs2} = exprs(Es, Bs0, RT, Lf, Ef, W), Shell ! {shell_rep,self(),R}, Bs2 - catch + catch exit:normal -> exit(normal); Class:Reason:Stacktrace -> @@ -704,14 +795,14 @@ exprs([E0|Es], Bs1, RT, Lf, Ef, Bs0, W) -> W =:= pmt -> {W,V0}; true -> case result_will_be_saved() of - true -> V0; - false -> - erlang:garbage_collect(), - ignored - end + true -> V0; + false -> + erlang:garbage_collect(), + ignored + end end, {{value,V,Bs,get()},Bs}; - true -> + true -> exprs(Es, Bs, RT, Lf, Ef, Bs0, W) end; {error,Error} -> @@ -734,7 +825,7 @@ used_record_defs(E, RT) -> %% Be careful to return a list where used records come before %% records that use them. The linter wants them ordered that way. UR = case used_records(E, [], RT, []) of - [] -> + [] -> []; L0 -> L1 = lists:zip(L0, lists:seq(1, length(L0))), @@ -782,7 +873,7 @@ used_records({call,_,{atom,_,record_info},[A,{atom,_,Name}]}) -> used_records({call,A,{tuple,_,[M,F]},As}) -> used_records({call,A,{remote,A,M,F},As}); used_records({type,_,record,[{atom,_,Name}|Fs]}) -> - {name, Name, Fs}; + {name, Name, Fs}; used_records(T) when is_tuple(T) -> {expr, tuple_to_list(T)}; used_records(E) -> @@ -801,63 +892,63 @@ severity_tag(fatal) -> <<"*** ">>; severity_tag(serious) -> <<"** ">>; severity_tag(benign) -> <<"* ">>. -restrict_handlers(RShMod, Shell, RT) -> - { fun(F,As,Binds) -> - local_allowed(F, As, RShMod, Binds, Shell, RT) +restrict_handlers(RShMod, Shell, RT, FT) -> + { fun(F,As,Binds) -> + local_allowed(F, As, RShMod, Binds, Shell, RT, FT) end, - fun(MF,As) -> - non_local_allowed(MF, As, RShMod, Shell) + fun(MF,As) -> + non_local_allowed(MF, As, RShMod, Shell) end }. -define(BAD_RETURN(M, F, V), try erlang:error(reason) - catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, + catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, [{M,F,3} | S]) end). -local_allowed(F, As, RShMod, Bs, Shell, RT) when is_atom(F) -> - {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT), - case not_restricted(F, As) of % Not restricted is the same as builtin. - % variable and record manipulations local - % to the shell process. Those are never - % restricted. - true -> - local_func(F, As, Bs, Shell, RT, {eval,LFH}, {value,NLFH}); - false -> +local_allowed(F, As, RShMod, Bs, Shell, RT, FT) when is_atom(F) -> + {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT, FT), + case not_restricted(F, As) of % Not restricted is the same as builtin. + % variable and record manipulations local + % to the shell process. Those are never + % restricted. + true -> + local_func(F, As, Bs, Shell, RT, FT, {eval,LFH}, {value,NLFH}); + false -> {AsEv,Bs1} = expr_list(As, Bs, {eval,LFH}, {value,NLFH}), - case RShMod:local_allowed(F, AsEv, {get(restricted_shell_state), - get(restricted_expr_state)}) of - {Result,{RShShSt,RShExprSt}} -> - put(restricted_shell_state, RShShSt), - put(restricted_expr_state, RShExprSt), - if not Result -> - shell_req(Shell, {update_dict,get()}), - exit({restricted_shell_disallowed,{F,AsEv}}); - true -> % This is never a builtin, - % those are handled above. - non_builtin_local_func(F,AsEv,Bs1) - end; - Unexpected -> % The user supplied non conforming module + case RShMod:local_allowed(F, AsEv, {get(restricted_shell_state), + get(restricted_expr_state)}) of + {Result,{RShShSt,RShExprSt}} -> + put(restricted_shell_state, RShShSt), + put(restricted_expr_state, RShExprSt), + if not Result -> + shell_req(Shell, {update_dict,get()}), + exit({restricted_shell_disallowed,{F,AsEv}}); + true -> % This is never a builtin, + % those are handled above. + non_builtin_local_func(F,AsEv,Bs1, FT) + end; + Unexpected -> % The user supplied non conforming module ?BAD_RETURN(RShMod, local_allowed, Unexpected) - end + end end. non_local_allowed(MForFun, As, RShMod, Shell) -> case RShMod:non_local_allowed(MForFun, As, {get(restricted_shell_state), - get(restricted_expr_state)}) of - {Result,{RShShSt,RShExprSt}} -> - put(restricted_shell_state, RShShSt), - put(restricted_expr_state, RShExprSt), - case Result of - false -> - shell_req(Shell, {update_dict,get()}), - exit({restricted_shell_disallowed,{MForFun,As}}); + get(restricted_expr_state)}) of + {Result,{RShShSt,RShExprSt}} -> + put(restricted_shell_state, RShShSt), + put(restricted_expr_state, RShExprSt), + case Result of + false -> + shell_req(Shell, {update_dict,get()}), + exit({restricted_shell_disallowed,{MForFun,As}}); {redirect, NewMForFun, NewAs} -> apply_fun(NewMForFun, NewAs, Shell); - _ -> - apply_fun(MForFun, As, Shell) - end; - Unexpected -> % The user supplied non conforming module + _ -> + apply_fun(MForFun, As, Shell) + end; + Unexpected -> % The user supplied non conforming module ?BAD_RETURN(RShMod, non_local_allowed, Unexpected) end. @@ -880,6 +971,16 @@ not_restricted(catch_exception, [_]) -> true; not_restricted(exit, []) -> true; +not_restricted(fl, []) -> + true; +not_restricted(fd, [_]) -> + true; +not_restricted(ft, [_]) -> + true; +not_restricted(td, [_]) -> + true; +not_restricted(rd, [_]) -> + true; not_restricted(rd, [_,_]) -> true; not_restricted(rf, []) -> @@ -902,7 +1003,7 @@ not_restricted(_, _) -> false. %% When erlang:garbage_collect() is called from the shell, -%% the shell process process that spawned the evaluating +%% the shell process process that spawned the evaluating %% process is garbage collected as well. %% To garbage collect the evaluating process only the command %% garbage_collect(self()). can be used. @@ -940,7 +1041,7 @@ expand_records(UsedRecords, E0) -> prep_rec({value,_CommandN,_V}=Value) -> %% erl_expand_records cannot handle the history expansion {value,_,_}. {atom,Value,ok}; -prep_rec({atom,{value,_CommandN,_V}=Value,ok}) -> +prep_rec({atom,{value,_CommandN,_V}=Value,ok}) -> %% Undo the effect of the previous clause... Value; prep_rec(T) when is_tuple(T) -> list_to_tuple(prep_rec(tuple_to_list(T))); @@ -952,39 +1053,101 @@ init_dict([{K,V}|Ds]) -> init_dict(Ds); init_dict([]) -> true. -%% local_func(Function, Args, Bindings, Shell, RecordTable, +%% local_func(Function, Args, Bindings, Shell, RecordTable, %% LocalFuncHandler, ExternalFuncHandler) -> {value,Val,Bs} %% Evaluate local functions, including shell commands. %% -%% Note that the predicate not_restricted/2 has to correspond to what's -%% handled internally - it should return 'true' for all local functions -%% handled in this module (i.e. those that are not eventually handled by +%% Note that the predicate not_restricted/2 has to correspond to what's +%% handled internally - it should return 'true' for all local functions +%% handled in this module (i.e. those that are not eventually handled by %% non_builtin_local_func/3 (user_default/shell_default). - -local_func(v, [{integer,_,V}], Bs, Shell, _RT, _Lf, _Ef) -> +local_func() -> [v,h,b,f,fl,rd,rf,rl,rp,rr,history,results,catch_exception]. +local_func(Func) -> + lists:member(Func, local_func()). +local_func(v, [{integer,_,V}], Bs, Shell, _RT, _FT, _Lf, _Ef) -> %% This command is validated and expanded prior. {_Ces,Value,_N} = shell_req(Shell, {get_cmd, V}), {value,Value,Bs}; -local_func(h, [], Bs, Shell, RT, _Lf, _Ef) -> +local_func(h, [], Bs, Shell, RT, _FT, _Lf, _Ef) -> Cs = shell_req(Shell, get_cmd), Cs1 = lists:filter(fun({{command, _},_}) -> true; - ({{result, _},_}) -> true; - (_) -> false - end, - Cs), + ({{result, _},_}) -> true; + (_) -> false + end, + Cs), Cs2 = lists:map(fun({{T, N}, V}) -> {{N, T}, V} end, - Cs1), + Cs1), Cs3 = lists:keysort(1, Cs2), {value,list_commands(Cs3, RT),Bs}; -local_func(b, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(b, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> {value,list_bindings(erl_eval:bindings(Bs), RT),Bs}; -local_func(f, [], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,ok,erl_eval:new_bindings()}; -local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,ok,erl_eval:del_binding(Name, Bs)}; -local_func(f, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,f,1}]); -local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(fl, [], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + {value, ets:tab2list(FT), Bs}; +local_func(fd, [{atom,_,FunName}, FunExpr], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + {value, Fun, []} = erl_eval:expr(FunExpr, []), + {arity, Arity} = erlang:fun_info(Fun, arity), + ets:insert(FT, [{{function, {shell_default, FunName, Arity}}, Fun}]), + {value, ok, Bs}; +local_func(fd, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, fd, 1}]); +local_func(ft, [{string, _, TypeDef}], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok, {attribute,_,spec,{{FunName, Arity},_}}=AttrForm} -> + ets:insert(FT, [{{function_type, {shell_default, FunName, Arity}}, AttrForm}]), + {value, ok, Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(ft, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, ft, 1}]); +local_func(td, [{string, _, TypeDef}], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok, {attribute,_,type,{TypeName, _, _}}=AttrForm} -> + ets:insert(FT, [{{type, TypeName}, AttrForm}]), + {value, ok, Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(td, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, td, 1}]); +local_func(rd, [{string, _, TypeDef}], Bs, _Shell, RT, _FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok,{attribute,_,_,_}=AttrForm} -> + [_] = add_records([AttrForm], Bs, RT), + {value,ok,Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(rd, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, rd, 1}]); +local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _FT, _Lf, _Ef) -> RecDef = expand_value(RecDef0), RDs = lists:flatten(erl_pp:expr(RecDef)), RecName = io_lib:write_atom_as_latin1(RecName0), @@ -998,99 +1161,105 @@ local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), exit(lists:flatten(ErrStr)) end; -local_func(rd, [_,_], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(rd, [_,_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,rd,2}]); -local_func(rf, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rf, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> true = ets:delete_all_objects(RT), {value,initiate_records(Bs, RT),Bs}; -local_func(rf, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rf, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[Recs],Bs} = expr_list([A], Bs0, Lf, Ef), if '_' =:= Recs -> true = ets:delete_all_objects(RT); - true -> + true -> lists:foreach(fun(Name) -> true = ets:delete(RT, Name) end, listify(Recs)) end, {value,ok,Bs}; -local_func(rl, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rl, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> {value,list_records(ets:tab2list(RT)),Bs}; -local_func(rl, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rl, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[Recs],Bs} = expr_list([A], Bs0, Lf, Ef), {value,list_records(record_defs(RT, listify(Recs))),Bs}; -local_func(rp, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rp, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[V],Bs} = expr_list([A], Bs0, Lf, Ef), Cs = pp(V, _Column=1, _Depth=-1, RT), io:requests([{put_chars, unicode, Cs}, nl]), {value,ok,Bs}; -local_func(rr, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File],Bs} = expr_list([A], Bs0, Lf, Ef), {value,read_and_add_records(File, '_', [], Bs, RT),Bs}; -local_func(rr, [_,_]=As0, Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [_,_]=As0, Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File,Sel],Bs} = expr_list(As0, Bs0, Lf, Ef), {value,read_and_add_records(File, Sel, [], Bs, RT),Bs}; -local_func(rr, [_,_,_]=As0, Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [_,_,_]=As0, Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File,Sel,Options],Bs} = expr_list(As0, Bs0, Lf, Ef), {value,read_and_add_records(File, Sel, Options, Bs, RT),Bs}; -local_func(history, [{integer,_,N}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(history, [{integer,_,N}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,history(N),Bs}; -local_func(history, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(history, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,history,1}]); -local_func(results, [{integer,_,N}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(results, [{integer,_,N}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,results(N),Bs}; -local_func(results, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(results, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,results,1}]); -local_func(catch_exception, [{atom,_,Bool}], Bs, _Shell, _RT, _Lf, _Ef) - when Bool; not Bool -> +local_func(catch_exception, [{atom,_,Bool}], Bs, _Shell, _RT, _FT, _Lf, _Ef) + when Bool; not Bool -> {value,catch_exception(Bool),Bs}; -local_func(catch_exception, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(catch_exception, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,catch_exception,1}]); -local_func(exit, [], _Bs, Shell, _RT, _Lf, _Ef) -> - shell_req(Shell, exit), %This terminates us +local_func(exit, [], _Bs, Shell, _RT, _FT, _Lf, _Ef) -> + shell_req(Shell, exit), %This terminates us exit(normal); -local_func(F, As0, Bs0, _Shell, _RT, Lf, Ef) when is_atom(F) -> +local_func(F, As0, Bs0, _Shell, _RT, FT, Lf, Ef) when is_atom(F) -> {As,Bs} = expr_list(As0, Bs0, Lf, Ef), - non_builtin_local_func(F,As,Bs). + non_builtin_local_func(F,As,Bs, FT). -non_builtin_local_func(F,As,Bs) -> +non_builtin_local_func(F,As,Bs, FT) -> Arity = length(As), case erlang:function_exported(user_default, F, Arity) of - true -> + true -> {eval,erlang:make_fun(user_default, F, Arity),As,Bs}; - false -> - shell_default(F,As,Bs) + false -> + shell_default(F,As,Bs, FT) end. -shell_default(F,As,Bs) -> +shell_default(F,As,Bs, FT) -> M = shell_default, A = length(As), case code:ensure_loaded(M) of - {module, _} -> - case erlang:function_exported(M,F,A) of - true -> - {eval,erlang:make_fun(M, F, A),As,Bs}; - false -> - shell_undef(F,A) - end; - {error, _} -> - shell_undef(F,A) + {module, _} -> + case erlang:function_exported(M,F,A) of + true -> + {eval,erlang:make_fun(M, F, A),As,Bs}; + false -> + shell_default_local_func(F,As, Bs, FT) + end; + {error, _} -> + shell_default_local_func(F,As, Bs, FT) + end. + +shell_default_local_func(F, As, Bs, FT) -> + case ets:lookup(FT, {function, {shell_default, F, length(As)}}) of + [] -> shell_undef(F, length(As)); + [{_, Fun}] -> {eval, Fun, As, Bs} end. shell_undef(F,A) -> erlang:error({shell_undef,F,A,[]}). -local_func_handler(Shell, RT, Ef) -> - H = fun(Lf) -> - fun(F, As, Bs) -> - local_func(F, As, Bs, Shell, RT, {eval,Lf(Lf)}, Ef) +local_func_handler(Shell, RT, FT, Ef) -> + H = fun(Lf) -> + fun(F, As, Bs) -> + local_func(F, As, Bs, Shell, RT, FT, {eval,Lf(Lf)}, Ef) end - end, + end, {eval,H(H)}. record_print_fun(RT) -> fun(Tag, NoFields) -> case ets:lookup(RT, Tag) of - [{_,{attribute,_,record,{Tag,Fields}}}] - when length(Fields) =:= NoFields -> + [{_,{attribute,_,record,{Tag,Fields}}}] + when length(Fields) =:= NoFields -> record_fields(Fields); _ -> no @@ -1109,7 +1278,7 @@ record_fields([]) -> initiate_records(Bs, RT) -> RNs1 = init_rec(shell_default, Bs, RT), RNs2 = case code:is_loaded(user_default) of - {file,_File} -> + {file,_File} -> init_rec(user_default, Bs, RT); false -> [] @@ -1145,11 +1314,12 @@ read_records(File, Selected, Options) -> RAs; RAs -> Sel = listify(Selected), - [RA || {attribute,_,_,{Name,_}}=RA <- RAs, + [RA || {attribute,_,_,{Name,_}}=RA <- RAs, lists:member(Name, Sel)] end. add_records(RAs, Bs0, RT) -> + %% TODO store File name to support type completion Recs = [{Name,D} || {attribute,_,_,{Name,_}}=D <- RAs], Bs1 = record_bindings(Recs, Bs0), case check_command([], Bs1) of @@ -1183,12 +1353,12 @@ expr_list(Es, Bs, Lf, Ef) -> record_bindings([], Bs) -> Bs; record_bindings(Recs0, Bs0) -> - {Recs1, _} = lists:mapfoldl(fun ({Name,Def}, I) -> {{Name,I,Def},I+1} + {Recs1, _} = lists:mapfoldl(fun ({Name,Def}, I) -> {{Name,I,Def},I+1} end, 0, Recs0), Recs2 = lists:keysort(2, lists:ukeysort(1, Recs1)), lists:foldl(fun ({Name,I,Def}, Bs) -> - erl_eval:add_binding({record,I,Name}, Def, Bs) - end, Bs0, Recs2). + erl_eval:add_binding({record,I,Name}, Def, Bs) + end, Bs0, Recs2). %%% Read record information from file(s) @@ -1214,7 +1384,7 @@ read_records(FileOrModule, Opts0) -> find_file(Mod) when is_atom(Mod) -> case code:which(Mod) of - File when is_list(File) -> + File when is_list(File) -> %% Special cases: %% - Modules not in the code path (loaded with code:load_abs/1): %% code:get_object_code/1 only searches in the code path @@ -1228,8 +1398,8 @@ find_file(Mod) when is_atom(Mod) -> error -> {error, nofile} end; - preloaded -> - {_M, Beam, File} = code:get_object_code(Mod), + preloaded -> + {_M, Beam, File} = code:get_object_code(Mod), {beam, Beam, File}; _Else -> % non_existing, interpreted, cover_compiled {error,nofile} @@ -1291,10 +1461,10 @@ try_sources([Src|Rest], Os) -> is_file(Name) -> case filelib:is_file(Name) of - true -> - not filelib:is_dir(Name); - false -> - false + true -> + not filelib:is_dir(Name); + false -> + false end. parse_file(File, Opts) -> @@ -1327,7 +1497,7 @@ record_attrs(Forms) -> shell_req(Shell, Req) -> Shell ! {shell_req,self(),Req}, receive - {shell_rep,Shell,Rep} -> Rep + {shell_rep,Shell,Rep} -> Rep end. list_commands([{{N,command},Es0}, {{N,result}, V} |Ds], RT) -> @@ -1337,7 +1507,7 @@ list_commands([{{N,command},Es0}, {{N,result}, V} |Ds], RT) -> I = iolist_size(Ns), io:requests([{put_chars, latin1, Ns}, {format,<<"~ts\n">>,[erl_pp:exprs(Es, I, enc())]}, - {format,<<"-> ">>,[]}, + {format,<<"-> ">>,[]}, {put_chars, unicode, VS}, nl]), list_commands(Ds, RT); @@ -1379,7 +1549,7 @@ list_bindings([], _RT) -> ok. list_records(Records) -> - lists:foreach(fun({_Name,Attr}) -> + lists:foreach(fun({_Name,Attr}) -> io:fwrite(<<"~ts">>, [erl_pp:attribute(Attr, enc())]) end, Records). @@ -1410,11 +1580,11 @@ prep_list_commands(E) -> substitute_v1(F, {value,_,_}=Value) -> F(Value); -substitute_v1(F, T) when is_tuple(T) -> +substitute_v1(F, T) when is_tuple(T) -> list_to_tuple(substitute_v1(F, tuple_to_list(T))); -substitute_v1(F, [E | Es]) -> +substitute_v1(F, [E | Es]) -> [substitute_v1(F, E) | substitute_v1(F, Es)]; -substitute_v1(_F, E) -> +substitute_v1(_F, E) -> E. a0() -> @@ -1464,8 +1634,8 @@ encoding() -> enc() -> case lists:keyfind(encoding, 1, io:getopts()) of - false -> [{encoding,latin1}]; % should never happen - Enc -> [Enc] + false -> [{encoding,latin1}]; % should never happen + Enc -> [Enc] end. garb(Shell) -> @@ -1476,32 +1646,32 @@ garb(Shell) -> get_env(V, Def) -> case application:get_env(stdlib, V) of - {ok, Val} when is_integer(Val), Val >= 0 -> - Val; - _ -> - Def + {ok, Val} when is_integer(Val), Val >= 0 -> + Val; + _ -> + Def end. - + check_env(V) -> case application:get_env(stdlib, V) of - undefined -> - ok; - {ok, Val} when is_integer(Val), Val >= 0 -> - ok; + undefined -> + ok; + {ok, Val} when is_integer(Val), Val >= 0 -> + ok; {ok, Val} -> Txt = io_lib:fwrite ("Invalid value of STDLIB configuration parameter" "~tw: ~tp\n", [V, Val]), - error_logger:info_report(lists:flatten(Txt)) + error_logger:info_report(lists:flatten(Txt)) end. - + set_env(App, Name, Val, Default) -> Prev = case application:get_env(App, Name) of - undefined -> - Default; - {ok, Old} -> - Old - end, + undefined -> + Default; + {ok, Old} -> + Old + end, application_controller:set_env(App, Name, Val), Prev. @@ -1521,7 +1691,7 @@ results(L) when is_integer(L), L >= 0 -> Bool :: boolean(). catch_exception(Bool) -> - set_env(stdlib, shell_catch_exception, Bool, ?DEF_CATCH_EXCEPTION). + set_env(stdlib, shell_catch_exception, Bool, ?DEF_CATCH_EXCEPTION). -spec prompt_func(PromptFunc) -> PromptFunc2 when PromptFunc :: 'default' | {module(),atom()}, diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl index 3d820c9131..118e9ff338 100644 --- a/lib/stdlib/src/shell_default.erl +++ b/lib/stdlib/src/shell_default.erl @@ -25,17 +25,17 @@ -export([help/0,lc/1,c/1,c/2,c/3,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1,lm/0,mm/0, memory/0,memory/1,uptime/0, - erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, + erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, y/1, y/2, - xm/1, bt/1, q/0, + xm/1, bt/1, q/0, h/1, h/2, h/3, ht/1, ht/2, ht/3, hcb/1, hcb/2, hcb/3, - ni/0, nregs/0]). + ni/0, nregs/0]). -export([ih/0,iv/0,im/0,ii/1,ii/2,iq/1,ini/1,ini/2,inq/1,ib/2,ib/3, - ir/2,ir/3,ibd/2,ibe/2,iba/3,ibc/3, - ic/0,ir/1,ir/0,il/0,ipb/0,ipb/1,iaa/1,iaa/2,ist/1,ia/1,ia/2,ia/3, - ia/4,ip/0]). - + ir/2,ir/3,ibd/2,ibe/2,iba/3,ibc/3, + ic/0,ir/1,ir/0,il/0,ipb/0,ipb/1,iaa/1,iaa/2,ist/1,ia/1,ia/2,ia/3, + ia/4,ip/0]). +-export(['$handle_undefined_function'/2]). -import(io, [format/1]). help() -> @@ -78,13 +78,13 @@ help() -> %% these are in alphabetic order it would be nice if they %% were to *stay* so! -bi(I) -> c:bi(I). -bt(Pid) -> c:bt(Pid). -c(File) -> c:c(File). +bi(I) -> c:bi(I). +bt(Pid) -> c:bt(Pid). +c(File) -> c:c(File). c(File, Opt) -> c:c(File, Opt). c(File, Opt, Filter) -> c:c(File, Opt, Filter). cd(D) -> c:cd(D). -erlangrc(X) -> c:erlangrc(X). +erlangrc(X) -> c:erlangrc(X). flush() -> c:flush(). h(M) -> c:h(M). h(M,F) -> c:h(M,F). @@ -95,25 +95,25 @@ ht(M,F,A) -> c:ht(M,F,A). hcb(M) -> c:hcb(M). hcb(M,F) -> c:hcb(M,F). hcb(M,F,A) -> c:hcb(M,F,A). -i() -> c:i(). -i(X,Y,Z) -> c:i(X,Y,Z). -l(Mod) -> c:l(Mod). -lc(X) -> c:lc(X). +i() -> c:i(). +i(X,Y,Z) -> c:i(X,Y,Z). +l(Mod) -> c:l(Mod). +lc(X) -> c:lc(X). ls() -> c:ls(). ls(S) -> c:ls(S). -m() -> c:m(). -m(Mod) -> c:m(Mod). +m() -> c:m(). +m(Mod) -> c:m(Mod). lm() -> c:lm(). mm() -> c:mm(). memory() -> c:memory(). memory(Type) -> c:memory(Type). -nc(X) -> c:nc(X). +nc(X) -> c:nc(X). ni() -> c:ni(). -nl(Mod) -> c:nl(Mod). +nl(Mod) -> c:nl(Mod). nregs() -> c:nregs(). -pid(X,Y,Z) -> c:pid(X,Y,Z). +pid(X,Y,Z) -> c:pid(X,Y,Z). pwd() -> c:pwd(). -q() -> c:q(). +q() -> c:q(). regs() -> c:regs(). uptime() -> c:uptime(). xm(Mod) -> c:xm(Mod). @@ -154,3 +154,11 @@ iv() -> calli(iv, []). calli(F, Args) -> c:appcall(debugger, i, F, Args). + +'$handle_undefined_function'(Func, Args) -> + case shell:get_function(Func, length(Args)) of + undefined -> + error_handler:raise_undef_exception(?MODULE, Func, Args); + Fun when is_function(Fun, length(Args)) -> + apply(Fun, Args) + end.
\ No newline at end of file diff --git a/lib/stdlib/src/shell_docs.erl b/lib/stdlib/src/shell_docs.erl index e42b5bb5b8..51e1272eb3 100644 --- a/lib/stdlib/src/shell_docs.erl +++ b/lib/stdlib/src/shell_docs.erl @@ -1005,18 +1005,18 @@ nl(Chars) -> init_ansi(#config{ ansi = undefined, io_opts = Opts }) -> %% We use this as our heuristic to see if we should print ansi or not case {application:get_env(kernel, shell_docs_ansi), + proplists:get_value(terminal, Opts, false), proplists:is_defined(echo, Opts) andalso - proplists:is_defined(expand_fun, Opts), - os:type()} of + proplists:is_defined(expand_fun, Opts)} of {{ok,false}, _, _} -> put(ansi, noansi); {{ok,true}, _, _} -> put(ansi, []); - {_, _, {win32,_}} -> - put(ansi, noansi); - {_, true,_} -> + {_, true, _} -> + put(ansi, []); + {_, _, true} -> put(ansi, []); - {_, false,_} -> + {_, _, false} -> put(ansi, noansi) end; init_ansi(#config{ ansi = true }) -> diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 358ebf471d..728f376be7 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -36,7 +36,9 @@ digraph, digraph_utils, edlin, + edlin_context, edlin_expand, + edlin_type_suggestion, epp, eval_bits, erl_abstract_code, @@ -112,6 +114,6 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-8.5.1","erts-13.1","crypto-4.5", + {runtime_dependencies, ["sasl-3.0","kernel-@OTP-17932@","erts-13.1","crypto-4.5", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index a418754caf..3c62dd9333 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -76,7 +76,9 @@ -import(lists,[member/2]). -compile({no_auto_import,[length/1]}). -compile({inline, [btoken/2, rev/1, append/2, stack/2, search_compile/1]}). --define(ASCII_LIST(CP1,CP2), CP1 < 256, CP2 < 256, CP1 =/= $\r). +-define(ASCII_LIST(CP1,CP2), + is_integer(CP1), 0 =< CP1, CP1 < 256, + is_integer(CP2), 0 =< CP2, CP2 < 256, CP1 =/= $\r). -export_type([grapheme_cluster/0]). @@ -198,7 +200,7 @@ slice(CD, N, Length) [] when is_binary(CD) -> <<>>; L -> slice_trail(L, Length) end; -slice(CD, N, infinity) -> +slice(CD, N, infinity) when is_integer(N), N >= 0 -> case slice_l0(CD, N) of [] when is_binary(CD) -> <<>>; Res -> Res @@ -261,11 +263,13 @@ trim(Str, Dir) -> Dir :: direction() | 'both', Characters :: [grapheme_cluster()]. trim(Str, _, []) -> Str; -trim(Str, leading, [Sep]) when is_list(Str), Sep < 256 -> +trim(Str, leading, [Sep]) + when is_list(Str), is_integer(Sep), 0 =< Sep, Sep < 256 -> trim_ls(Str, Sep); trim(Str, leading, Sep) when is_list(Sep) -> trim_l(Str, Sep); -trim(Str, trailing, [Sep]) when is_list(Str), Sep < 256 -> +trim(Str, trailing, [Sep]) + when is_list(Str), is_integer(Sep), 0 =< Sep, Sep < 256 -> trim_ts(Str, Sep); trim(Str, trailing, Seps0) when is_list(Seps0) -> Seps = search_pattern(Seps0), @@ -630,9 +634,10 @@ slice_l0(<<CP1/utf8, Bin/binary>>, N) when N > 0 -> slice_l0(L, N) -> slice_l(L, N). -slice_l([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> +slice_l([CP1|[CP2|_]=Cont], N) + when ?ASCII_LIST(CP1,CP2), is_integer(N), N > 0 -> slice_l(Cont, N-1); -slice_l(CD, N) when N > 0 -> +slice_l(CD, N) when is_integer(N), N > 0 -> case unicode_util:gc(CD) of [_|Cont] -> slice_l(Cont, N-1); [] -> []; @@ -641,7 +646,8 @@ slice_l(CD, N) when N > 0 -> slice_l(Cont, 0) -> Cont. -slice_lb(<<CP2/utf8, Bin/binary>>, CP1, N) when ?ASCII_LIST(CP1,CP2), N > 1 -> +slice_lb(<<CP2/utf8, Bin/binary>>, CP1, N) + when ?ASCII_LIST(CP1,CP2), is_integer(N), N > 1 -> slice_lb(Bin, CP2, N-1); slice_lb(Bin, CP1, N) -> [_|Rest] = unicode_util:gc([CP1|Bin]), @@ -693,9 +699,13 @@ slice_bin(CD, CP1, N) when N > 0 -> slice_bin(CD, CP1, 0) -> byte_size(CD)+byte_size(<<CP1/utf8>>). -uppercase_list([CP1|[CP2|_]=Cont], _Changed) when $a =< CP1, CP1 =< $z, CP2 < 256 -> +uppercase_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $a =< CP1, CP1 =< $z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1-32|uppercase_list(Cont, true)]; -uppercase_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +uppercase_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|uppercase_list(Cont, Changed)]; uppercase_list([], true) -> []; @@ -709,16 +719,16 @@ uppercase_list(CPs0, Changed) -> end. uppercase_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $a =< CP1, CP1 =< $z, CP2 < 256 -> + when is_integer(CP1), $a =< CP1, CP1 =< $z, CP2 < 256 -> [CP1-32|uppercase_bin(CP2, Bin, true)]; uppercase_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|uppercase_bin(CP2, Bin, Changed)]; uppercase_bin(CP1, Bin, Changed) -> case unicode_util:uppercase([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|uppercase_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -729,7 +739,7 @@ uppercase_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|uppercase_bin(Next, Rest, true)]; [] -> [Char]; @@ -738,9 +748,13 @@ uppercase_bin(CP1, Bin, Changed) -> end end. -lowercase_list([CP1|[CP2|_]=Cont], _Changed) when $A =< CP1, CP1 =< $Z, CP2 < 256 -> +lowercase_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $A =< CP1, CP1 =< $Z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1+32|lowercase_list(Cont, true)]; -lowercase_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +lowercase_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|lowercase_list(Cont, Changed)]; lowercase_list([], true) -> []; @@ -754,16 +768,16 @@ lowercase_list(CPs0, Changed) -> end. lowercase_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $A =< CP1, CP1 =< $Z, CP2 < 256 -> + when is_integer(CP1), $A =< CP1, CP1 =< $Z, CP2 < 256 -> [CP1+32|lowercase_bin(CP2, Bin, true)]; lowercase_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|lowercase_bin(CP2, Bin, Changed)]; lowercase_bin(CP1, Bin, Changed) -> case unicode_util:lowercase([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|lowercase_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -774,7 +788,7 @@ lowercase_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|lowercase_bin(Next, Rest, true)]; [] -> [Char]; @@ -783,9 +797,13 @@ lowercase_bin(CP1, Bin, Changed) -> end end. -casefold_list([CP1|[CP2|_]=Cont], _Changed) when $A =< CP1, CP1 =< $Z, CP2 < 256 -> +casefold_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $A =< CP1, CP1 =< $Z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1+32|casefold_list(Cont, true)]; -casefold_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +casefold_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|casefold_list(Cont, Changed)]; casefold_list([], true) -> []; @@ -799,16 +817,16 @@ casefold_list(CPs0, Changed) -> end. casefold_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $A =< CP1, CP1 =< $Z, CP2 < 256 -> + when is_integer(CP1), $A =< CP1, CP1 =< $Z, CP2 < 256 -> [CP1+32|casefold_bin(CP2, Bin, true)]; casefold_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|casefold_bin(CP2, Bin, Changed)]; casefold_bin(CP1, Bin, Changed) -> case unicode_util:casefold([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|casefold_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -819,7 +837,7 @@ casefold_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|casefold_bin(Next, Rest, true)]; [] -> [Char]; @@ -1734,7 +1752,7 @@ bin_search_str_2(Bin0, Start, Cont, First, SearchCPs) -> <<_:Start/binary, Bin/binary>> = Bin0, case binary:match(Bin, First) of nomatch -> {nomatch, byte_size(Bin0), Cont}; - {Where0, _} -> + {Where0, _} when is_integer(Where0) -> Where = Start+Where0, <<Keep:Where/binary, Cs0/binary>> = Bin0, [GC|Cs]=unicode_util:gc(Cs0), @@ -1979,7 +1997,7 @@ chars(C, N) -> chars(C, N, []). Tail :: string(), String :: string(). -chars(C, N, Tail) when N > 0 -> +chars(C, N, Tail) when is_integer(N), N > 0 -> chars(C, N-1, [C|Tail]); chars(C, 0, Tail) when is_integer(C) -> Tail. @@ -2109,7 +2127,7 @@ left(String, Len) when is_integer(Len) -> left(String, Len, $\s). Number :: non_neg_integer(), Character :: char(). -left(String, Len, Char) when is_integer(Char) -> +left(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, 1, Len); @@ -2134,7 +2152,7 @@ right(String, Len) when is_integer(Len) -> right(String, Len, $\s). Number :: non_neg_integer(), Character :: char(). -right(String, Len, Char) when is_integer(Char) -> +right(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, Slen-Len+1); @@ -2161,7 +2179,7 @@ centre(String, Len) when is_integer(Len) -> centre(String, Len, $\s). centre(String, 0, Char) when is_list(String), is_integer(Char) -> []; % Strange cases to centre string -centre(String, Len, Char) when is_integer(Char) -> +centre(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, (Slen-Len) div 2 + 1, Len); @@ -2186,7 +2204,8 @@ sub_string(String, Start) -> substr(String, Start). Start :: pos_integer(), Stop :: pos_integer(). -sub_string(String, Start, Stop) -> substr(String, Start, Stop - Start + 1). +sub_string(String, Start, Stop) when is_integer(Start), is_integer(Stop) -> + substr(String, Start, Stop - Start + 1). %% ISO/IEC 8859-1 (latin1) letters are converted, others are ignored %% diff --git a/lib/stdlib/src/timer.erl b/lib/stdlib/src/timer.erl index 182f5cb4f2..780ae4c7f1 100644 --- a/lib/stdlib/src/timer.erl +++ b/lib/stdlib/src/timer.erl @@ -22,7 +22,8 @@ -export([apply_after/4, send_after/3, send_after/2, exit_after/3, exit_after/2, kill_after/2, kill_after/1, - apply_interval/4, send_interval/3, send_interval/2, + apply_interval/4, apply_repeatedly/4, + send_interval/3, send_interval/2, cancel/1, sleep/1, tc/1, tc/2, tc/3, now_diff/2, seconds/1, minutes/1, hours/1, hms/3]). @@ -61,7 +62,7 @@ Reason :: term(). apply_after(0, M, F, A) when ?valid_mfa(M, F, A) -> - do_apply({M, F, A}), + _ = do_apply({M, F, A}, false), {ok, {instant, make_ref()}}; apply_after(Time, M, F, A) when ?valid_time(Time), @@ -160,6 +161,21 @@ apply_interval(Time, M, F, A) apply_interval(_Time, _M, _F, _A) -> {error, badarg}. +-spec apply_repeatedly(Time, Module, Function, Arguments) -> + {'ok', TRef} | {'error', Reason} + when Time :: time(), + Module :: module(), + Function :: atom(), + Arguments :: [term()], + TRef :: tref(), + Reason :: term(). +apply_repeatedly(Time, M, F, A) + when ?valid_time(Time), + ?valid_mfa(M, F, A) -> + req(apply_repeatedly, {system_time(), Time, self(), {M, F, A}}); +apply_repeatedly(_Time, _M, _F, _A) -> + {error, badarg}. + -spec send_interval(Time, Destination, Message) -> {'ok', TRef} | {'error', Reason} when Time :: time(), Destination :: pid() | (RegName :: atom()) | {RegName :: atom(), Node :: node()}, @@ -382,15 +398,11 @@ maybe_req(Req, Arg) -> handle_call({apply_once, {Started, Time, MFA}}, _From, Tab) -> Timeout = Started + Time, Reply = try - erlang:start_timer( - Timeout, - self(), - {apply_once, MFA}, - [{abs, true}] - ) + erlang:start_timer(Timeout, self(), {apply_once, MFA}, + [{abs, true}]) of SRef -> - ets:insert(Tab, {SRef, SRef}), + ets:insert(Tab, {SRef}), {ok, {once, SRef}} catch error:badarg -> @@ -399,25 +411,13 @@ handle_call({apply_once, {Started, Time, MFA}}, _From, Tab) -> {reply, Reply, Tab}; %% Start an interval timer. handle_call({apply_interval, {Started, Time, Pid, MFA}}, _From, Tab) -> - NextTimeout = Started + Time, - TRef = monitor(process, Pid), - Reply = try - erlang:start_timer( - NextTimeout, - self(), - {apply_interval, NextTimeout, Time, TRef, MFA}, - [{abs, true}] - ) - of - SRef -> - ets:insert(Tab, {TRef, SRef}), - {ok, {interval, TRef}} - catch - error:badarg -> - demonitor(TRef, [flush]), - {error, badarg} - end, - {reply, Reply, Tab}; + {TRef, TPid, Tag} = start_interval_loop(Started, Time, Pid, MFA, false), + ets:insert(Tab, {TRef, TPid, Tag}), + {reply, {ok, {interval, TRef}}, Tab}; +handle_call({apply_repeatedly, {Started, Time, Pid, MFA}}, _From, Tab) -> + {TRef, TPid, Tag} = start_interval_loop(Started, Time, Pid, MFA, true), + ets:insert(Tab, {TRef, TPid, Tag}), + {reply, {ok, {interval, TRef}}, Tab}; %% Cancel a one-shot timer. handle_call({cancel, {once, TRef}}, _From, Tab) -> _ = remove_timer(TRef, Tab), @@ -440,31 +440,14 @@ handle_call(_Req, _From, Tab) -> when Tab :: ets:tid(). %% One-shot timer timeout. handle_info({timeout, TRef, {apply_once, MFA}}, Tab) -> - case ets:take(Tab, TRef) of - [{TRef, _SRef}] -> - do_apply(MFA); - [] -> - ok - end, - {noreply, Tab}; -%% Interval timer timeout. -handle_info({timeout, _, {apply_interval, CurTimeout, Time, TRef, MFA}}, Tab) -> - case ets:member(Tab, TRef) of - true -> - NextTimeout = CurTimeout + Time, - SRef = erlang:start_timer( - NextTimeout, - self(), - {apply_interval, NextTimeout, Time, TRef, MFA}, - [{abs, true}] - ), - ets:update_element(Tab, TRef, {2, SRef}), - do_apply(MFA); - false -> - ok - end, + _ = case ets:take(Tab, TRef) of + [{TRef}] -> + do_apply(MFA, false); + [] -> + ok + end, {noreply, Tab}; -%% A process related to an interval timer died. +%% An interval timer loop process died. handle_info({'DOWN', TRef, process, _Pid, _Reason}, Tab) -> _ = remove_timer(TRef, Tab), {noreply, Tab}; @@ -480,34 +463,121 @@ handle_cast(_Req, Tab) -> {noreply, Tab}. -spec terminate(term(), _Tab) -> 'ok'. -terminate(_Reason, _Tab) -> - ok. +terminate(_Reason, undefined) -> + ok; +terminate(Reason, Tab) -> + _ = ets:foldl(fun + ({TRef}, Acc) -> + _ = cancel_timer(TRef), + Acc; + ({_TRef, TPid, Tag}, Acc) -> + TPid ! {cancel, Tag}, + Acc + end, + undefined, + Tab), + true = ets:delete(Tab), + terminate(Reason, undefined). -spec code_change(term(), State, term()) -> {'ok', State}. code_change(_OldVsn, Tab, _Extra) -> %% According to the man for gen server no timer can be set here. {ok, Tab}. +start_interval_loop(Started, Time, TargetPid, MFA, WaitComplete) -> + Tag = make_ref(), + TimeServerPid = self(), + {TPid, TRef} = spawn_monitor(fun() -> + TimeServerRef = monitor(process, TimeServerPid), + TargetRef = monitor(process, TargetPid), + TimerRef = schedule_interval_timer(Started, Time, + MFA), + _ = interval_loop(TimeServerRef, TargetRef, Tag, + WaitComplete, TimerRef) + end), + {TRef, TPid, Tag}. + +%% Interval timer loop. +interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef0) -> + receive + {cancel, Tag} -> + ok = cancel_timer(TimerRef0); + {'DOWN', TimerServerMon, process, _, _} -> + ok = cancel_timer(TimerRef0); + {'DOWN', TargetMon, process, _, _} -> + ok = cancel_timer(TimerRef0); + {timeout, TimerRef0, {apply_interval, CurTimeout, Time, MFA}} -> + case do_apply(MFA, WaitComplete) of + {ok, {spawn, ActionMon}} -> + receive + {cancel, Tag} -> + ok; + {'DOWN', TimerServerMon, process, _, _} -> + ok; + {'DOWN', TargetMon, process, _, _} -> + ok; + {'DOWN', ActionMon, process, _, _} -> + TimerRef1 = schedule_interval_timer(CurTimeout, Time, MFA), + interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef1) + end; + _ -> + TimerRef1 = schedule_interval_timer(CurTimeout, Time, MFA), + interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef1) + end + end. + +schedule_interval_timer(CurTimeout, Time, MFA) -> + NextTimeout = CurTimeout + Time, + case NextTimeout =< system_time() of + true -> + TimerRef = make_ref(), + self() ! {timeout, TimerRef, {apply_interval, NextTimeout, Time, MFA}}, + TimerRef; + false -> + erlang:start_timer(NextTimeout, self(), {apply_interval, NextTimeout, Time, MFA}, [{abs, true}]) + end. + %% Remove a timer. remove_timer(TRef, Tab) -> case ets:take(Tab, TRef) of - [{TRef, SRef}] -> - ok = erlang:cancel_timer(SRef, [{async, true}, {info, false}]), + [{TRef}] -> % One-shot timer. + ok = cancel_timer(TRef), + true; + [{TRef, TPid, Tag}] -> % Interval timer. + TPid ! {cancel, Tag}, true; [] -> % TimerReference does not exist, do nothing false end. +%% Cancel a timer. +cancel_timer(TRef) -> + erlang:cancel_timer(TRef, [{async, true}, {info, false}]). + %% Help functions %% If send op. send directly (faster than spawn) -do_apply({?MODULE, send, A}) -> - catch send(A); +do_apply({?MODULE, send, A}, _) -> + try send(A) + of _ -> {ok, send} + catch _:_ -> error + end; %% If exit op. resolve registered name -do_apply({erlang, exit, [Name, Reason]}) -> - catch exit(get_pid(Name), Reason); -do_apply({M,F,A}) -> - catch spawn(M, F, A). +do_apply({erlang, exit, [Name, Reason]}, _) -> + try exit(get_pid(Name), Reason) + of _ -> {ok, exit} + catch _:_ -> error + end; +do_apply({M,F,A}, false) -> + try spawn(M, F, A) + of _ -> {ok, spawn} + catch _:_ -> error + end; +do_apply({M, F, A}, true) -> + try spawn_monitor(M, F, A) + of {_, Ref} -> {ok, {spawn, Ref}} + catch _:_ -> error + end. %% Get current time in milliseconds, %% ceil'ed to the next millisecond. diff --git a/lib/stdlib/src/unicode.erl b/lib/stdlib/src/unicode.erl index 20c92a1a4a..72532e02d6 100644 --- a/lib/stdlib/src/unicode.erl +++ b/lib/stdlib/src/unicode.erl @@ -91,9 +91,9 @@ characters_to_binary(_, _) -> -spec characters_to_list(Data, InEncoding) -> Result when Data :: latin1_chardata() | chardata() | external_chardata(), InEncoding :: encoding(), - Result :: list() - | {error, list(), RestData} - | {incomplete, list(), binary()}, + Result ::string() + | {error, string(), RestData} + | {incomplete, string(), binary()}, RestData :: latin1_chardata() | chardata() | external_chardata(). characters_to_list(_, _) -> @@ -103,9 +103,9 @@ characters_to_list(_, _) -> -spec characters_to_list(Data) -> Result when Data :: latin1_chardata() | chardata() | external_chardata(), - Result :: list() - | {error, list(), RestData} - | {incomplete, list(), binary()}, + Result :: string() + | {error, string(), RestData} + | {incomplete, string(), binary()}, RestData :: latin1_chardata() | chardata() | external_chardata(). characters_to_list(ML) -> diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index c59398a84e..0809dbb492 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -35,7 +35,7 @@ %% zip server -export([zip_open/1, zip_open/2, - zip_get/1, zip_get/2, + zip_get/1, zip_get/2, zip_get_crc32/2, zip_t/1, zip_tt/1, zip_list_dir/1, zip_list_dir/2, zip_close/1]). @@ -267,6 +267,13 @@ do_openzip_get(#openzip{files = Files, in = In0, input = Input, do_openzip_get(_) -> throw(einval). +%% retrieve the crc32 checksum from an open archive +openzip_get_crc32(FileName, #openzip{files = Files}) -> + case file_name_search(FileName, Files) of + {_,#zip_file_extra{crc32=CRC}} -> {ok, CRC}; + _ -> throw(file_not_found) + end. + %% retrieve a file from an open archive openzip_get(FileName, OpenZip) -> case ?CATCH(do_openzip_get(FileName, OpenZip)) of @@ -1165,6 +1172,9 @@ server_loop(Parent, OpenZip) -> {From, {get, FileName}} -> From ! {self(), openzip_get(FileName, OpenZip)}, server_loop(Parent, OpenZip); + {From, {get_crc32, FileName}} -> + From ! {self(), openzip_get_crc32(FileName, OpenZip)}, + server_loop(Parent, OpenZip); {From, list_dir} -> From ! {self(), openzip_list_dir(OpenZip)}, server_loop(Parent, OpenZip); @@ -1223,6 +1233,15 @@ zip_close(Pid) when is_pid(Pid) -> zip_get(FileName, Pid) when is_pid(Pid) -> request(self(), Pid, {get, FileName}). +-spec(zip_get_crc32(FileName, ZipHandle) -> {ok, CRC} | {error, Reason} when + FileName :: file:name(), + ZipHandle :: handle(), + CRC :: non_neg_integer(), + Reason :: term()). + +zip_get_crc32(FileName, Pid) when is_pid(Pid) -> + request(self(), Pid, {get_crc32, FileName}). + -spec(zip_list_dir(ZipHandle) -> {ok, Result} | {error, Reason} when Result :: [zip_comment() | zip_file()], ZipHandle :: handle(), diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index b8e4d89996..a12b2213ec 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -23,6 +23,7 @@ MODULES= \ dummy_h \ dummy_via \ edlin_expand_SUITE \ + edlin_context_SUITE \ epp_SUITE \ erl_anno_SUITE \ erl_eval_SUITE \ @@ -49,6 +50,7 @@ MODULES= \ io_SUITE \ io_proto_SUITE \ lists_SUITE \ + lists_property_test_SUITE \ log_mf_h_SUITE \ math_SUITE \ ms_transform_SUITE \ @@ -104,10 +106,12 @@ MODULES= \ ERTS_MODULES= erts_test_utils SASL_MODULES= otp_vsns +KERNEL_MODULES= rtnode ERL_FILES= $(MODULES:%=%.erl) \ $(ERTS_MODULES:%=$(ERL_TOP)/erts/emulator/test/%.erl) \ - $(SASL_MODULES:%=$(ERL_TOP)/lib/sasl/test/%.erl) + $(SASL_MODULES:%=$(ERL_TOP)/lib/sasl/test/%.erl) \ + $(KERNEL_MODULES:%=$(ERL_TOP)/lib/kernel/test/%.erl) EXTRA_FILES= $(ERL_TOP)/otp_versions.table @@ -135,7 +139,7 @@ COVERFILE=stdlib.cover make_emakefile: $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) \ - $(MODULES) $(ERTS_MODULES) $(SASL_MODULES) \ + $(MODULES) $(ERTS_MODULES) $(SASL_MODULES) $(KERNEL_MODULES) \ > $(EMAKEFILE) tests $(TYPES): make_emakefile @@ -161,6 +165,7 @@ release_tests_spec: make_emakefile $(ERL_FILES) $(COVERFILE) $(EXTRA_FILES) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) + $(INSTALL_DIR) "$(RELSYSDIR)/stdlib_SUITE_data" $(INSTALL_DATA) $(ERL_TOP)/make/otp_version_tickets "$(RELSYSDIR)/stdlib_SUITE_data" release_docs_spec: diff --git a/lib/stdlib/test/base64_SUITE.erl b/lib/stdlib/test/base64_SUITE.erl index 1fc4c3fc0e..d1625108bd 100644 --- a/lib/stdlib/test/base64_SUITE.erl +++ b/lib/stdlib/test/base64_SUITE.erl @@ -26,9 +26,11 @@ -export([all/0, suite/0, groups/0, group/1]). %% Test cases must be exported. --export([base64_encode/1, base64_decode/1, base64_otp_5635/1, - base64_otp_6279/1, big/1, illegal/1, mime_decode/1, - mime_decode_to_string/1, +-export([base64_encode/1, base64_encode_modes/1, + base64_decode/1, base64_decode_modes/1, + base64_otp_5635/1, base64_otp_6279/1, big/1, illegal/1, + mime_decode/1, mime_decode_modes/1, + mime_decode_to_string/1, mime_decode_to_string_modes/1, roundtrip_1/1, roundtrip_2/1, roundtrip_3/1, roundtrip_4/1]). %%------------------------------------------------------------------------- @@ -40,8 +42,11 @@ suite() -> {timetrap,{minutes,4}}]. all() -> - [base64_encode, base64_decode, base64_otp_5635, - base64_otp_6279, big, illegal, mime_decode, mime_decode_to_string, + [base64_encode, base64_encode_modes, + base64_decode, base64_decode_modes, + base64_otp_5635, base64_otp_6279, big, illegal, + mime_decode, mime_decode_modes, + mime_decode_to_string, mime_decode_to_string_modes, {group, roundtrip}]. groups() -> @@ -67,6 +72,20 @@ base64_encode(Config) when is_list(Config) -> "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" = base64:encode_to_string(<<"0123456789!@#0^&*();:<>,. []{}">>), ok. + +%%------------------------------------------------------------------------- +%% Test base64:encode/2. +base64_encode_modes(Config) when is_list(Config) -> + Data = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + + <<"F+o/o++B/a+r">> = base64:encode(Data, standard), + <<"F-o_o--B_a-r">> = base64:encode(Data, urlsafe), + + "F+o/o++B/a+r" = base64:encode_to_string(Data, standard), + "F-o_o--B_a-r" = base64:encode_to_string(Data, urlsafe), + + ok. + %%------------------------------------------------------------------------- %% Test base64:decode/1. base64_decode(Config) when is_list(Config) -> @@ -91,6 +110,24 @@ base64_decode(Config) when is_list(Config) -> base64:decode_to_string( <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9">>), ok. + +%%------------------------------------------------------------------------- +%% Test base64:decode/2. +base64_decode_modes(Config) when is_list(Config) -> + DataBin = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + DataStr = [23, 234, 63, 163, 239, 129, 253, 175, 171], + + DataBin = base64:decode("F+o/o++B/a+r", standard), + DataBin = base64:decode("F-o_o--B_a-r", urlsafe), + {'EXIT', _} = catch base64:decode("F-o_o--B_a-r", standard), + {'EXIT', _} = catch base64:decode("F+o/o++B/a+r", urlsafe), + + DataStr = base64:decode_to_string("F+o/o++B/a+r", standard), + DataStr = base64:decode_to_string("F-o_o--B_a-r", urlsafe), + {'EXIT', _} = catch base64:decode_to_string("F-o_o--B_a-r", standard), + {'EXIT', _} = catch base64:decode_to_string("F+o/o++B/a+r", urlsafe), + + ok. %%------------------------------------------------------------------------- %% OTP-5635: Some data doesn't pass through base64:decode/1 correctly. base64_otp_5635(Config) when is_list(Config) -> @@ -171,6 +208,31 @@ mime_decode(Config) when is_list(Config) -> <<"o">> = MimeDecode(<<"bw=\000=">>), ok. +%% Test base64:mime_decode/2. +mime_decode_modes(Config) when is_list(Config) -> + MimeDecode = fun (In, Mode) -> + Out = base64:mime_decode(In, Mode), + Out = base64:mime_decode(binary_to_list(In), Mode) + end, + + %% The following all decode to the same data. + Data = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + Data = MimeDecode(<<"F+o/o++B/a+r">>, standard), + Data = MimeDecode(<<"F-o_o--B_a-r">>, urlsafe), + + %% The following decodes to different data depending on mode. + Base64 = <<"AB+C+D/E/FG-H-I_J_KL">>, + %% In standard mode, "-" and "_" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "AB+C+D/E/FGHIJKL". + <<0, 31, 130, 248, 63, 196, 252, 81, 135, 32, 146, 139>> = + MimeDecode(Base64, standard), + %% In urlsafe mode, "+" and "/" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "ABCDEFG-H-I_J_KL". + <<0, 16, 131, 16, 81, 190, 31, 226, 63, 39, 242, 139>> = + MimeDecode(Base64, urlsafe), + + ok. + %%------------------------------------------------------------------------- %% Repeat of mime_decode() tests @@ -221,6 +283,32 @@ mime_decode_to_string(Config) when is_list(Config) -> "o" = MimeDecodeToString(<<"bw=\000=">>), ok. + +%% Test base64:mime_decode_to_string/2. +mime_decode_to_string_modes(Config) when is_list(Config) -> + MimeDecode = fun (In, Mode) -> + Out = base64:mime_decode_to_string(In, Mode), + Out = base64:mime_decode_to_string(binary_to_list(In), Mode) + end, + + %% The following all decode to the same data. + Data = [23, 234, 63, 163, 239, 129, 253, 175, 171], + Data = MimeDecode(<<"F+o/o++B/a+r">>, standard), + Data = MimeDecode(<<"F-o_o--B_a-r">>, urlsafe), + + %% The following decodes to different data depending on mode. + Base64 = <<"AB+C+D/E/FG-H-I_J_KL">>, + %% In standard mode, "-" and "_" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "AB+C+D/E/FGHIJKL". + [0, 31, 130, 248, 63, 196, 252, 81, 135, 32, 146, 139] = + MimeDecode(Base64, standard), + %% In urlsafe mode, "+" and "/" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "ABCDEFG-H-I_J_KL". + [0, 16, 131, 16, 81, 190, 31, 226, 63, 39, 242, 139] = + MimeDecode(Base64, urlsafe), + + ok. + %%------------------------------------------------------------------------- roundtrip_1(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/base64_property_test_SUITE.erl b/lib/stdlib/test/base64_property_test_SUITE.erl index 7717b62767..68ac5e9ee7 100644 --- a/lib/stdlib/test/base64_property_test_SUITE.erl +++ b/lib/stdlib/test/base64_property_test_SUITE.erl @@ -24,18 +24,18 @@ all() -> [ - encode_case, - encode_to_string_case, - decode_case, - decode_malformed_case, - decode_noisy_case, - decode_to_string_case, - decode_to_string_malformed_case, - decode_to_string_noisy_case, - mime_decode_case, - mime_decode_malformed_case, - mime_decode_to_string_case, - mime_decode_to_string_malformed_case + encode_1_case, encode_2_case, + encode_to_string_1_case, encode_to_string_2_case, + decode_1_case, decode_2_case, + decode_1_malformed_case, decode_2_malformed_case, + decode_1_noisy_case, decode_2_noisy_case, + decode_to_string_1_case, decode_to_string_2_case, + decode_to_string_1_malformed_case, decode_to_string_2_malformed_case, + decode_to_string_1_noisy_case, decode_to_string_2_noisy_case, + mime_decode_1_case, mime_decode_2_case, + mime_decode_1_malformed_case, mime_decode_2_malformed_case, + mime_decode_to_string_1_case, mime_decode_to_string_2_case, + mime_decode_to_string_1_malformed_case, mime_decode_to_string_2_malformed_case ]. init_per_suite(Config) -> @@ -44,41 +44,77 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. -encode_case(Config) -> - do_proptest(prop_encode, Config). +encode_1_case(Config) -> + do_proptest(prop_encode_1, Config). -encode_to_string_case(Config) -> - do_proptest(prop_encode_to_string, Config). +encode_2_case(Config) -> + do_proptest(prop_encode_2, Config). -decode_case(Config) -> - do_proptest(prop_decode, Config). +encode_to_string_1_case(Config) -> + do_proptest(prop_encode_to_string_1, Config). -decode_malformed_case(Config) -> - do_proptest(prop_decode_malformed, Config). +encode_to_string_2_case(Config) -> + do_proptest(prop_encode_to_string_2, Config). -decode_noisy_case(Config) -> - do_proptest(prop_decode_noisy, Config). +decode_1_case(Config) -> + do_proptest(prop_decode_1, Config). -decode_to_string_case(Config) -> - do_proptest(prop_decode_to_string, Config). +decode_2_case(Config) -> + do_proptest(prop_decode_2, Config). -decode_to_string_malformed_case(Config) -> - do_proptest(prop_decode_to_string_malformed, Config). +decode_1_malformed_case(Config) -> + do_proptest(prop_decode_1_malformed, Config). -decode_to_string_noisy_case(Config) -> - do_proptest(prop_decode_to_string_noisy, Config). +decode_2_malformed_case(Config) -> + do_proptest(prop_decode_2_malformed, Config). -mime_decode_case(Config) -> - do_proptest(prop_mime_decode, Config). +decode_1_noisy_case(Config) -> + do_proptest(prop_decode_1_noisy, Config). -mime_decode_malformed_case(Config) -> - do_proptest(prop_mime_decode_malformed, Config). +decode_2_noisy_case(Config) -> + do_proptest(prop_decode_2_noisy, Config). -mime_decode_to_string_case(Config) -> - do_proptest(prop_mime_decode_to_string, Config). +decode_to_string_1_case(Config) -> + do_proptest(prop_decode_to_string_1, Config). -mime_decode_to_string_malformed_case(Config) -> - do_proptest(prop_mime_decode_to_string_malformed, Config). +decode_to_string_2_case(Config) -> + do_proptest(prop_decode_to_string_2, Config). + +decode_to_string_1_malformed_case(Config) -> + do_proptest(prop_decode_to_string_1_malformed, Config). + +decode_to_string_2_malformed_case(Config) -> + do_proptest(prop_decode_to_string_2_malformed, Config). + +decode_to_string_1_noisy_case(Config) -> + do_proptest(prop_decode_to_string_1_noisy, Config). + +decode_to_string_2_noisy_case(Config) -> + do_proptest(prop_decode_to_string_2_noisy, Config). + +mime_decode_1_case(Config) -> + do_proptest(prop_mime_decode_1, Config). + +mime_decode_2_case(Config) -> + do_proptest(prop_mime_decode_2, Config). + +mime_decode_1_malformed_case(Config) -> + do_proptest(prop_mime_decode_1_malformed, Config). + +mime_decode_2_malformed_case(Config) -> + do_proptest(prop_mime_decode_2_malformed, Config). + +mime_decode_to_string_1_case(Config) -> + do_proptest(prop_mime_decode_to_string_1, Config). + +mime_decode_to_string_2_case(Config) -> + do_proptest(prop_mime_decode_to_string_2, Config). + +mime_decode_to_string_1_malformed_case(Config) -> + do_proptest(prop_mime_decode_to_string_1_malformed, Config). + +mime_decode_to_string_2_malformed_case(Config) -> + do_proptest(prop_mime_decode_to_string_2_malformed, Config). do_proptest(Prop, Config) -> ct_property_test:quickcheck( diff --git a/lib/stdlib/test/dets_SUITE.erl b/lib/stdlib/test/dets_SUITE.erl index c50a2d5baf..8fa877f5a5 100644 --- a/lib/stdlib/test/dets_SUITE.erl +++ b/lib/stdlib/test/dets_SUITE.erl @@ -46,7 +46,7 @@ otp_5487/1, otp_6206/1, otp_6359/1, otp_4738/1, otp_7146/1, otp_8070/1, otp_8856/1, otp_8898/1, otp_8899/1, otp_8903/1, otp_8923/1, otp_9282/1, otp_11245/1, otp_11709/1, otp_13229/1, - otp_13260/1, otp_13830/1]). + otp_13260/1, otp_13830/1, receive_optimisation/1]). -export([dets_dirty_loop/0]). @@ -94,7 +94,7 @@ all() -> insert_new, repair_continuation, otp_5487, otp_6206, otp_6359, otp_4738, otp_7146, otp_8070, otp_8856, otp_8898, otp_8899, otp_8903, otp_8923, otp_9282, otp_11245, otp_11709, - otp_13229, otp_13260, otp_13830 + otp_13229, otp_13260, otp_13830, receive_optimisation ]. groups() -> @@ -3492,6 +3492,34 @@ otp_13830(Config) -> {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, default}]), ok = dets:close(Tab). +receive_optimisation(Config) -> + Tab = dets_receive_optimisation_test, + FName = filename(Tab, Config), + + % Spam message box + lists:foreach(fun(_) -> self() ! {spam, it} end, lists:seq(1, 1_000_000)), + + {ok, _} = dets:open_file(Tab,[{file, FName}]), + ok = dets:insert(Tab,{one, record}), + + StartTime = os:system_time(millisecond), + + % We expect one thousand of simple lookups to finish in one second + Lookups = 1000, + Timeout = 1000, + Loop = fun Loop(N) when N =< 0 -> ok; + Loop(N) -> + Now = os:system_time(millisecond), + (Now - StartTime > Timeout) andalso throw({timeout_after, Lookups - N}), + [{one, record}] = dets:lookup(Tab, one), + Loop(N-1) + end, + + ok = Loop(Lookups), + + ok = dets:close(Tab), + ok = file:delete(FName). + %% %% Parts common to several test cases %% diff --git a/lib/stdlib/test/edlin_context_SUITE.erl b/lib/stdlib/test/edlin_context_SUITE.erl new file mode 100644 index 0000000000..a9685d98e2 --- /dev/null +++ b/lib/stdlib/test/edlin_context_SUITE.erl @@ -0,0 +1,189 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_context_SUITE). +-include_lib("common_test/include/ct.hrl"). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1]). + +-export([get_context/1]). + +suite() -> + [{timetrap,{minutes,1}}]. +all() -> + [get_context]. +groups() -> + []. +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +get_context(_Config) -> + {term, [], {atom, "h"}} = edlin_context:get_context(lists:reverse("h")), + {term} = edlin_context:get_context(lists:reverse("h(file")), + {term} = edlin_context:get_context(lists:reverse("h(file,open")), + {term, [{call, "h(file,open)"}], {atom, "h"}} = edlin_context:get_context(lists:reverse("h(file,open), h")), + {term} = edlin_context:get_context(lists:reverse("h(file,open), h(file")), + {term} = edlin_context:get_context(lists:reverse("h(file,open), h(file,open")), + {term, [{call, "h(file,open)"}], {call, "h(file,open)"}} = edlin_context:get_context(lists:reverse("h(file,open), h(file,open)")), + {function} = edlin_context:get_context(lists:reverse("file:")), + {function} = edlin_context:get_context(lists:reverse("file:open")), + {function, "file", "open", [], [], []} = edlin_context:get_context(lists:reverse("file:open(")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"/")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"Word")), + {function, "file", "open", [], {string, "\"\""}, []} = edlin_context:get_context(lists:reverse("file:open(\"\"")), + {function, "file", "open", [{string, "\"\""}], [], []} = edlin_context:get_context(lists:reverse("file:open(\"\",")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[")), + {function, "file", "open", [{string, "\"\""}], [], [{tuple, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",{")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[atom")), + {function, "file", "open", [{string, "\"\""}], [], [{tuple, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",{atom")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{atom")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [{atom, "atom"}], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{atom,")), + {function, "file", "open", [{string, "\"\""}], [], [{map, ["atom"], "atom", [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",#{ atom =>")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("#{list")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("{list")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("[list")), + {map, "M", []} = edlin_context:get_context(lists:reverse("M#{")), + {map, "M", []} = edlin_context:get_context(lists:reverse("M#{key")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key:=")), %% map value + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>0")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>0,")), + {map, "M", ["key", "key2"]} = edlin_context:get_context(lists:reverse("M#{key=>0,key2=>")), + {map_or_record} = edlin_context:get_context(lists:reverse("#")), + {record, "record", [], [], [], [], []} = edlin_context:get_context(lists:reverse("#record{")), + {record, "record", [], [], [], [], []} = edlin_context:get_context(lists:reverse("#record.")), + {record, "record", [], [], [], {atom, "field"}, []} = edlin_context:get_context(lists:reverse("#record{field")), + {record, "record", ["field"], "field", [], [], []} = edlin_context:get_context(lists:reverse("#record{field=>")), + {record, "record", ["field"], "field", [], [], []} = edlin_context:get_context(lists:reverse("#record{field:=")), %% record_field value + {record, "record", ["field"], [], [{integer, "0"}], [], []} = edlin_context:get_context(lists:reverse("R#record{field=>0,")), + {record, "record", ["field", "field2"], "field2", [{integer,"0"}], [], [{list, [], []},{tuple, [{atom, "atom"}], []}]} = edlin_context:get_context(lists:reverse("R#record{field=>0,field2=>[{atom,")), + {term,[],{atom,"fun"}} = edlin_context:get_context(lists:reverse("fun")), + {term,[],{atom,"fun"}} = edlin_context:get_context(lists:reverse("fun ")), + {fun_} = edlin_context:get_context(lists:reverse("fun m")), + {fun_, "m"} = edlin_context:get_context(lists:reverse("fun m:")), + {fun_, "m"} = edlin_context:get_context(lists:reverse("fun m:f")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/1")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/1 ")), + {term,[{fun_,"fun m:f/1"}],[]} = edlin_context:get_context(lists:reverse("fun m:f/1 ,")), + {function,"user_defined","my_fun", + [{keyword,"receive X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(receive X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"maybe X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(maybe X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"try a end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(try a end, ")), + {function,"user_defined","my_fun", + [{keyword,"catch X -> X end"}], + [],[]}= edlin_context:get_context(lists:reverse("my_fun(catch X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"try a catch _:_ -> b end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(try a catch _:_ -> b end, ")), + {function,"user_defined","my_fun", + [{keyword,"begin X end"}], + [],[]}= edlin_context:get_context(lists:reverse("my_fun(begin X end, ")), + {function,"user_defined","my_fun", + [{keyword,"if X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(if X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"case X of _ -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(case X of _ -> X end, ")), + {binding} = edlin_context:get_context(lists:reverse("fun() -> X")), + {term,[],{atom,"x"}} = edlin_context:get_context(lists:reverse("fun() -> x")), + {term} = edlin_context:get_context(lists:reverse("fun() ->")), + {macro} = edlin_context:get_context(lists:reverse("?")), %% not supported in edlin_expand + % unknown map + {term,[{operation,"one = a"},{operation,"two = b"}],[]} = edlin_context:get_context(lists:reverse("#{ one = a, two = b, ")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("#{ one := a, two := b, ")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("#{ one => a, two => b, ")), + {term,[{operation,"A = a"},{operation,"B = b"}],[]} = edlin_context:get_context(lists:reverse("A = a, B = b, ")), + {term,[{operation,"one = a"}],[]} = edlin_context:get_context(lists:reverse("#{ one = a, two = ")), + {term,[{atom,"a"}],[]} = edlin_context:get_context(lists:reverse("#{ one := a, two := ")), + {term,[{atom,"a"}],[]} = edlin_context:get_context(lists:reverse("#{ one => a, two => ")), + {term,[{operation,"A = a"}],[]} = edlin_context:get_context(lists:reverse("A = a, B = ")), + {term,[],{operation,"A = a"}} = edlin_context:get_context(lists:reverse("A = a")), + {'end'} = edlin_context:get_context(lists:reverse("a.")), + {record,"record",[],[],[],[],[]} = edlin_context:get_context(lists:reverse("#record.")), + {record,"record",[],[],[],[],[]} = edlin_context:get_context(lists:reverse("{#record.")), + {record,"record",[],[],[],{atom,"a"},[]} = edlin_context:get_context(lists:reverse("#record.a")), + {record,"record",[],[],[],{atom,"a"},[]} = edlin_context:get_context(lists:reverse("{#record.a")), + {term,[],{record,"#record{}"}} = edlin_context:get_context(lists:reverse("#record{}")), + {term,[],{map,"#{ a => b}"}} = edlin_context:get_context(lists:reverse("#{ a => b}")), + {term,[{atom,"a"}],{atom,"tuple"}} = edlin_context:get_context(lists:reverse("{a, tuple")), + {term,[],{tuple,"{a, tuple}"}} = edlin_context:get_context(lists:reverse("{a, tuple}")), + {term,[],{call,"lists:my_fun()"}} = edlin_context:get_context(lists:reverse("lists:my_fun()")), + {term} = edlin_context:get_context(lists:reverse("(")), + {term,[],{parenthesis,"()"}} = edlin_context:get_context(lists:reverse("()")), + {new_fun,"()"} = edlin_context:get_context(lists:reverse("fun()")), + %% 256 $] + {term,[],{list,"[]"}} = edlin_context:get_context(lists:reverse("[]")), + {term,[{atom,"a"}],{atom,"b"}} = edlin_context:get_context(lists:reverse("fun() when a, b")), + {term,[{atom,"a"}],{atom,"b"}} = edlin_context:get_context(lists:reverse("fun() -> a, b")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("fun() -> a, b, ")), + {term, [], {pid, "<1.0.1>"}} = edlin_context:get_context(lists:reverse("<1.0.1>")), + {term, [], {funref, "#Fun<erl_eval.0.1>"}} = edlin_context:get_context(lists:reverse("#Fun<erl_eval.0.1>")), + {term, [], {ref, "#Ref<1.0.1>"}} = edlin_context:get_context(lists:reverse("#Ref<1.0.1>")), + %{term, [], {port, "#Port<1.0>"}} = edlin_context:get_context(lists:reverse("#Port<1.0>")), + {term, [], {binary, "<<0>>"}} = edlin_context:get_context(lists:reverse("<<0>>")), + {term,[],{keyword,"fun (X) -> X end"}} = edlin_context:get_context(lists:reverse("fun (X) -> X end")), + {term,[],{keyword,"fun(X) -> X end"}} = edlin_context:get_context(lists:reverse("fun(X) -> X end")), %% should be fun_ too + {term,[],{keyword,"receive X -> X end"}} = edlin_context:get_context(lists:reverse("receive X -> X end")), + {error, _} = edlin_context:get_context(lists:reverse("no_keyword -> X end")), + {term} = edlin_context:get_context(lists:reverse("@")), %% No valid argument 305 + {term,[],{char,"$@"}} = edlin_context:get_context(lists:reverse("$@")), + {term,[],{char,"$ "}} = edlin_context:get_context(lists:reverse("$ ")), + {term,[],{float,"1.0"}} = edlin_context:get_context(lists:reverse("1.0")), + {term,[],{integer,"10#10"}} = edlin_context:get_context(lists:reverse("10#10")), + {term,[],{integer,"1"}} = edlin_context:get_context(lists:reverse("1")), + {binding} = edlin_context:get_context(lists:reverse("{X")), + {term,[{var, "X"}], []} = edlin_context:get_context(lists:reverse("{X, ")), + {error,_} = edlin_context:get_context(lists:reverse("<abc)")), + {error,_} = edlin_context:get_context(lists:reverse("<abc]")), + {error,_} = edlin_context:get_context(lists:reverse("<abc}")), + {term} = edlin_context:get_context(lists:reverse("(abc>")), + {term} = edlin_context:get_context(lists:reverse("\"\\\"\"")), %% odd quotes "\"" + {error, _} = edlin_context:get_context(lists:reverse("{\"\", $\"}")), %% odd quotes + {term} = edlin_context:get_context(lists:reverse("receive X -> ")), + %% read operator and argument order validity + {error,_} = edlin_context:get_context(lists:reverse("foo bar")), %% TODO what should be returned here, illegal + {term,[],{operation,"\" \" \" \""}} = edlin_context:get_context(lists:reverse("\" \" \" \"")), %% " " " " + {term,[],{operation,"1 + 2"}} = edlin_context:get_context(lists:reverse("1+2")), + {term,[],{operation,"1 andalso 2"}} = edlin_context:get_context(lists:reverse("1 andalso 2")), + {term,[],{operation,"1 and 2"}} = edlin_context:get_context(lists:reverse("1 and 2")), + {term,[],{operation,"1 orelse 2"}} = edlin_context:get_context(lists:reverse("1 orelse 2")), + {term,[],{operation,"1 or 2"}} = edlin_context:get_context(lists:reverse("1 or 2")), + {term,[],{operation,"1 or 2"}} = edlin_context:get_context(lists:reverse("1 or 2")), + {term,[],{operation,"1 =/= 2"}} = edlin_context:get_context(lists:reverse("1 =/= 2")), + {term,[],{operation,"1 =:= 2"}} = edlin_context:get_context(lists:reverse("1 =:= 2")), + {term,[],{operation,"1 <=> 2"}} = edlin_context:get_context(lists:reverse("1 <=> 2")), + {term,[],{operation,"<<1>> > <<2>>"}} = edlin_context:get_context(lists:reverse("<<1>>><<2>>")), + %{term,[],{operation,"<<1>> > <<2>>"}} = edlin_context:get_context(lists:reverse("<<1>> > <<2>>")), + {error,_} = edlin_context:get_context(lists:reverse("1 + + 2")), + {term,[],{integer,"2"}} = edlin_context:get_context(lists:reverse("1 -> 2")), + {term,[],{integer,"2"}} = edlin_context:get_context(lists:reverse("receive X -> 2")), + {term} = edlin_context:get_context(lists:reverse("receive X ->")), + {term,[{integer,"2"}],{operation,"1 + 3"}} = edlin_context:get_context(lists:reverse("receive X -> 2, 1+3")), + {term,[],{integer,"-1"}} = edlin_context:get_context(lists:reverse("-1")), + {term,[],{float,"-1.2"}} = edlin_context:get_context(lists:reverse("-1.2")), + ok.
\ No newline at end of file diff --git a/lib/stdlib/test/edlin_expand_SUITE.erl b/lib/stdlib/test/edlin_expand_SUITE.erl index 22b6ba03af..6ff6daf6f9 100644 --- a/lib/stdlib/test/edlin_expand_SUITE.erl +++ b/lib/stdlib/test/edlin_expand_SUITE.erl @@ -18,12 +18,22 @@ %% %CopyrightEnd% %% -module(edlin_expand_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_testcase/2, end_per_testcase/2, - init_per_group/2,end_per_group/2]). --export([normal/1, quoted_fun/1, quoted_module/1, quoted_both/1, erl_1152/1, - erl_352/1, unicode/1]). - +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, + init_per_group/2,end_per_group/2]). +-export([normal/1, type_completion/1, quoted_fun/1, quoted_module/1, quoted_both/1, erl_1152/1, get_coverage/1, + check_trailing/1, unicode/1, filename_completion/1, binding_completion/1, record_completion/1, + map_completion/1, function_parameter_completion/1, fun_completion/1]). +-record(a_record, + {a_field :: atom1 | atom2 | btom | 'my atom' | {atom3, {atom4, non_neg_integer()}} | 'undefined', + b_field :: boolean() | 'undefined', + c_field :: list(term()) | 'undefined', + d_field :: non_neg_integer() | 'undefined'}). +-record('Quoted_record', + {'A_field' :: atom1 | atom2 | btom | 'my atom' | {atom3, {atom4, non_neg_integer()}} | 'undefined', + b_field :: boolean() | 'undefined', + c_field :: list(term()) | 'undefined', + d_field :: non_neg_integer() | 'undefined'}). -include_lib("common_test/include/ct.hrl"). init_per_testcase(_Case, Config) -> @@ -37,17 +47,20 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,1}}]. -all() -> - [normal, quoted_fun, quoted_module, quoted_both, erl_1152, erl_352, +all() -> + [normal, filename_completion, binding_completion, get_coverage, type_completion, + record_completion, fun_completion, map_completion, function_parameter_completion, + quoted_fun, quoted_module, quoted_both, erl_1152, check_trailing, unicode]. -groups() -> +groups() -> []. init_per_suite(Config) -> Config. end_per_suite(_Config) -> + cleanup(), ok. init_per_group(_GroupName, Config) -> @@ -60,53 +73,466 @@ cleanup() -> [try code:purge(M), code:delete(M) - catch _:_ -> ok end || M <- [expand_test, expand_test1, - 'ExpandTestCaps', 'ExpandTestCaps2']]. + catch _:_ -> ok end || M <- [expand_test, expand_test1, expand_function_parameter, + 'ExpandTestCaps', 'ExpandTestCaps1', + complete_function_parameter]]. normal(Config) when is_list(Config) -> {module,expand_test} = compile_and_load(Config,expand_test), %% These tests might fail if another module with the prefix %% "expand_" happens to also be loaded. - {yes, "test:", []} = do_expand("expand_"), + {yes,"test:",[]} = do_expand("expand_"), {no, [], []} = do_expand("expandXX_"), - {no,[], - [{"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"expand0arity_entirely",0}, - {"module_info",0}, - {"module_info",1}]} = do_expand("expand_test:"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("expand_test:a_"), + {no,[],[#{ + title:="functions", + elems:=[{"a_fun_name",[{ending,"("}]}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"expand0arity_entirely",_}, + {"module_info",_}] + }]} = do_expand("expand_test:"), + {yes,[],[#{title:="functions", + elems:=[{"a_fun_name",_},{"a_less_fun_name",_}]}]} = do_expand("expand_test:a_"), {yes,"arity_entirely()",[]} = do_expand("expand_test:expand0"), ok. +to_atom(Str) -> + case erl_scan:string(Str) of + {ok, [{atom,_,A}], _} -> + {ok, A}; + _ -> + error + end. + +type_completion(_Config) -> + ct:timetrap({minutes, 20}), + {Time,_} = timer:tc(fun() -> do_expand("erl_pp:expr(") end), + case Time of + Time when Time > 2600000 -> {skip, "Expansion too slow on this machine"}; + _ -> + parallelforeach( + fun(Mod) -> + Exports = edlin_expand:get_exports(Mod), + [try + Str = io_lib:write_atom(Mod) ++ ":" ++ atom_to_list(Func) ++ "(", + do_expand(Str) + catch E:R:ST -> + erlang:raise(E, {R, Mod, Func}, ST) + end || {Func, _}<- Exports] + end, [list_to_atom(M) || {M,_,_} <- code:all_available()]), + ok + end. + +parallelforeach(Fun, List) -> + case parallelforeach(Fun, List, #{}, erlang:system_info(schedulers_online)) of + [] -> ok; + Else -> ct:fail(Else) + end. +parallelforeach(_Fun, [], Workers, _MaxWorkers) when map_size(Workers) =:= 0 -> + []; +parallelforeach(Fun, List, Workers, MaxWorkers) when MaxWorkers =:= map_size(Workers); + List =:= [] -> + receive + {'DOWN', Ref, _, _, normal} -> + parallelforeach(Fun, List, maps:remove(Ref, Workers), MaxWorkers); + + {'DOWN', Ref, _, _, Reason} -> + {Arg, NewWorkers} = maps:take(Ref, Workers), + [{Arg, Reason} | parallelforeach(Fun, List, NewWorkers, MaxWorkers)] + end; +parallelforeach(Fun, [H|T], Workers, MaxWorkers) -> + {_Pid, Ref} = spawn_monitor(fun() -> Fun(H) end), + parallelforeach(Fun, T, Workers#{ Ref => H }, MaxWorkers). + +filename_completion(Config) -> + DataDir = proplists:get_value(data_dir, Config), + {ok, CWD} = file:get_cwd(), + file:set_cwd(DataDir), + {yes,"e\"",[]} = do_expand("\"visible\\ fil"), + {no,[],[]} = do_expand("\"visible fil"), + {no,[],[]} = do_expand("\" "), + {no,[],[]} = do_expand("\"\""), + {yes, "e\"", []} = do_expand("\".hidden\\ fil"), + {yes,"/", []} = do_expand("\".."), + {yes,"ta/", _} = do_expand("\"../edlin_expand_SUITE_da"), + {yes,"erl\"",[]} = do_expand("\"complete_function_parameter."), + R = case {os:type(), file:native_name_encoding()} of + {{win32,_}, _} -> {skip, "Unicode on filenames in windows are tricky"}; + {_,latin1} -> {skip, "Cannot interpret unicode filenames when native_name_encoding is latin1"}; + _ -> + {yes,"isible", + [{"visible file",_},{"visible_file",_},{"visible😀_file",_}]} = do_expand("\"v"), + {yes,"e\"",[]} = do_expand("\"visible😀_fil"), + {yes,[], + [{"../",[]}, + {".hidden file",_}, + {".hidden_file",_}, + {".hidden😀_file",_}]} = do_expand("\"."), + ok + end, + file:set_cwd(CWD), + R. + +record_completion(_Config) -> + %% test record completion for loaded records + %% test record field name completion + %% test record field completion + {yes,"ord{", []} = do_expand("#a_rec"), + {yes,"uoted_record'{", []} = do_expand("#'Q"), + {no, [], [#{title:="fields", elems:=[{"a_field",_}, {"b_field",_}, {"c_field",_}, {"d_field",_}]}]} = do_expand("#a_record{"), + {no, [], [#{title:="fields", elems:=[{"a_field",_}, {"b_field",_}, {"c_field",_}, {"d_field",_}]}]} = do_expand("#a_record."), + {yes,"eld=", []} = do_expand("#a_record{a_fi"), + {no,[],[#{title:="types",elems:= + [{"atom1",[]}, + {"atom2",[]}, + {"btom",[]}, + {"'my atom'",[]}, + {"{atom3, ...}",[]}], + options:=[{hide,title}]}]} = do_expand("#a_record{a_field="), + %% test that an already specified field does not get suggested again + {no,[], + [#{title:="fields", elems:= + [{"a_field",[{ending,"="}]}, + {"b_field",[{ending,"="}]}, + {"c_field",[{ending,"="}]}, + {"d_field",[{ending,"="}]}], + options:=[highlight_all]}]} = do_expand("#a_record{a_field={atom3,b},"), + %% test match argument and closing parenthesis completion + {yes,", ",[]} = do_expand("#a_record{a_field={atom3"), + {no,[],[#{title:="types",elems:=[{"{atom4, ...}",[]}],options:=[{hide,title}]}]} = do_expand("#a_record{a_field={atom3,"), + {no,[],[#{title:="types",elems:=[{"integer() >= 0",[]}],options:=[{hide,title}]}]} = do_expand("#a_record{a_field={atom3,{atom4, "), + {yes,"}",_} = do_expand("#a_record{a_field={atom3,{atom4, 1"), + {yes,"}",_} = do_expand("#a_record{a_field={atom3,{atom4, 1}"), + ok. + +fun_completion(_Config) -> + {yes, "/1", []} = do_expand("fun lists:unzip3"), + {no, [], []} = do_expand("fun lists:unzip3/1,"), + {no, [], []} = do_expand("lists:unzip3/1"), + {no, [], [{"2",_},{"3",_}]} = do_expand("lists:seq/"), + %{yes, ", ", _} = do_expand("lists:all(fun erlang:is_atom/1"), + {no, [], [#{}]} = do_expand("lists:all(fun erlang:is_atom/1, "), + ok. + +binding_completion(_Config) -> + %% test that bindings in the shell can be completed + {yes,"ding",[]} = do_expand("Bin"), + {yes,"ding",[]} = do_expand("file:open(Bin"), + {yes,"ding",[]} = do_expand("fun (X, Y) -> Bin"), + %% test unicode + {yes,"öndag", []} = do_expand("S"), + {yes,"", []} = do_expand("Ö"), + ok. + +map_completion(_Config) -> + %% test that key suggestion works for a known map in bindings + {no,[],[{"a_key",[{ending, "=>"}]},{"b_key",_},{"c_key",_}]} = do_expand("MapBinding#{"), + {yes, "_key=>", []} = do_expand("MapBinding#{b"), + {yes, "_key=>", []} = do_expand("MapBinding # { b"), + %% test that an already specified key does not get suggested again + {no, [], [{"a_key",_},{"c_key", _}]} = do_expand("MapBinding#{b_key=>1,"), + %% test that unicode works + ok. + +function_parameter_completion(Config) -> + %% test first and second parameter + %% test multiple arities with same type on first parameter + %% test multiple arities with different type on first parameter + %% test that recursive types does not trigger endless loop + %% test that getting type of out of bound parameter does not trigger crash + compile_and_load2(Config,complete_function_parameter), + {no, [], [#{elems:=[#{title:="complete_function_parameter:an_untyped_fun/2", elems:=[]}]}]} = do_expand("complete_function_parameter:an_untyped_fun("), + {yes,":",[]} = do_expand("complete_function_parameter:an_untyped_fun(complete_function_parameter"), + {no, [], [#{elems:=[#{elems:=[#{title:="types",elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:a_fun_name("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:a_fun_name(1,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter : a_fun_name ( 1 , "), + {yes, ")", []} = do_expand("complete_function_parameter:a_fun_name(1,2"), + {no, [], []} = do_expand("complete_function_parameter:a_fun_name(1,2,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"any()",[]},{"[any() | [Deeplist]]",[]}]}]}]}]} = do_expand("complete_function_parameter:a_deeplist_fun("), + {no,[],[#{title:="typespecs", + elems:=[#{title:= + "complete_function_parameter:multi_arity_fun(T1)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,1}]}, + #{title:= + "complete_function_parameter:multi_arity_fun(T1, T2)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,1}]}], + options:=[highlight_all]}]} = do_expand("complete_function_parameter:multi_arity_fun("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"true",[]},{"false",[]}]}]}]}]} = do_expand("complete_function_parameter:multi_arity_fun(1,"), + {no,[], + [#{elems := + [#{elems := + [#{elems := [{"integer()",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(T1)"}, + #{elems := + [#{elems := [{"true",[]},{"false",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(B1, T1)"}], + options := [highlight_all], + title := "typespecs"}]} = do_expand("complete_function_parameter:different_multi_arity_fun("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:different_multi_arity_fun(false,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"{atom1, ...}",[]}, + {"atom1",[]}, + {"atom2",[]}, + {"[atom4 | atom5]",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom1",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter({"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"{integer() >= 0, ...}",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter({atom1,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom4",[]},{"atom5",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter(["), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom4",[]},{"atom5",[]}]}]}]}]} = do_expand("complete_function_parameter : advanced_nested_parameter ( [ , "), + {no,[], + [#{elems := + [#{elems := + [#{elems := [{"integer()",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(T1)"}, + #{elems := + [#{elems := [{"true",[]},{"false",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(B1, T1)"}], + options := [highlight_all], + title := "typespecs"}]} = do_expand("complete_function_parameter:different_multi_arity_fun("), + %% Hide results where the type of the first parameters does not match the prototype header + {no,[], + [#{title:="typespecs", + elems:=[#{title:="complete_function_parameter:different_multi_arity_fun(B1, T1)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,2}]}], + options:=[highlight_all]}]} = do_expand("complete_function_parameter:different_multi_arity_fun(false,"), + {no, _, []} = do_expand("complete_function_parameter:different_multi_arity_fun(atom,"), + {yes, _, _} = do_expand("complete_function_parameter:'emoji"), + + ok. + +get_coverage(Config) -> + compile_and_load2(Config,complete_function_parameter), + do_expand("\""), + do_expand("\"."), + do_expand("\"../"), + do_expand("\"/"), + do_expand("fun m:f"), + do_expand("fun m:f/"), + do_expand("#"), + do_expand("MapBinding#{"), + do_expand("B"), + do_expand("#a_record{"), + do_expand("#a_record{a_field=>"), + + %% match_arguments and is_type tests + do_expand("complete_function_parameter:map_parameter_function(#{"), + do_expand("complete_function_parameter:map_parameter_function(#{a"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>err"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>error"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>error}"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,b=>2,c=>3,d=>error"), + do_expand("complete_function_parameter:map_parameter_function(#{}, "), + do_expand("complete_function_parameter:map_parameter_function(#{V=>1}, "), + do_expand("complete_function_parameter:map_parameter_function(#{a=>V}, "), + do_expand("complete_function_parameter:tuple_parameter_function({a,b}, "), + do_expand("complete_function_parameter:tuple_parameter_function({a,V}, "), + do_expand("complete_function_parameter:list_parameter_function([], "), + do_expand("complete_function_parameter:list_parameter_function([atom], "), + true = {no, [], []} =/= do_expand("complete_function_parameter:list_parameter_function([V], "), + do_expand("complete_function_parameter:non_empty_list_parameter_function([atom], "), + do_expand("complete_function_parameter:non_empty_list_parameter_function([], "), + do_expand("complete_function_parameter:binary_parameter_function(<<0>>, "), + do_expand("complete_function_parameter:binary_parameter_function(<<V>>, "), + do_expand("complete_function_parameter:integer_parameter_function(0, "), + do_expand("complete_function_parameter:non_neg_integer_parameter_function(1, "), + do_expand("complete_function_parameter:neg_integer_parameter_function(-1, "), + do_expand("complete_function_parameter:float_parameter_function(0.1, "), + do_expand("complete_function_parameter:pid_parameter_function(<0.1.0>, "), + do_expand("complete_function_parameter:port_parameter_function(#Port<0.1>, "), + do_expand("complete_function_parameter:record_parameter_function(#a_record{a = 1}, "), + do_expand("complete_function_parameter:function_parameter_function(#Fun<erl_eval.1.0>, "), + do_expand("complete_function_parameter:function_parameter_function(fun(X) -> X end, "), %% Todo verify fun arity + do_expand("complete_function_parameter:function_parameter_function(receive X -> X end, "), + do_expand("complete_function_parameter:function_parameter_function(V, "), + do_expand("complete_function_parameter:function_parameter_function((1+2), "), + do_expand("complete_function_parameter:function_parameter_function(some_call(), "), + do_expand("complete_function_parameter:function_parameter_function(#Nope<1.0>, "), + do_expand("complete_function_parameter:reference_parameter_function(#Ref<1.0.1.0>, "), + do_expand("complete_function_parameter:any_parameter_function(#Ref<1.0.1.0>, "), + do_expand("complete_function_parameter:ann_type_parameter_function(atom, "), + true = {no, [], []} =/= do_expand("complete_function_parameter:ann_type_parameter_function2(1, "), + do_expand("complete_function_parameter:atom_parameter_function(atom, "), + do_expand("complete_function_parameter:ann_type_parameter_function(atom"), + do_expand("complete_function_parameter:atom_parameter_function(atom"), + %% user_defined function + {yes, "func(", _} = + do_expand("my_"), + {no,[],[#{elems := + [#{elems := + [#{elems := [{"#my_record",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := "shell_default:my_func(A)"}], + options := [highlight_all], + title := "typespecs"}]} = + do_expand("my_func("), + {yes,"=",[]} = + do_expand("my_func(#my_record{ field"), + {no,[], + [#{elems := + [#{elems := [{"a_value",[]},{"b_value",[]}], + options := [{separator," :: "},{highlight_all}], + title := "erlang:my_type()"}], + options := [{hide,title}], + title := "types"}]} = + do_expand("my_func(#my_record{ field=>"), + {yes,"ue, ",[]} = + do_expand("my_func(#my_record{field=>a_val"), + %% bifs() + {yes, "st(", _} = + do_expand("integer_to_li"), + %% commands() + {yes, "(", _} = + do_expand("bt"), + {yes, ":", _}=edlin_expand:expand(lists:reverse("complete_function_parameter")), + {no, [], _}=edlin_expand:expand(lists:reverse("#")), + {no, [], _}=edlin_expand:expand(lists:reverse("UnbindedMap#")), + {no, [], _}=edlin_expand:expand(lists:reverse("UnbindedMap#{")), + {no, [], _}=edlin_expand:expand(lists:reverse("#{")), + {yes, "(", _}=edlin_expand:expand(lists:reverse("complete_function_parameter:a_fun_name")), + + do_expand("fun l"), + %{yes, ">", _} = + do_expand("fun () -"), + %{yes, "n ", _} = + do_expand("fun () whe "), + %{no, [], []} = + do_expand("fun () when "), + %{no, [], []} = + do_expand("fun () -> "), + %% {keyword, ...} + do_expand("M#"), + do_expand("#non_existant_record"), + do_expand("#a_record{ non_existand_field"), + + + %% match_arguments coverage + do_expand("complete_function_parameter:integer_parameter_function(atom,"), %% match_argument -> false + do_expand("complete_function_parameter:a_zero_arity_fun()"), %% match_argument, parameters empty + do_expand("erlang:system_info(thread"), + do_expand("erlang:system_info(thread_nope"), + do_expand("erlang:system_info(threads"), + do_expand("erlang:process_flag(priori"), + do_expand("erlang:process_flag(priority_nope"), + do_expand("erlang:process_flag(priority, "), + do_expand("erlang:process_flag(priority, atom"), + do_expand("erlang:process_flag(priority, 1"), + do_expand("erlang:system_info({allocator,"), + do_expand("lists:seq(1"), + do_expand("lists:seq(1, 10"), + do_expand("ssh:connect({"), + do_expand("ssh:connect({255,"), + do_expand("ssh:connect({'$i"), + do_expand("ssh:connect({'$x"), + do_expand("ssh:connect({1000,"), + do_expand("ssh:connect({255,255,255,255,255,255,255,255"), + do_expand("ssh:connect({255,255,255,255}, ["), + do_expand("ssh:connect(receive V -> V end, "), + do_expand("ssh:connect((1+2), "), + do_expand("ssh:connect(hej(), "), + do_expand("ssh:connect(fun() -> ok end, "), + do_expand("ssh:connect(fun a:b/1, "), + do_expand("ssh:connect(V, [in"), + do_expand("ssh:connect(V, [inet"), + do_expand("ssh:connect(V, [inet, "), + do_expand("atom_to_list("), + do_expand("help("), + do_expand("h(lists"), + do_expand("ht(lists"), + do_expand("h(file,"), + do_expand("ht(file,"), + do_expand("ht(file,mode"), + do_expand("file:get_cwd()"), + do_expand("fl"), + do_expand("fl("), + do_expand("fl()"), + do_expand("MapBinding#"), + do_expand("RecordBinding#"), + do_expand("TupleBinding#"), + do_expand("Binding#"), + do_expand("MyVar"), + {_, _, M0} = do_expand("ssh:connect("), + do_format(M0), + {_, _, M1} = do_expand("ssh:connect({"), + do_format(M1), + {_, _, M6} = do_expand("ssh:connect({},["), + do_format(M6), + lists:flatten(edlin_expand:format_matches(M6, 20)), + {_, _, M2} = do_expand("e"), + do_format(M2), + {_, _, M3} = do_expand("erlang:i"), + do_format(M3), + {_, _, M4} = do_expand("complete_function_parameter:an_untyped_fun("), + do_format(M4), + lists:flatten(edlin_expand:format_matches(M4, 20)), + {_,_,M5}=edlin_expand:expand("e"), + do_format(M5), + {_,_,M7}=edlin_expand:expand("erlang:"), + do_format(M7), + {_,_,M8}=edlin_expand:expand("e"), + do_format(M8), + lists:flatten(edlin_expand:format_matches(M8, 20)), + {_,_,M9}=edlin_expand:expand("complete_function_parameter:an_untyped_fun("), + lists:flatten(edlin_expand:format_matches(M9, 20)), + do_format(M5), + {_, _, M10} = edlin_expand:expand("ssh:connect({},["), + do_format(M10), + lists:flatten(edlin_expand:format_matches(M10, 20)), + ok. + %% Normal module name, some function names using quoted atoms. quoted_fun(Config) when is_list(Config) -> {module,expand_test} = compile_and_load(Config,expand_test), {module,expand_test1} = compile_and_load(Config,expand_test1), %% should be no colon after test this time - {yes, "test", []} = do_expand("expand_"), + {yes, "test", [#{title:="modules", elems:=[{"expand_test",[{ending, ":"}]},{"expand_test1",_}]}]} = do_expand("expand_"), {no, [], []} = do_expand("expandXX_"), - {no,[],[{"'#weird-fun-name'",1}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}, - {"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("expand_test1:"), - {yes,"_",[]} = do_expand("expand_test1:a"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("expand_test1:a_"), - {yes,[], - [{"'#weird-fun-name'",1}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("expand_test1:'"), - {yes,"uoted_fun_",[]} = do_expand("expand_test1:'Q"), - {yes,[], - [{"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("expand_test1:'Quoted_fun_"), + {no,[],[#{title:="functions", + elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}, + {"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("expand_test1:"), + {yes,"_",[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("expand_test1:a"), + {yes,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("expand_test1:a_"), + {yes,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'"), + {yes,"uoted_fun_",[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'Q"), + {yes,[],[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'Quoted_fun_"), {yes,"weird-fun-name'(",[]} = do_expand("expand_test1:'#"), %% Since there is a module_info/1 as well as a module_info/0 @@ -116,146 +542,127 @@ quoted_fun(Config) when is_list(Config) -> quoted_module(Config) when is_list(Config) -> {module,'ExpandTestCaps'} = compile_and_load(Config,'ExpandTestCaps'), - {yes, "Caps':", []} = do_expand("'ExpandTest"), - {no,[], - [{"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("'ExpandTestCaps':"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("'ExpandTestCaps':a_"), + {yes, "Caps':",[]} = do_expand("'ExpandTest"), + {no,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("'ExpandTestCaps':"), + {yes,[],[#{title:="functions", elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps':a_"), ok. quoted_both(Config) when is_list(Config) -> {module,'ExpandTestCaps'} = compile_and_load(Config,'ExpandTestCaps'), {module,'ExpandTestCaps1'} = compile_and_load(Config,'ExpandTestCaps1'), %% should be no colon (or quote) after test this time - {yes, "Caps", []} = do_expand("'ExpandTest"), - {no,[],[{"'#weird-fun-name'",0}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}, - {"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("'ExpandTestCaps1':"), - {yes,"_",[]} = do_expand("'ExpandTestCaps1':a"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("'ExpandTestCaps1':a_"), - {yes,[], - [{"'#weird-fun-name'",0}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("'ExpandTestCaps1':'"), - {yes,"uoted_fun_",[]} = do_expand("'ExpandTestCaps1':'Q"), - {yes,[], - [{"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("'ExpandTestCaps1':'Quoted_fun_"), + {yes, "Caps", [#{elems:=[{"'ExpandTestCaps'",[{ending, ":"}]},{"'ExpandTestCaps1'",_}]}]} = do_expand("'ExpandTest"), + {no,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}, + {"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("'ExpandTestCaps1':"), + {yes,"_",[#{elems:=[{"a_fun_name",_},{"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps1':a"), + {yes,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps1':a_"), + {yes,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'"), + {yes,"uoted_fun_",[#{elems:=[{"'Quoted_fun_name'",_},{"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'Q"), + {yes,[],[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'Quoted_fun_"), {yes,"weird-fun-name'()",[]} = do_expand("'ExpandTestCaps1':'#"), ok. %% Note: pull request #1152. erl_1152(Config) when is_list(Config) -> - "\n"++"foo"++" "++[1089]++_ = do_format(["foo",[1089]]), + "foo"++" "++[1089]++_ = do_format(["foo",[1089]]), ok. -erl_352(Config) when is_list(Config) -> - erl_352_test(3, 3), - - erl_352_test(3, 75), - erl_352_test(3, 76, [trailing]), - erl_352_test(4, 74), - erl_352_test(4, 75, [leading]), - erl_352_test(4, 76, [leading, trailing]), - - erl_352_test(75, 3), - erl_352_test(76, 3, [leading]), - erl_352_test(74, 4), - erl_352_test(75, 4, [leading]), - erl_352_test(76, 4, [leading]), - - erl_352_test(74, 74, [leading]), - erl_352_test(74, 75, [leading]), - erl_352_test(74, 76, [leading, trailing]). - -erl_352_test(PrefixLen, SuffixLen) -> - erl_352_test(PrefixLen, SuffixLen, []). - -erl_352_test(PrefixLen, SuffixLen, Dots) -> - io:format("\nPrefixLen = ~w, SuffixLen = ~w\n", [PrefixLen, SuffixLen]), - - PrefixM = lists:duplicate(PrefixLen, $p), - SuffixM = lists:duplicate(SuffixLen, $s), - LM = [PrefixM ++ S ++ SuffixM || S <- ["1", "2"]], - StrM = do_format(LM), - check_leading(StrM, "", PrefixM, SuffixM, Dots), - - PrefixF = lists:duplicate(PrefixLen, $p), - SuffixF = lists:duplicate(SuffixLen-2, $s), - LF = [{PrefixF ++ S ++ SuffixF, 1} || S <- ["1", "2"]], - StrF = do_format(LF), - true = check_leading(StrF, "/1", PrefixF, SuffixF, Dots), - +check_trailing(Config) when is_list(Config) -> + Str = lists:duplicate(80, $1), + StrF = do_format([Str]), + {_, "...\n"} = lists:split(76, StrF), ok. -check_leading(FormStr, ArityStr, Prefix, Suffix, Dots) -> - List = string:tokens(FormStr, "\n "), - io:format("~p\n", [List]), - true = lists:all(fun(L) -> length(L) < 80 end, List), - case lists:member(leading, Dots) of - true -> - true = lists:all(fun(L) -> - {"...", Rest} = lists:split(3, L), - check_trailing(Rest, ArityStr, - Suffix, Dots) - end, List); - false -> - true = lists:all(fun(L) -> - {Prefix, Rest} = - lists:split(length(Prefix), L), - check_trailing(Rest, ArityStr, - Suffix, Dots) - end, List) - end. - -check_trailing([I|Str], ArityStr, Suffix, Dots) -> - true = lists:member(I, [$1, $2]), - case lists:member(trailing, Dots) of - true -> - {Rest, "..." ++ ArityStr} = - lists:split(length(Str) - (3 + length(ArityStr)), Str), - true = lists:prefix(Rest, Suffix); - false -> - {Rest, ArityStr} = - lists:split(length(Str) - length(ArityStr), Str), - Rest =:= Suffix - end. - unicode(Config) when is_list(Config) -> {module,unicode_expand} = compile_and_load(Config,'unicode_expand'), - {no,[],[{"'кlирилли́ческий атом'",0}, - {"'кlирилли́ческий атом'",1}, - {"'кlирилли́ческий атомB'",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("unicode_expand:"), - {yes,"рилли́ческий атом", []} = do_expand("unicode_expand:'кlи"), - {yes,"еский атом", []} = do_expand("unicode_expand:'кlирилли́ч"), - {yes,"(",[]} = do_expand("unicode_expand:'кlирилли́ческий атомB'"), - "\n'кlирилли́ческий атом'/0 'кlирилли́ческий атом'/1 " - "'кlирилли́ческий атомB'/1 \nmodule_info/0 " - "module_info/1 \n" = - do_format([{"'кlирилли́ческий атом'",0}, - {"'кlирилли́ческий атом'",1}, - {"'кlирилли́ческий атомB'",1}, - {"module_info",0}, - {"module_info",1}]), + {no,[], [#{elems:=[{"'кlирилли́ческий атом'",_}, + {"'кlирилли́ческий атомB'",_}, + {"module_info",_}]}]} = do_expand("unicode_expand:"), + {yes,"рилли́ческий атом", [#{elems:=[{"'кlирилли́ческий атом'",_}, + {"'кlирилли́ческий атомB'",_}]}]} = do_expand("unicode_expand:'кlи"), + {yes,"еский атом", [#{elems:=[{"'кlирилли́ческий атом'",_}, + {"'кlирилли́ческий атомB'",_}]}]} = do_expand("unicode_expand:'кlирилли́ч"), + {yes,"(", []} = do_expand("unicode_expand:'кlирилли́ческий атомB'"), + "'кlирилли́ческий атом' 'кlирилли́ческий атомB' module_info\n" = + do_format([{"'кlирилли́ческий атом'",[]}, + {"'кlирилли́ческий атомB'",[]}, + {"module_info",[]}]), ok. do_expand(String) -> - edlin_expand:expand(lists:reverse(String)). + erlang:display(String), + Bs = [ + {'Binding', 0}, + {'MapBinding', #{a_key=>0, b_key=>1, c_key=>2}}, + {'RecordBinding', {some_record, 1, 2}}, + {'TupleBinding', {0, 1, 2}}, + {'Söndag', 0}, + {'Ö', 0}], + Rt = ets:new(records, []), + + Rt2 = [{my_record, {attribute,[{text,"record"}, + {location,{1,2}}], + record, + {my_record, [{typed_record_field,{record_field,[{text,"field"}, + {location,{1,20}}], + {atom,[{text,"field"},{location,{1,20}}],field}, + {atom,[{text,"a_value"},{location,{1,28}}],a_value}}, + {user_type,[{text,"my_type"},{location,{1,33}}], + my_type,[]}}]}}}], + Ft = [{{function,{shell_default,my_func,1}},fun(_A)->0 end}, + {{function_type,{shell_default,my_func,1}}, + {attribute,[{text,"spec"},{location,{1,2}}], + spec, + {{my_func,1}, + [{type,[{text,"("},{location,{1,14}}], + bounded_fun, + [{type,[{text,"("},{location,{1,14}}], + 'fun', + [{type,[{text,"("},{location,{1,14}}], + product, + [{var,[{text,"A"},{location,{1,15}}],'A'}]}, + {type,[{text,"integer"},{location,{1,21}}],integer,[]}]}, + [{type,[{text,"A"},{location,{1,36}}], + constraint, + [{atom,[{text,"A"},{location,{1,36}}],is_subtype}, + [{var,[{text,"A"},{location,{1,36}}],'A'}, + {type,[{text,"#"},{location,{1,41}}], + record, + [{atom,[{text,"my_record"},{location,{1,42}}], + my_record}]}]]}]]}]}}}, + {{type,my_type}, + {attribute,[{text,"type"},{location,{1,2}}], + type, + {my_type,{type,[{text,"a"},{location,{1,20}}], + union, + [{atom,[{text,"a_value"},{location,{1,20}}],a_value}, + {atom,[{text,"b_value"},{location,{1,24}}],b_value}]}, + []}}}], + shell:read_and_add_records(edlin_expand_SUITE, '_', [], Bs, Rt), + edlin_expand:expand(lists:reverse(String), [], {shell_state, Bs, ets:tab2list(Rt)++Rt2, Ft}). do_format(StringList) -> - lists:flatten(edlin_expand:format_matches(StringList)). + lists:flatten(edlin_expand:format_matches(StringList, 79)). + +compile_and_load2(Config, Module) -> + Filename = filename:join( + proplists:get_value(data_dir,Config), + atom_to_list(Module)), + PrivDir = proplists:get_value(priv_dir,Config), + c:c(Filename, [debug_info, {outdir, PrivDir}]). compile_and_load(Config,Module) -> Filename = filename:join( diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden😀_file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden😀_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden😀_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl b/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl new file mode 100644 index 0000000000..fc135b23a0 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl @@ -0,0 +1,164 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(complete_function_parameter). + +-export( + [a_fun_name/2, + an_untyped_fun/2, + a_deeplist_fun/1, + a_zero_arity_fun/0, + multi_arity_fun/0, + multi_arity_fun/1, + multi_arity_fun/2, + different_multi_arity_fun/1, + different_multi_arity_fun/2, + advanced_nested_parameter/1, + test_year/1, + 'emoji_function🤯'/1, + map_parameter_function/1, + map_parameter_function/2, + tuple_parameter_function/2, + list_parameter_function/2, + non_empty_list_parameter_function/2, + binary_parameter_function/2, + neg_integer_parameter_function/2, + non_neg_integer_parameter_function/2, + integer_parameter_function/2, + float_parameter_function/2, + port_parameter_function/2, + pid_parameter_function/2, + record_parameter_function/2, + function_parameter_function/2, + reference_parameter_function/2, + any_parameter_function/2, + ann_type_parameter_function/2, + ann_type_parameter_function2/2, + atom_parameter_function/2 + ]). +-record(a_record, {}). +%% test first and second parameter + %% test multiple arities with same type on first parameter + %% test multiple arities with different type on first parameter + %% test that recursive types does not trigger endless loop + %% test that getting type of out of bound parameter does not trigger crash +-spec a_fun_name(Start, End) -> Return when + Start :: integer(), + End :: integer(), + Return:: integer(). +a_fun_name(_Start, _End) -> 0. + +an_untyped_fun(_Start, _End) -> 1. + +-spec a_deeplist_fun(Deeplist) -> integer() when + Deeplist :: T | [Deeplist], + T :: term(). +a_deeplist_fun(Deeplist) -> lists:flatten(Deeplist). + +a_zero_arity_fun() -> 0. + +-spec multi_arity_fun() -> integer(). +multi_arity_fun() -> 0. + +-spec multi_arity_fun(T1) -> integer() when + T1 :: integer(). +multi_arity_fun(_T1) -> 1. + +-spec multi_arity_fun(T1,T2) -> integer() when + T1 :: integer(), + T2 :: boolean(). +multi_arity_fun(_T1, _T2) -> 2. + +-spec different_multi_arity_fun(T1) -> integer() when + T1 :: integer(). +different_multi_arity_fun(_T1) -> 1. +-spec different_multi_arity_fun(B1, T1) -> integer() when + B1 :: boolean(), + T1 :: integer(). +different_multi_arity_fun(_T1, _T2) -> 2. + +-spec advanced_nested_parameter(T1) -> integer() when + T1 :: {atom1, {non_neg_integer(), non_neg_integer()}} | 'atom1' | 'atom2' | ['atom4' | 'atom5']. +advanced_nested_parameter(_T1) -> 0. + +-spec test_year(Y) -> integer() when + Y :: calendar:year(). +test_year(_Y) -> 0. + +-spec 'emoji_function🤯'(integer()) -> integer(). +'emoji_function🤯'(_) -> 0. + +-spec map_parameter_function(Map) -> boolean() when + Map :: #{a => 1, b => 2, c => 3, d => error}. +map_parameter_function(_) -> false. +-spec map_parameter_function(Map, any()) -> boolean() when + Map :: #{a => 1, b => 2, c => 3, d => error}. +map_parameter_function(_,_) -> false. + +-spec binary_parameter_function(binary(), any()) -> boolean(). +binary_parameter_function(_,_) -> false. + +-spec tuple_parameter_function(tuple(), any()) -> boolean(). +tuple_parameter_function(_,_) -> false. + +-spec list_parameter_function(list(), any()) -> boolean(). +list_parameter_function(_,_) -> false. + +-spec non_empty_list_parameter_function(nonempty_list(), any()) -> boolean(). +non_empty_list_parameter_function(_,_) -> false. + +-spec integer_parameter_function(integer(), any()) -> boolean(). +integer_parameter_function(_,_) -> false. + +-spec non_neg_integer_parameter_function(non_neg_integer(), any()) -> boolean(). +non_neg_integer_parameter_function(_,_) -> false. + +-spec neg_integer_parameter_function(neg_integer(), any()) -> boolean(). +neg_integer_parameter_function(_,_) -> false. + +-spec float_parameter_function(float(), any()) -> boolean(). +float_parameter_function(_,_) -> false. + +-spec pid_parameter_function(pid(), any()) -> boolean(). +pid_parameter_function(_,_) -> false. + +-spec port_parameter_function(port(), any()) -> boolean(). +port_parameter_function(_,_) -> false. + +-spec record_parameter_function(A, any()) -> boolean() when + A :: #a_record{}. +record_parameter_function(_,_) -> false. + +-spec function_parameter_function(fun((any()) -> any()), any()) -> boolean(). +function_parameter_function(_,_) -> false. + +-spec reference_parameter_function(reference(), any()) -> boolean(). +reference_parameter_function(_,_) -> false. + +-spec any_parameter_function(any(), any()) -> boolean(). +any_parameter_function(_,_) -> false. + +-spec atom_parameter_function(atom, any()) -> boolean(). +atom_parameter_function(_,_) -> false. + +-spec ann_type_parameter_function(V::atom(), W::any()) -> boolean(). +ann_type_parameter_function(_,_) -> false. + +-spec ann_type_parameter_function2(W::any(), V::atom()) -> boolean(). +ann_type_parameter_function2(_,_) -> false. diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible file b/lib/stdlib/test/edlin_expand_SUITE_data/visible file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible_file b/lib/stdlib/test/edlin_expand_SUITE_data/visible_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible😀_file b/lib/stdlib/test/edlin_expand_SUITE_data/visible😀_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible😀_file diff --git a/lib/stdlib/test/erl_expand_records_SUITE.erl b/lib/stdlib/test/erl_expand_records_SUITE.erl index c48a1ac90e..092c579b4c 100644 --- a/lib/stdlib/test/erl_expand_records_SUITE.erl +++ b/lib/stdlib/test/erl_expand_records_SUITE.erl @@ -39,7 +39,7 @@ -export([attributes/1, expr/1, guard/1, init/1, pattern/1, strict/1, update/1, otp_5915/1, otp_7931/1, otp_5990/1, - otp_7078/1, otp_7101/1, maps/1, + otp_7078/1, maps/1, side_effects/1]). init_per_testcase(_Case, Config) -> @@ -59,7 +59,7 @@ all() -> groups() -> [{tickets, [], - [otp_5915, otp_7931, otp_5990, otp_7078, otp_7101]}]. + [otp_5915, otp_7931, otp_5990, otp_7078]}]. init_per_suite(Config) -> Config. @@ -703,67 +703,8 @@ otp_7078(Config) when is_list(Config) -> run(Config, Ts, [strict_record_tests]), ok. --record(otp_7101, {a,b,c=[],d=[],e=[]}). - id(I) -> I. -%% OTP-7101. Record update: more than one call to setelement/3. -otp_7101(Config) when is_list(Config) -> - %% Ensure the compiler won't do any funny constant propagation tricks. - id(#otp_7101{a=a,b=b,c=c,d=d,e=e}), - Rec = id(#otp_7101{}), - - %% Spawn a tracer process to count the number of setelement/3 calls. - %% The tracer will forward all trace messages to us. - Self = self(), - Tracer = spawn_link(fun() -> otp_7101_tracer(Self, 0) end), - 1 = erlang:trace_pattern({erlang,setelement,3}, true), - erlang:trace(self(), true, [{tracer,Tracer},call]), - - %% Update the record. - #otp_7101{a=2,b=1,c=[],d=[],e=[]} = otp_7101_update1(Rec), - #otp_7101{a=1,b=2,c=[],d=[],e=[]} = otp_7101_update2(Rec), - #otp_7101{a=2,b=1,c=[],d=[],e=[]} = otp_7101_update3(Rec), - #otp_7101{a=1,b=2,c=[],d=[],e=[]} = otp_7101_update4(Rec), - - %% Verify that setelement/3 was called the same number of times as - %% the number of record updates. - Ref = erlang:trace_delivered(Self), - receive - {trace_delivered, Self, Ref} -> - Tracer ! done - end, - 1 = erlang:trace_pattern({erlang,setelement,3}, false), - receive - 4 -> - ok; - Other -> - ct:fail({unexpected,Other}) - end. - -otp_7101_tracer(Parent, N) -> - receive - {trace,Parent,call,{erlang,setelement,[_,_,_]}} -> - otp_7101_tracer(Parent, N+1); - done -> - Parent ! N - end. - -otp_7101_update1(R) -> - R#otp_7101{b=1, - a=2}. - -otp_7101_update2(R) -> - R#otp_7101{a=1, - b=2}. - -otp_7101_update3(R) -> - R#otp_7101{b=1,a=2}. - -otp_7101_update4(R) -> - R#otp_7101{a=1,b=2}. - - -record(side_effects, {a,b,c}). %% Make sure that the record expression is only evaluated once. diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index b37bcd43da..2f2f0fc23e 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -78,7 +78,8 @@ underscore_match/1, unused_record/1, unused_type2/1, - eep49/1]). + eep49/1, + redefined_builtin_type/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -106,7 +107,8 @@ all() -> no_load_nif, inline_nifs, warn_missing_spec, otp_16824, underscore_match, unused_record, unused_type2, - eep49]. + eep49, + redefined_builtin_type]. groups() -> [{unused_vars_warn, [], @@ -966,13 +968,13 @@ binary_types(Config) when is_list(Config) -> Ts = [{binary1, <<"-type nonempty_binary() :: term().">>, [nowarn_unused_type], - {errors,[{{1,22},erl_lint, - {builtin_type,{nonempty_binary,0}}}],[]}}, + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_binary,0}}}]}}, {binary2, <<"-type nonempty_bitstring() :: term().">>, [nowarn_unused_type], - {errors,[{{1,22},erl_lint, - {builtin_type,{nonempty_bitstring,0}}}],[]}}], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_bitstring,0}}}]}}], [] = run(Config, Ts), ok. @@ -2810,9 +2812,9 @@ otp_11772(Config) when is_list(Config) -> t() -> 1. ">>, - {errors,[{{7,14},erl_lint,{builtin_type,{node,0}}}, - {{8,14},erl_lint,{builtin_type,{mfa,0}}}], - []} = run_test2(Config, Ts, []), + {warnings,[{{7,14},erl_lint,{redefine_builtin_type,{node,0}}}, + {{8,14},erl_lint,{redefine_builtin_type,{mfa,0}}}]} = + run_test2(Config, Ts, []), ok. %% OTP-11771. Do not allow redefinition of the types arity(_) &c.. @@ -2835,11 +2837,11 @@ otp_11771(Config) when is_list(Config) -> t() -> 1. ">>, - {errors,[{{7,14},erl_lint,{builtin_type,{arity,0}}}, - {{8,14},erl_lint,{builtin_type,{bitstring,0}}}, - {{9,14},erl_lint,{builtin_type,{iodata,0}}}, - {{10,14},erl_lint,{builtin_type,{boolean,0}}}], - []} = run_test2(Config, Ts, []), + {warnings,[{{7,14},erl_lint,{redefine_builtin_type,{arity,0}}}, + {{8,14},erl_lint,{redefine_builtin_type,{bitstring,0}}}, + {{9,14},erl_lint,{redefine_builtin_type,{iodata,0}}}, + {{10,14},erl_lint,{redefine_builtin_type,{boolean,0}}}]} = + run_test2(Config, Ts, []), ok. %% OTP-11872. The type map() undefined when exported. @@ -2851,15 +2853,16 @@ otp_11872(Config) when is_list(Config) -> -export_type([map/0, product/0]). - -opaque map() :: dict(). + -opaque map() :: unknown_type(). -spec t() -> map(). t() -> 1. ">>, - {errors,[{{6,14},erl_lint,{undefined_type,{product,0}}}, - {{8,14},erl_lint,{builtin_type,{map,0}}}], []} = + {error,[{{6,14},erl_lint,{undefined_type,{product,0}}}, + {{8,30},erl_lint,{undefined_type,{unknown_type,0}}}], + [{{8,14},erl_lint,{redefine_builtin_type,{map,0}}}]} = run_test2(Config, Ts, []), ok. @@ -3869,7 +3872,7 @@ maps_type(Config) when is_list(Config) -> t(M) -> M. ">>, [], - {errors,[{{3,7},erl_lint,{builtin_type,{map,0}}}],[]}}], + {warnings,[{{3,7},erl_lint,{redefine_builtin_type,{map,0}}}]}}], [] = run(Config, Ts), ok. @@ -4837,6 +4840,72 @@ eep49(Config) when is_list(Config) -> [] = run(Config, Ts), ok. +%% GH-6132: Allow local redefinition of types. +redefined_builtin_type(Config) -> + Ts = [{redef1, + <<"-type nonempty_binary() :: term(). + -type map() :: {_,_}.">>, + [nowarn_unused_type, + nowarn_redefined_builtin_type], + []}, + {redef2, + <<"-type nonempty_bitstring() :: term(). + -type map() :: {_,_}.">>, + [nowarn_unused_type, + {nowarn_redefined_builtin_type,{map,0}}], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_bitstring,0}}}]}}, + {redef3, + <<"-compile({nowarn_redefined_builtin_type,{map,0}}). + -compile({nowarn_redefined_builtin_type,[{nonempty_bitstring,0}]}). + -type nonempty_bitstring() :: term(). + -type map() :: {_,_}. + -type list() :: erlang:map().">>, + [nowarn_unused_type, + {nowarn_redefined_builtin_type,{map,0}}], + {warnings,[{{5,16},erl_lint, + {redefine_builtin_type,{list,0}}}]}}, + {redef4, + <<"-type tuple() :: 'tuple'. + -type map() :: 'map'. + -type list() :: 'list'. + -spec t(tuple() | map()) -> list(). + t(_) -> ok. + ">>, + [], + {warnings,[{{1,22},erl_lint,{redefine_builtin_type,{tuple,0}}}, + {{2,16},erl_lint,{redefine_builtin_type,{map,0}}}, + {{3,16},erl_lint,{redefine_builtin_type,{list,0}}} + ]}}, + {redef5, + <<"-type atom() :: 'atom'. + -type integer() :: 'integer'. + -type reference() :: 'ref'. + -type pid() :: 'pid'. + -type port() :: 'port'. + -type float() :: 'float'. + -type iodata() :: 'iodata'. + -type ref_set() :: gb_sets:set(reference()). + -type pid_map() :: #{pid() => port()}. + -type atom_int_fun() :: fun((atom()) -> integer()). + -type collection(Type) :: {'collection', Type}. + -callback b1(I :: iodata()) -> atom(). + -spec t(collection(float())) -> {pid_map(), ref_set(), atom_int_fun()}. + t(_) -> ok. + ">>, + [], + {warnings,[{{1,22},erl_lint,{redefine_builtin_type,{atom,0}}}, + {{2,16},erl_lint,{redefine_builtin_type,{integer,0}}}, + {{3,16},erl_lint,{redefine_builtin_type,{reference,0}}}, + {{4,16},erl_lint,{redefine_builtin_type,{pid,0}}}, + {{5,16},erl_lint,{redefine_builtin_type,{port,0}}}, + {{6,16},erl_lint,{redefine_builtin_type,{float,0}}}, + {{7,16},erl_lint,{redefine_builtin_type,{iodata,0}}} + ]}} + ], + [] = run(Config, Ts), + ok. + format_error(E) -> lists:flatten(erl_lint:format_error(E)). diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index ee8bc8420f..6c4694bebe 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -676,6 +676,8 @@ illegal() -> crashes() -> {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error + {'EXIT',_} = (catch erl_scan:string("'a" ++ [999999999] ++ "c'")), + {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}), {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}), {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}), @@ -698,6 +700,7 @@ crashes() -> (catch {foo, erl_scan:string("% foo"++[a],{1,1})}), {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error + {'EXIT',_} = (catch {foo, erl_scan:string("A" ++ [999999999])}), ok. @@ -867,8 +870,8 @@ unicode() -> erl_scan:string([1089]), {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = erl_scan:string([1089], {1,1}), - {error,{{1,3},erl_scan,{illegal,character}},{1,4}} = - erl_scan:string("'a" ++ [999999999] ++ "c'", {1,1}), + {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = + erl_scan:string([16#D800], {1,1}), test("\"a"++[1089]++"b\""), {ok,[{char,1,1}],1} = diff --git a/lib/stdlib/test/escript_SUITE_data/arg_overflow b/lib/stdlib/test/escript_SUITE_data/arg_overflow index dd5accc051..e3138cabbd 100755 --- a/lib/stdlib/test/escript_SUITE_data/arg_overflow +++ b/lib/stdlib/test/escript_SUITE_data/arg_overflow @@ -1,5 +1,5 @@ #! /usr/bin/env escript %% -*- erlang -*- -%%!x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x +%%!-x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x main(_) -> halt(0). diff --git a/lib/stdlib/test/escript_SUITE_data/linebuf_overflow b/lib/stdlib/test/escript_SUITE_data/linebuf_overflow index 33133c1ce9..018be1f26d 100755 --- a/lib/stdlib/test/escript_SUITE_data/linebuf_overflow +++ b/lib/stdlib/test/escript_SUITE_data/linebuf_overflow @@ -1,5 +1,5 @@ #! /usr/bin/env escript %% -*- erlang -*- -%%!xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +%%!-v xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx main(_) -> halt(0). diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index de3e100192..afac20c6db 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -40,7 +40,7 @@ -export([tab2file/1, tab2file2/1, tabfile_ext1/1, tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). -export([heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). --export([lookup_element_mult/1]). +-export([lookup_element_mult/1, lookup_element_default/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). -export([t_delete_object/1, t_init_table/1, t_whitebox/1, select_bound_chunk/1, t_delete_all_objects/1, t_test_ms/1, @@ -194,7 +194,7 @@ groups() -> privacy]}, {insert, [], [empty, badinsert]}, {lookup, [], [badlookup, lookup_order]}, - {lookup_element, [], [lookup_element_mult]}, + {lookup_element, [], [lookup_element_mult, lookup_element_default]}, {delete, [], [delete_elem, delete_tab, delete_large_tab, delete_large_named_table, evil_delete, table_leak, @@ -4112,6 +4112,41 @@ fill_tab(Tab,Val) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +lookup_element_default(Config) when is_list(Config) -> + EtsMem = etsmem(), + + TabSet = ets_new(foo, [set]), + ets:insert(TabSet, {key, 42}), + 42 = ets:lookup_element(TabSet, key, 2, 13), + 13 = ets:lookup_element(TabSet, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabSet, key, 3, 13), + true = ets:delete(TabSet), + + TabOrderedSet = ets_new(foo, [ordered_set]), + ets:insert(TabOrderedSet, {key, 42}), + 42 = ets:lookup_element(TabOrderedSet, key, 2, 13), + 13 = ets:lookup_element(TabOrderedSet, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabOrderedSet, key, 3, 13), + true = ets:delete(TabOrderedSet), + + TabBag = ets_new(foo, [bag]), + ets:insert(TabBag, {key, 42}), + ets:insert(TabBag, {key, 43, 44}), + [42, 43] = ets:lookup_element(TabBag, key, 2, 13), + 13 = ets:lookup_element(TabBag, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabBag, key, 3, 13), + true = ets:delete(TabBag), + + TabDuplicateBag = ets_new(foo, [duplicate_bag]), + ets:insert(TabDuplicateBag, {key, 42}), + ets:insert(TabDuplicateBag, {key, 42}), + ets:insert(TabDuplicateBag, {key, 43, 44}), + [42, 42, 43] = ets:lookup_element(TabDuplicateBag, key, 2, 13), + 13 = ets:lookup_element(TabDuplicateBag, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabDuplicateBag, key, 3, 13), + true = ets:delete(TabDuplicateBag), + + verify_etsmem(EtsMem). %% OTP-2386. Multiple return elements. lookup_element_mult(Config) when is_list(Config) -> @@ -9157,6 +9192,9 @@ error_info(_Config) -> {lookup_element, [OneKeyTab, one, 4]}, + {lookup_element, ['$Tab', no_key, 1, default_value], [no_fail]}, + {lookup_element, [OneKeyTab, one, 4, default_value]}, + {match, [bad_continuation], [no_table]}, {match, ['$Tab', <<1,2,3>>], [no_fail]}, diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 525b479fef..bc96992ce2 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -22,55 +22,27 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). --export([init_per_testcase/2, end_per_testcase/2]). - -export([setopts_getopts/1,unicode_options/1,unicode_options_gen/1, binary_options/1, read_modes_gl/1, - read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1,unicode_prompt/1]). + read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1, + unicode_prompt/1, shell_slogan/1]). -export([io_server_proxy/1,start_io_server_proxy/0, proxy_getall/1, proxy_setnext/2, proxy_quit/1]). %% For spawn --export([toerl_server/3,answering_machine1/3, - answering_machine2/3]). - --export([uprompt/1]). +-export([answering_machine1/3, answering_machine2/3]). -%%-define(without_test_server, true). - --ifdef(without_test_server). --define(line, put(line, ?LINE), ). --define(config(X,Y), foo). --define(t, test_server). --define(privdir(_), "./io_SUITE_priv"). --else. --include_lib("common_test/include/ct.hrl"). --define(privdir(Conf), proplists:get_value(priv_dir, Conf)). --endif. +-export([uprompt/1, slogan/0, session_slogan/0]). %%-define(debug, true). -ifdef(debug). --define(format(S, A), io:format(S, A)). -define(dbg(Data),io:format(standard_error, "DBG: ~p\r\n",[Data])). --define(RM_RF(Dir),begin io:format(standard_error, "Not Removed: ~p\r\n",[Dir]), - ok end). -else. --define(format(S, A), ok). -define(dbg(Data),noop). --define(RM_RF(Dir),rm_rf(Dir)). -endif. -init_per_testcase(_Case, Config) -> - Term = os:getenv("TERM", "dumb"), - os:putenv("TERM","vt100"), - [{term, Term} | Config]. -end_per_testcase(_Case, Config) -> - Term = proplists:get_value(term,Config), - os:putenv("TERM",Term), - ok. - suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,5}}]. @@ -78,16 +50,21 @@ suite() -> all() -> [setopts_getopts, unicode_options, unicode_options_gen, binary_options, read_modes_gl, read_modes_ogl, - broken_unicode, eof_on_pipe, unicode_prompt]. + broken_unicode, eof_on_pipe, unicode_prompt, + shell_slogan]. groups() -> []. init_per_suite(Config) -> - DefShell = get_default_shell(), - [{default_shell,DefShell}|Config]. + Term = os:getenv("TERM", "dumb"), + os:putenv("TERM","vt100"), + DefShell = rtnode:get_default_shell(), + [{default_shell,DefShell},{term, Term}|Config]. -end_per_suite(_Config) -> +end_per_suite(Config) -> + Term = proplists:get_value(term,Config), + os:putenv("TERM",Term), ok. init_per_group(_GroupName, Config) -> @@ -96,13 +73,11 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - - -record(state, { - q = [], - nxt = eof, - mode = list - }). + q = [], + nxt = eof, + mode = list + }). uprompt(_L) -> [1050,1072,1082,1074,1086,32,1077,32,85,110,105,99,111,100,101,32,63]. @@ -111,41 +86,73 @@ uprompt(_L) -> unicode_prompt(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), case proplists:get_value(default_shell,Config) of - old -> - ok; new -> - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, - {getline, "default"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"} - ],[],[],"-pa \""++ PA++"\"") + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2"}, + {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, + {expect, "[\n ]default"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect,"[\n ]hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-pa",PA]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, - {getline_re, ".*default"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"} - ],[],[],"-oldshell -pa \""++PA++"\""), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2"}, + {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, + {expect, "default"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]\\?*ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect,"[\n ]\\?*hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-oldshell","-pa",PA]), ok. +%% Test that an Unicode prompt does not crash the shell. +shell_slogan(Config) when is_list(Config) -> + PA = filename:dirname(code:which(?MODULE)), + case proplists:get_value(default_shell,Config) of + new -> + rtnode:run( + [{expect, "\\Q"++string:trim(erlang:system_info(system_version))++"\\E"}, + {expect, "\\Q"++io_lib:format("Eshell V~s (press Ctrl+G to abort, type help(). for help)",[erlang:system_info(version)])++"\\E"} + ],[],"",[]), + rtnode:run( + [{expect, "\nTest slogan"}, + {expect, "\nTest session slogan \\("} + ],[],"",["-stdlib","shell_slogan","\"Test slogan\"", + "-stdlib","shell_session_slogan","\"Test session slogan\""]), + rtnode:run( + [{expect, "\nTest slogan"}, + {expect, "\\Q\nTest session slogan (\\E"} + ],[],"",["-stdlib","shell_slogan","fun io_proto_SUITE:slogan/0", + "-stdlib","shell_session_slogan","fun io_proto_SUITE:session_slogan/0", + "-pa",PA]); + _ -> + ok + end. + +slogan() -> + "Test slogan". +session_slogan() -> + "Test session slogan". %% Check io:setopts and io:getopts functions. setopts_getopts(Config) when is_list(Config) -> @@ -222,40 +229,42 @@ setopts_getopts(Config) when is_list(Config) -> eof = io:get_line(RFile,''), file:close(RFile), case proplists:get_value(default_shell,Config) of - old -> - ok; new -> %% So, lets test another node with new interactive shell - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline, "{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"} - ],[]) + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline_re, ".*{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"} - ],[],[],"-oldshell"), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-oldshell"]), ok. @@ -419,42 +428,38 @@ unicode_options(Config) when is_list(Config) -> [ ok = CannotWriteFile(F,FailDir) || F <- AllNoBom ], case proplists:get_value(default_shell,Config) of - old -> - ok; new -> %% OK, time for the group_leaders... - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(encoding,1,io:getopts())."}, - {getline, "{encoding,latin1}"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline, "\\x{400}"}, - {putline, "io:setopts([unicode])."}, - {getline, "ok"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline, - binary_to_list(unicode:characters_to_binary( - [1024],unicode,utf8))} - ],[],"LC_CTYPE=\""++get_lc_ctype()++"\"; " - "export LC_CTYPE; ") + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(encoding,1,io:getopts())."}, + {expect, "{encoding,latin1}"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "\\Q\\x{400}\\E"}, + {putline, "io:setopts([unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "[\n ]"++[1024]} + ],[],"",["-env","LC_ALL",get_lc_ctype()]); + _ -> + ok end, - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(encoding,1,io:getopts())."}, - {getline_re, ".*{encoding,latin1}"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline_re, ".*\\\\x{400\\}"}, - {putline, "io:setopts([{encoding,unicode}])."}, - {getline_re, ".*ok"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline_re, - ".*"++binary_to_list(unicode:characters_to_binary( - [1024],unicode,utf8))} - ],[],"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; ", - " -oldshell "), - + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(encoding,1,io:getopts())."}, + {expect, "[\n ]{encoding,latin1}"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "\\Q\\x{400}\\E"}, + {putline, "io:setopts([{encoding,unicode}])."}, + {expect, "[\n ]ok"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "[\n ]"++[1024]} + ],[],"", + ["-oldshell","-env","LC_ALL",get_lc_ctype()]), ok. %% Tests various unicode options on random generated files. @@ -709,115 +714,124 @@ binary_options(Config) when is_list(Config) -> %% OK, time for the group_leaders... case proplists:get_value(default_shell,Config) of - old -> - ok; new -> - rtnode([{putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline, "{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true},unicode])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"}, - {putline, "io:get_line('')."}, - {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, - {getline, "<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>"} - ],[]) + rtnode:run( + [{putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true},unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"}, + {putline, "io:get_line('')."}, + {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, + {expect, latin1, "[\n ]\\Q<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>\\E"} + ],[]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline_re, ".*{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true},unicode])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"}, - {putline, "io:get_line('')."}, - {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, - {getline_re, ".*<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\\\n\"/utf8>>"} - ],[],[],"-oldshell"), + rtnode:run( + [{putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "[\n ]\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true},unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"}, + {putline, "io:get_line('')."}, + {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, + {expect, latin1, "[\n ]\\Q<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>\\E"} + ],[],"",["-oldshell"]), ok. - - - answering_machine1(OthNode,OthReg,Me) -> TestDataLine1 = [229,228,246], TestDataUtf = binary_to_list(unicode:characters_to_binary(TestDataLine1)), - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, - {getline, "<"}, - %% get_line - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% get_chars - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% fread - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"} - - ],Me,"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; "), + TestDataLine1Oct = "\\\\345( \b)*\\\\344( \b)*\\\\366", + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "2"}, + {putline, "io:getopts()."}, + {expect, ">"}, + {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, + {expect, "<"}, + %% get_line + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% get_chars + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% fread + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"} + + ],Me,"",["-env","LC_ALL",get_lc_ctype()]), O = list_to_atom(OthReg), O ! {self(),done}, ok. @@ -825,70 +839,77 @@ answering_machine1(OthNode,OthReg,Me) -> answering_machine2(OthNode,OthReg,Me) -> TestDataLine1 = [229,228,246], TestDataUtf = binary_to_list(unicode:characters_to_binary(TestDataLine1)), - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, - {getline_re, ".*<[0-9].*"}, - %% get_line - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% get_chars - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% fread - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"} - - ],Me,"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; "," -oldshell "), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "2"}, + {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, + {expect, "<[0-9].*"}, + %% get_line + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% get_chars + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% fread + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"} + + ],Me,"",["-oldshell","-env","LC_ALL",get_lc_ctype()]), O = list_to_atom(OthReg), O ! {self(),done}, ok. @@ -896,20 +917,20 @@ answering_machine2(OthNode,OthReg,Me) -> %% Test various modes when reading from the group leade from another machine. read_modes_ogl(Config) when is_list(Config) -> - case get_progs() of - {error,Reason} -> - {skipped,Reason}; + case proplists:get_value(default_shell,Config) of + noshell -> + {skipped,"No run_erl"}; _ -> read_modes_gl_1(Config,answering_machine2) end. %% Test various modes when reading from the group leade from another machine. read_modes_gl(Config) when is_list(Config) -> - case {get_progs(),proplists:get_value(default_shell,Config)} of - {{error,Reason},_} -> - {skipped,Reason}; - {_,old} -> - {skipper,"No new shell"}; + case proplists:get_value(default_shell,Config) of + noshell -> + {skipped,"No run_erl"}; + old -> + {skipped,"No new shell"}; _ -> read_modes_gl_1(Config,answering_machine1) end. @@ -919,7 +940,7 @@ read_modes_gl_1(_Config,Machine) -> TestDataLine1BinUtf = unicode:characters_to_binary(TestDataLine1), TestDataLine1BinLatin = list_to_binary(TestDataLine1), - {ok,N2List} = create_nodename(), + N2List = peer:random_name(?FUNCTION_NAME), MyNodeList = atom2list(node()), register(io_proto_suite,self()), AM1 = spawn(?MODULE,Machine, @@ -1023,14 +1044,10 @@ loop_through_file2(_,{error,_Err},_,_) -> loop_through_file2(F,Bin,Chunk,Enc) when is_binary(Bin) -> loop_through_file2(F,io:get_chars(F,'',Chunk),Chunk,Enc). - - %% Test eof before newline on stdin when erlang is in pipe. eof_on_pipe(Config) when is_list(Config) -> - case {get_progs(),os:type()} of - {{error,Reason},_} -> - {skipped,Reason}; - {{_,_,Erl},{unix,linux}} -> + case {ct:get_progname(),os:type()} of + {Erl,{unix,linux}} -> %% Not even Linux is reliable - echo can be both styles try EchoLine = case os:cmd("echo -ne \"test\\ntest\"") of @@ -1074,453 +1091,6 @@ eof_on_pipe(Config) when is_list(Config) -> {skipped,"Only on linux"} end. - -%% -%% Tool for running interactive shell (stolen from the kernel -%% test suite interactive_shell_SUITE) -%% --undef(line). --define(line,). -rtnode(C,N) -> - rtnode(C,N,[]). -rtnode(Commands,Nodename,ErlPrefix) -> - rtnode(Commands,Nodename,ErlPrefix,[]). -rtnode(Commands,Nodename,ErlPrefix,Extra) -> - case get_progs() of - {error,_Reason} -> - {skip,"No runerl present"}; - {RunErl,ToErl,Erl} -> - case create_tempdir() of - {error, Reason2} -> - {skip, Reason2}; - Tempdir -> - SPid = start_runerl_node(RunErl, ErlPrefix++ - "\\\""++Erl++"\\\"", - Tempdir, Nodename, Extra), - CPid = start_toerl_server(ToErl, Tempdir), - put(getline_skipped, []), - Res = (catch get_and_put(CPid, Commands, 1)), - case stop_runerl_node(CPid) of - {error,_} -> - CPid2 = start_toerl_server(ToErl, Tempdir), - put(getline_skipped, []), - ok = get_and_put - (CPid2, - [{putline,[7]}, - {sleep, - timeout(short)}, - {putline,""}, - {getline," -->"}, - {putline,"s"}, - {putline,"c"}, - {putline,""}], 1), - stop_runerl_node(CPid2); - _ -> - ok - end, - wait_for_runerl_server(SPid), - ok = ?RM_RF(Tempdir), - ok = Res - end - end. - -timeout(long) -> - 2 * timeout(normal); -timeout(short) -> - timeout(normal) div 10; -timeout(normal) -> - 10000 * test_server:timetrap_scale_factor(). - - -%% start_noshell_node(Name) -> -%% PADir = filename:dirname(code:which(?MODULE)), -%% {ok, Node} = test_server:start_node(Name,slave,[{args," -noshell -pa "++ -%% PADir++" "}]), -%% Node. -%% stop_noshell_node(Node) -> -%% test_server:stop_node(Node). - --ifndef(debug). -rm_rf(Dir) -> - try - {ok,List} = file:list_dir(Dir), - Files = [filename:join([Dir,X]) || X <- List], - [case file:list_dir(Y) of - {error, enotdir} -> - ok = file:delete(Y); - _ -> - ok = rm_rf(Y) - end || Y <- Files], - ok = file:del_dir(Dir), - ok - catch - _:Exception -> {error, {Exception,Dir}} - end. --endif. - -get_and_put(_CPid,[],_) -> - ok; -get_and_put(CPid, [{sleep, X}|T],N) -> - ?dbg({sleep, X}), - receive - after X -> - get_and_put(CPid,T,N+1) - end; -get_and_put(CPid, [{getline_pred,Pred,Msg}|T]=T0, N) - when is_function(Pred) -> - ?dbg({getline, Match}), - CPid ! {self(), {get_line, timeout(normal)}}, - receive - {get_line, timeout} -> - error_logger:error_msg("~p: getline timeout waiting for \"~s\" " - "(command number ~p, skipped: ~p)~n", - [?MODULE,Msg,N,get(getline_skipped)]), - {error, timeout}; - {get_line, Data} -> - ?dbg({data,Data}), - case Pred(Data) of - yes -> - put(getline_skipped, []), - get_and_put(CPid, T,N+1); - no -> - error_logger:error_msg("~p: getline match failure " - "\"~s\" " - "(command number ~p)\n", - [?MODULE,Msg,N]), - {error, no_match}; - 'maybe' -> - List = get(getline_skipped), - put(getline_skipped, List ++ [Data]), - get_and_put(CPid, T0, N) - end - end; -get_and_put(CPid, [{getline, Match}|T],N) -> - ?dbg({getline, Match}), - F = fun(Data) -> - case lists:prefix(Match, Data) of - true -> yes; - false -> 'maybe' - end - end, - get_and_put(CPid, [{getline_pred,F,Match}|T], N); -get_and_put(CPid, [{getline_re, Match}|T],N) -> - F = fun(Data) -> - case re:run(Data, Match, [{capture,none}]) of - match -> yes; - _ -> 'maybe' - end - end, - get_and_put(CPid, [{getline_pred,F,Match}|T], N); -get_and_put(CPid, [{putline_raw, Line}|T],N) -> - ?dbg({putline_raw, Line}), - CPid ! {self(), {send_line, Line}}, - Timeout = timeout(normal), - receive - {send_line, ok} -> - get_and_put(CPid, T,N+1) - after Timeout -> - error_logger:error_msg("~p: putline_raw timeout (~p) sending " - "\"~s\" (command number ~p)~n", - [?MODULE, Timeout, Line, N]), - {error, timeout} - end; - -get_and_put(CPid, [{putline, Line}|T],N) -> - ?dbg({putline, Line}), - CPid ! {self(), {send_line, Line}}, - Timeout = timeout(normal), - receive - {send_line, ok} -> - get_and_put(CPid, [{getline, []}|T],N) - after Timeout -> - error_logger:error_msg("~p: putline timeout (~p) sending " - "\"~s\" (command number ~p)~n[~p]~n", - [?MODULE, Timeout, Line, N,get()]), - {error, timeout} - end. - -wait_for_runerl_server(SPid) -> - Ref = erlang:monitor(process, SPid), - Timeout = timeout(long), - receive - {'DOWN', Ref, process, SPid, _} -> - ok - after Timeout -> - {error, timeout} - end. - - - -stop_runerl_node(CPid) -> - Ref = erlang:monitor(process, CPid), - CPid ! {self(), kill_emulator}, - Timeout = timeout(long), - receive - {'DOWN', Ref, process, CPid, noproc} -> - ok; - {'DOWN', Ref, process, CPid, normal} -> - ok; - {'DOWN', Ref, process, CPid, {error, Reason}} -> - {error, Reason} - after Timeout -> - {error, timeout} - end. - -get_progs() -> - case os:type() of - {unix,freebsd} -> - {error,"cant use run_erl on freebsd"}; - {unix,openbsd} -> - {error,"cant use run_erl on openbsd"}; - {unix,_} -> - case os:find_executable("run_erl") of - RE when is_list(RE) -> - case os:find_executable("to_erl") of - TE when is_list(TE) -> - case os:find_executable("erl") of - E when is_list(E) -> - {RE,TE,E}; - _ -> - {error, "Could not find erl command"} - end; - _ -> - {error, "Could not find to_erl command"} - end; - _ -> - {error, "Could not find run_erl command"} - end; - _ -> - {error, "Not a unix OS"} - end. - -create_tempdir() -> - create_tempdir(filename:join(["/tmp","rtnode"++os:getpid()]),$A). - -create_tempdir(Dir,X) when X > $Z, X < $a -> - create_tempdir(Dir,$a); -create_tempdir(Dir,X) when X > $z -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason eexist", - [Dir++[$z]])), - {error, Estr}; -create_tempdir(Dir0, Ch) -> - %% Expect fairly standard unix. - Dir = Dir0++[Ch], - case file:make_dir(Dir) of - {error, eexist} -> - create_tempdir(Dir0, Ch+1); - {error, Reason} -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason ~p", - [Dir,Reason])), - {error,Estr}; - ok -> - Dir - end. - -create_nodename() -> - create_nodename($A). - -create_nodename(X) when X > $Z, X < $a -> - create_nodename($a); -create_nodename(X) when X > $z -> - {error,out_of_nodenames}; -create_nodename(X) -> - NN = "rtnode"++os:getpid()++[X], - case file:read_file_info(filename:join(["/tmp",NN])) of - {error,enoent} -> - Host = lists:nth(2,string:tokens(atom_to_list(node()),"@")), - {ok,NN++"@"++Host}; - _ -> - create_nodename(X+1) - end. - - -start_runerl_node(RunErl,Erl,Tempdir,Nodename,Extra) -> - XArg = case Nodename of - [] -> - []; - _ -> - " -sname "++(if is_atom(Nodename) -> atom_to_list(Nodename); - true -> Nodename - end)++ - " -setcookie "++atom_to_list(erlang:get_cookie()) - end, - XXArg = case Extra of - [] -> - []; - _ -> - " "++Extra - end, - spawn(fun() -> - ?dbg("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++ - " \""++Erl++XArg++XXArg++"\""), - os:cmd("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++ - " \""++Erl++XArg++XXArg++"\"") - end). - -start_toerl_server(ToErl,Tempdir) -> - Pid = spawn(?MODULE,toerl_server,[self(),ToErl,Tempdir]), - receive - {Pid,started} -> - Pid; - {Pid,error,Reason} -> - {error,Reason} - end. - -try_to_erl(_Command, 0) -> - {error, cannot_to_erl}; -try_to_erl(Command, N) -> - ?dbg({?LINE,N}), - Port = open_port({spawn, Command},[eof,{line,1000}]), - Timeout = timeout(normal) div 2, - receive - {Port, eof} -> - receive after Timeout -> - ok - end, - try_to_erl(Command, N-1) - after Timeout -> - ?dbg(Port), - Port - end. - -toerl_server(Parent,ToErl,Tempdir) -> - Port = try_to_erl("\""++ToErl++"\" "++Tempdir++"/ 2>/dev/null",8), - case Port of - P when is_port(P) -> - Parent ! {self(),started}; - {error,Other} -> - Parent ! {self(),error,Other}, - exit(Other) - end, - case toerl_loop(Port,[]) of - normal -> - ok; - {error, Reason} -> - error_logger:error_msg("toerl_server exit with reason ~p~n", - [Reason]), - exit(Reason) - end. - -toerl_loop(Port,Acc) -> - ?dbg({toerl_loop, Port, Acc}), - receive - {Port,{data,{Tag0,Data}}} when is_port(Port) -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - case Acc of - [{noeol,Data0}|T0] -> - toerl_loop(Port,[{Tag0, Data0++Data}|T0]); - _ -> - toerl_loop(Port,[{Tag0,Data}|Acc]) - end; - {Pid,{get_line,Timeout}} -> - case Acc of - [] -> - case get_data_within(Port,Timeout,[]) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[]); - {noeol,Data1} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data1}]); - {eol,Data2} -> - Pid ! {get_line, Data2}, - toerl_loop(Port,[]) - end; - [{noeol,Data3}] -> - case get_data_within(Port,Timeout,Data3) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,Acc); - {noeol,Data4} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data4}]); - {eol,Data5} -> - Pid ! {get_line, Data5}, - toerl_loop(Port,[]) - end; - List -> - {NewAcc,[{eol,Data6}]} = lists:split(length(List)-1,List), - Pid ! {get_line,Data6}, - toerl_loop(Port,NewAcc) - end; - {Pid, {send_line, Data7}} -> - Port ! {self(),{command, Data7++"\n"}}, - Pid ! {send_line, ok}, - toerl_loop(Port,Acc); - {_Pid, kill_emulator} -> - Port ! {self(),{command, "init:stop().\n"}}, - Timeout1 = timeout(long), - receive - {Port,eof} -> - normal - after Timeout1 -> - {error, kill_timeout} - end; - {Port, eof} -> - {error, unexpected_eof}; - Other -> - {error, {unexpected, Other}} - end. - -millistamp() -> - erlang:monotonic_time(millisecond). - -get_data_within(Port, X, Acc) when X =< 0 -> - ?dbg({get_data_within, X, Acc, ?LINE}), - receive - {Port,{data,{Tag0,Data}}} -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - {Tag0, Acc++Data} - after 0 -> - case Acc of - [] -> - timeout; - Noeol -> - {noeol,Noeol} - end - end; - - -get_data_within(Port, Timeout, Acc) -> - ?dbg({get_data_within, Timeout, Acc, ?LINE}), - T1 = millistamp(), - receive - {Port,{data,{noeol,Data}}} -> - ?dbg({?LINE,Port,{data,{noeol,Data}}}), - Elapsed = millistamp() - T1 + 1, - get_data_within(Port, Timeout - Elapsed, Acc ++ Data); - {Port,{data,{eol,Data1}}} -> - ?dbg({?LINE,Port,{data,{eol,Data1}}}), - {eol, Acc ++ Data1} - after Timeout -> - timeout - end. - -get_default_shell() -> - Match = fun(Data) -> - case lists:prefix("undefined", Data) of - true -> - yes; - false -> - case re:run(Data, "<\\d+[.]\\d+[.]\\d+>", - [{capture,none}]) of - match -> no; - _ -> 'maybe' - end - end - end, - try - rtnode([{putline,""}, - {putline, "whereis(user_drv)."}, - {getline_pred, Match, "matching of user_drv pid"}], []), - old - catch _E:_R -> - ?dbg({_E,_R}), - new - end. - %% %% Test I/O-server %% diff --git a/lib/stdlib/test/lists_property_test_SUITE.erl b/lib/stdlib/test/lists_property_test_SUITE.erl new file mode 100644 index 0000000000..fee088a396 --- /dev/null +++ b/lib/stdlib/test/lists_property_test_SUITE.erl @@ -0,0 +1,391 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(lists_property_test_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-compile(export_all). + +all() -> + [ + all_true_case, all_false_case, + any_true_case, any_false_case, + append_1_case, + append_2_case, + concat_case, + delete_case, delete_absent_case, + droplast_case, + dropwhile_case, + duplicate_case, + enumerate_1_case, + enumerate_2_case, + filter_case, + filtermap_case, + flatlength_case, + flatmap_case, + flatten_1_case, + flatten_2_case, + foldl_case, + foldr_case, + foreach_case, + join_case, + keydelete_case, keydelete_absent_case, + keyfind_case, keyfind_absent_case, + keymap_case, + keymember_case, keymember_absent_case, + keymerge_case, + keyreplace_case, keyreplace_absent_case, + keysearch_case, keysearch_absent_case, + keysort_case, + keystore_case, keystore_absent_case, + keytake_case, keytake_absent_case, + last_case, + map_case, + mapfoldl_case, + mapfoldr_case, + max_case, + member_case, member_absent_case, + merge_1_case, + merge_2_case, + merge_3_case, + merge3_case, + min_case, + nth_case, nth_outofrange_case, + nthtail_case, nthtail_outofrange_case, + partition_case, + prefix_case, + reverse_1_case, + reverse_2_case, + search_case, search_absent_case, + seq2_case, + seq3_case, + sort_1_case, + sort_2_case, + split_case, split_outofrange_case, + splitwith_case, + sublist_2_case, + sublist_3_case, + subtract_case, + suffix_case, + sum_case, + takewhile_case, + ukeymerge_case, + ukeysort_case, + umerge_1_case, + umerge_2_case, + umerge_3_case, + umerge3_case, + uniq_1_case, + uniq_2_case, + unzip_case, + unzip3_case, + usort_1_case, + usort_2_case, + zip_case, + zip3_case, + zipwith_case, + zipwith3_case + ]. + +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + persistent_term:erase({lists_prop, random_atoms}), + Config. + +do_proptest(Prop, Config) -> + ct_property_test:quickcheck(lists_prop:Prop(), Config). + +all_true_case(Config) -> + do_proptest(prop_all_true, Config). + +all_false_case(Config) -> + do_proptest(prop_all_false, Config). + +any_true_case(Config) -> + do_proptest(prop_any_true, Config). + +any_false_case(Config) -> + do_proptest(prop_any_false, Config). + +append_1_case(Config) -> + do_proptest(prop_append_1, Config). + +append_2_case(Config) -> + do_proptest(prop_append_2, Config). + +concat_case(Config) -> + do_proptest(prop_concat, Config). + +delete_case(Config) -> + do_proptest(prop_delete, Config). + +delete_absent_case(Config) -> + do_proptest(prop_delete_absent, Config). + +droplast_case(Config) -> + do_proptest(prop_droplast, Config). + +dropwhile_case(Config) -> + do_proptest(prop_dropwhile, Config). + +duplicate_case(Config) -> + do_proptest(prop_duplicate, Config). + +enumerate_1_case(Config) -> + do_proptest(prop_enumerate_1, Config). + +enumerate_2_case(Config) -> + do_proptest(prop_enumerate_2, Config). + +filter_case(Config) -> + do_proptest(prop_filter, Config). + +filtermap_case(Config) -> + do_proptest(prop_filtermap, Config). + +flatlength_case(Config) -> + do_proptest(prop_flatlength, Config). + +flatmap_case(Config) -> + do_proptest(prop_flatmap, Config). + +flatten_1_case(Config) -> + do_proptest(prop_flatten_1, Config). + +flatten_2_case(Config) -> + do_proptest(prop_flatten_2, Config). + +foldl_case(Config) -> + do_proptest(prop_foldl, Config). + +foldr_case(Config) -> + do_proptest(prop_foldr, Config). + +foreach_case(Config) -> + do_proptest(prop_foreach, Config). + +join_case(Config) -> + do_proptest(prop_join, Config). + +keydelete_case(Config) -> + do_proptest(prop_keydelete, Config). + +keydelete_absent_case(Config) -> + do_proptest(prop_keydelete_absent, Config). + +keyfind_case(Config) -> + do_proptest(prop_keyfind, Config). + +keyfind_absent_case(Config) -> + do_proptest(prop_keyfind_absent, Config). + +keymap_case(Config) -> + do_proptest(prop_keymap, Config). + +keymember_case(Config) -> + do_proptest(prop_keymember, Config). + +keymember_absent_case(Config) -> + do_proptest(prop_keymember_absent, Config). + +keymerge_case(Config) -> + do_proptest(prop_keymerge, Config). + +keyreplace_case(Config) -> + do_proptest(prop_keyreplace, Config). + +keyreplace_absent_case(Config) -> + do_proptest(prop_keyreplace_absent, Config). + +keysearch_case(Config) -> + do_proptest(prop_keysearch, Config). + +keysearch_absent_case(Config) -> + do_proptest(prop_keysearch_absent, Config). + +keysort_case(Config) -> + do_proptest(prop_keysort, Config). + +keystore_case(Config) -> + do_proptest(prop_keystore, Config). + +keystore_absent_case(Config) -> + do_proptest(prop_keystore_absent, Config). + +keytake_case(Config) -> + do_proptest(prop_keytake, Config). + +keytake_absent_case(Config) -> + do_proptest(prop_keytake_absent, Config). + +last_case(Config) -> + do_proptest(prop_last, Config). + +map_case(Config) -> + do_proptest(prop_map, Config). + +mapfoldl_case(Config) -> + do_proptest(prop_mapfoldl, Config). + +mapfoldr_case(Config) -> + do_proptest(prop_mapfoldr, Config). + +max_case(Config) -> + do_proptest(prop_max, Config). + +member_case(Config) -> + do_proptest(prop_member, Config). + +member_absent_case(Config) -> + do_proptest(prop_member_absent, Config). + +merge_1_case(Config) -> + do_proptest(prop_merge_1, Config). + +merge_2_case(Config) -> + do_proptest(prop_merge_2, Config). + +merge_3_case(Config) -> + do_proptest(prop_merge_3, Config). + +merge3_case(Config) -> + do_proptest(prop_merge3, Config). + +min_case(Config) -> + do_proptest(prop_min, Config). + +nth_case(Config) -> + do_proptest(prop_nth, Config). + +nth_outofrange_case(Config) -> + do_proptest(prop_nth_outofrange, Config). + +nthtail_case(Config) -> + do_proptest(prop_nthtail, Config). + +nthtail_outofrange_case(Config) -> + do_proptest(prop_nthtail_outofrange, Config). + +partition_case(Config) -> + do_proptest(prop_partition, Config). + +prefix_case(Config) -> + do_proptest(prop_prefix, Config). + +reverse_1_case(Config) -> + do_proptest(prop_reverse_1, Config). + +reverse_2_case(Config) -> + do_proptest(prop_reverse_2, Config). + +search_case(Config) -> + do_proptest(prop_search, Config). + +search_absent_case(Config) -> + do_proptest(prop_search_absent, Config). + +seq2_case(Config) -> + do_proptest(prop_seq2, Config). + +seq3_case(Config) -> + do_proptest(prop_seq3, Config). + +sort_1_case(Config) -> + do_proptest(prop_sort_1, Config). + +sort_2_case(Config) -> + do_proptest(prop_sort_2, Config). + +split_case(Config) -> + do_proptest(prop_split, Config). + +split_outofrange_case(Config) -> + do_proptest(prop_split_outofrange, Config). + +splitwith_case(Config) -> + do_proptest(prop_splitwith, Config). + +sublist_2_case(Config) -> + do_proptest(prop_sublist_2, Config). + +sublist_3_case(Config) -> + do_proptest(prop_sublist_3, Config). + +subtract_case(Config) -> + do_proptest(prop_subtract, Config). + +suffix_case(Config) -> + do_proptest(prop_suffix, Config). + +sum_case(Config) -> + do_proptest(prop_sum, Config). + +takewhile_case(Config) -> + do_proptest(prop_takewhile, Config). + +ukeymerge_case(Config) -> + do_proptest(prop_ukeymerge, Config). + +ukeysort_case(Config) -> + do_proptest(prop_ukeysort, Config). + +umerge_1_case(Config) -> + do_proptest(prop_umerge_1, Config). + +umerge_2_case(Config) -> + do_proptest(prop_umerge_2, Config). + +umerge_3_case(Config) -> + do_proptest(prop_umerge_3, Config). + +umerge3_case(Config) -> + do_proptest(prop_umerge3, Config). + +uniq_1_case(Config) -> + do_proptest(prop_uniq_1, Config). + +uniq_2_case(Config) -> + do_proptest(prop_uniq_2, Config). + +unzip_case(Config) -> + do_proptest(prop_unzip, Config). + +unzip3_case(Config) -> + do_proptest(prop_unzip3, Config). + +usort_1_case(Config) -> + do_proptest(prop_usort_1, Config). + +usort_2_case(Config) -> + do_proptest(prop_usort_2, Config). + +zip_case(Config) -> + do_proptest(prop_zip, Config). + +zip3_case(Config) -> + do_proptest(prop_zip3, Config). + +zipwith_case(Config) -> + do_proptest(prop_zipwith, Config). + +zipwith3_case(Config) -> + do_proptest(prop_zipwith3, Config). + diff --git a/lib/stdlib/test/peer_SUITE.erl b/lib/stdlib/test/peer_SUITE.erl index e5372416ee..a23be065b3 100644 --- a/lib/stdlib/test/peer_SUITE.erl +++ b/lib/stdlib/test/peer_SUITE.erl @@ -13,6 +13,8 @@ suite/0, all/0, groups/0, + init_per_suite/1, + end_per_suite/1, init_per_group/2, end_per_group/2 ]). @@ -46,6 +48,9 @@ old_release/0, old_release/1, ssh/0, ssh/1, docker/0, docker/1, + post_process_args/0, post_process_args/1, + attached/0, attached/1, + attached_cntrl_channel_handler_crash/0, attached_cntrl_channel_handler_crash/1, cntrl_channel_handler_crash/0, cntrl_channel_handler_crash/1, cntrl_channel_handler_crash_old_release/0, cntrl_channel_handler_crash_old_release/1 ]). @@ -53,18 +58,29 @@ suite() -> [{timetrap, {minutes, 1}}]. +init_per_suite(Config) -> + %% Restrict number of schedulers so that we do not run out of + %% file descriptors during test + os:putenv("ERL_FLAGS","+S1 +SDio 1"), + Config. + +end_per_suite(_Config) -> + os:unsetenv("ERL_FLAGS"), + ok. + shutdown_alternatives() -> [shutdown_halt, shutdown_halt_timeout, shutdown_stop, shutdown_stop_timeout, shutdown_close]. alternative() -> [basic, peer_states, cast, detached, dyn_peer, stop_peer, - io_redirect, multi_node, duplicate_name, cntrl_channel_handler_crash, - cntrl_channel_handler_crash_old_release | shutdown_alternatives()]. + io_redirect, multi_node, duplicate_name, attached, attached_cntrl_channel_handler_crash, + cntrl_channel_handler_crash, cntrl_channel_handler_crash_old_release | shutdown_alternatives()]. groups() -> [ {dist, [parallel], [errors, dist, peer_down_crash, peer_down_continue, peer_down_boot, - dist_up_down, dist_localhost, cntrl_channel_handler_crash, + dist_up_down, dist_localhost, post_process_args, attached, + attached_cntrl_channel_handler_crash, cntrl_channel_handler_crash, cntrl_channel_handler_crash_old_release | shutdown_alternatives()]}, {dist_seq, [], [dist_io_redirect, %% Cannot be run in parallel in dist group peer_down_crash_tcp]}, @@ -80,7 +96,7 @@ all() -> init_per_group(remote, Config) -> %% check that SSH can connect to localhost, skip the test if not - try os:cmd("ssh localhost echo ok") of + try os:cmd("timeout 10s ssh localhost echo ok") of "ok\n" -> Config; _ -> {skip, "'ssh localhost echo ok' did not return ok"} catch @@ -487,6 +503,29 @@ duplicate_name(Config) when is_list(Config) -> ?assertException(exit, _, peer:start_link(#{connection => standard_io, name => ?FUNCTION_NAME})), peer:stop(Peer). +post_process_args() -> + [{doc, "Test that the post_process_args option works"}]. + +post_process_args(Config) when is_list(Config) -> + case {os:type(),os:find_executable("bash")} of + {{win32,_}, _Bash} -> + {skip,"Test does not work on windows"}; + {_, false} -> + {skip,"Test needs bash to run"}; + {_, Bash} -> + Erl = string:split(ct:get_progname()," ",all), + [throw({skip, "Needs bash to run"}) || Bash =:= false], + {ok, Peer, _Node} = + peer:start_link( + #{ name => ?CT_PEER_NAME(), + exec => {Bash,["-c"|Erl]}, + post_process_args => + fun(["-c"|Args]) -> + ["-c", lists:flatten(lists:join($\s, Args))] + end }), + peer:stop(Peer) + end. + %% ------------------------------------------------------------------- %% Compatibility: old releases old_release() -> @@ -580,6 +619,54 @@ docker(Config) when is_list(Config) -> peer:stop(Peer) end. +attached() -> + [{doc, "Test that it is possible to start a peer node using run_erl aka attached"}]. + +attached(Config) -> + attached_test(false, Config). + +attached_cntrl_channel_handler_crash() -> + [{doc, "Test that peer node is halted if peer control channel handler process crashes and peer node is attached"}]. + +attached_cntrl_channel_handler_crash(Config) -> + attached_test(true, Config). + +attached_test(CrashConnectionHandler, Config) -> + RunErl = os:find_executable("run_erl"), + [throw({skip, "Could not find run_erl"}) || RunErl =:= false], + Erl = string:split(ct:get_progname()," ",all), + RunErlDir = filename:join(proplists:get_value(priv_dir, Config),?FUNCTION_NAME), + filelib:ensure_path(RunErlDir), + Connection = proplists:get_value(connection, Config), + Conn = if Connection =:= undefined -> #{ name => ?CT_PEER_NAME() }; + true -> + case CrashConnectionHandler of + false -> + #{ connection => Connection }; + true -> + #{name => ?CT_PEER_NAME(), + connection => Connection} + end + end, + try peer:start( + Conn#{ + exec => {RunErl, Erl}, + detached => false, + post_process_args => + fun(Args) -> + [RunErlDir ++ "/", RunErlDir, + lists:flatten(lists:join(" ",[[$',A,$'] || A <- Args]))] + end + }) of + {ok, Peer, Node} when Connection =:= undefined; Connection =:= 0 -> + case CrashConnectionHandler of + false -> peer:stop(Peer); + true -> cntrl_channel_handler_crash_test(Node) + end + catch error:{detached,_} when Connection =:= standard_io -> + ok + end. + cntrl_channel_handler_crash() -> [{doc, "Test that peer node is halted if peer control channel handler process crashes"}]. diff --git a/lib/stdlib/test/property_test/base64_prop.erl b/lib/stdlib/test/property_test/base64_prop.erl index a2e3026dcc..f77ce9b664 100644 --- a/lib/stdlib/test/property_test/base64_prop.erl +++ b/lib/stdlib/test/property_test/base64_prop.erl @@ -54,96 +54,200 @@ %%% Properties %%% %%%%%%%%%%%%%%%%%% -prop_encode() -> +prop_encode_1() -> ?FORALL( Str, oneof([list(byte()), binary()]), begin Enc = base64:encode(Str), Dec = base64:decode(Enc), - is_b64_binary(Enc) andalso str_equals(Str, Dec) + is_b64_binary(standard, Enc) andalso str_equals(Str, Dec) end ). -prop_encode_to_string() -> +prop_encode_2() -> + ?FORALL( + {Str, Mode}, + {oneof([list(byte()), binary()]), mode()}, + begin + Enc = base64:encode(Str, Mode), + Dec = base64:decode(Enc, Mode), + is_b64_binary(Mode, Enc) andalso str_equals(Str, Dec) + end + ). + +prop_encode_to_string_1() -> ?FORALL( Str, oneof([list(byte()), binary()]), begin Enc = base64:encode_to_string(Str), Dec = base64:decode_to_string(Enc), - is_b64_string(Enc) andalso str_equals(Str, Dec) + is_b64_string(standard, Enc) andalso str_equals(Str, Dec) + end + ). + +prop_encode_to_string_2() -> + ?FORALL( + {Str, Mode}, + {oneof([list(byte()), binary()]), mode()}, + begin + Enc = base64:encode_to_string(Str, Mode), + Dec = base64:decode_to_string(Enc, Mode), + is_b64_string(Mode, Enc) andalso str_equals(Str, Dec) end ). -prop_decode() -> +prop_decode_1() -> ?FORALL( {NormalizedB64, WspedB64}, - wsped_b64(), + wsped_b64(standard), begin Dec = base64:decode(WspedB64), Enc = base64:encode(Dec), - is_binary(Dec) andalso b64_equals(NormalizedB64, Enc) + is_binary(Dec) andalso b64_equals(standard, NormalizedB64, Enc) + end + ). + +prop_decode_2() -> + ?FORALL( + {{NormalizedB64, WspedB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:decode(WspedB64, Mode), + Enc = base64:encode(Dec, Mode), + is_binary(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) end ). -prop_decode_malformed() -> - common_decode_malformed(wsped_b64(), fun base64:decode/1). +prop_decode_1_malformed() -> + common_decode_malformed(fun wsped_b64/1, standard, fun(Data, _) -> base64:decode(Data) end). + +prop_decode_2_malformed() -> + common_decode_malformed(fun wsped_b64/1, mode(), fun base64:decode/2). -prop_decode_noisy() -> - common_decode_noisy(fun base64:decode/1). +prop_decode_1_noisy() -> + common_decode_noisy(standard, fun(Data, _) -> base64:decode(Data) end). -prop_decode_to_string() -> +prop_decode_2_noisy() -> + common_decode_noisy(mode(), fun base64:decode/2). + +prop_decode_to_string_1() -> ?FORALL( {NormalizedB64, WspedB64}, - wsped_b64(), + wsped_b64(standard), begin Dec = base64:decode_to_string(WspedB64), Enc = base64:encode(Dec), - is_bytelist(Dec) andalso b64_equals(NormalizedB64, Enc) + is_bytelist(Dec) andalso b64_equals(standard, NormalizedB64, Enc) + end + ). + +prop_decode_to_string_2() -> + ?FORALL( + {{NormalizedB64, WspedB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:decode_to_string(WspedB64, Mode), + Enc = base64:encode(Dec, Mode), + is_bytelist(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) end ). -prop_decode_to_string_malformed() -> - common_decode_malformed(wsped_b64(), fun base64:decode_to_string/1). +prop_decode_to_string_1_malformed() -> + common_decode_malformed(fun wsped_b64/1, standard, fun(Data, _) -> base64:decode_to_string(Data) end). + +prop_decode_to_string_2_malformed() -> + common_decode_malformed(fun wsped_b64/1, mode(), fun base64:decode_to_string/2). + +prop_decode_to_string_1_noisy() -> + common_decode_noisy(standard, fun(Data, _) -> base64:decode_to_string(Data) end). -prop_decode_to_string_noisy() -> - common_decode_noisy(fun base64:decode_to_string/1). +prop_decode_to_string_2_noisy() -> + common_decode_noisy(mode(), fun base64:decode_to_string/2). -prop_mime_decode() -> +prop_mime_decode_1() -> ?FORALL( {NormalizedB64, NoisyB64}, - noisy_b64(), + noisy_b64(standard), begin Dec = base64:mime_decode(NoisyB64), Enc = base64:encode(Dec), - is_binary(Dec) andalso b64_equals(NormalizedB64, Enc) + is_binary(Dec) andalso b64_equals(standard, NormalizedB64, Enc) end ). -prop_mime_decode_malformed() -> - common_decode_malformed(noisy_b64(), fun base64:mime_decode/1). +prop_mime_decode_2() -> + ?FORALL( + {{NormalizedB64, NoisyB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:mime_decode(NoisyB64, Mode), + Enc = base64:encode(Dec, Mode), + is_binary(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) + end + ). + +prop_mime_decode_1_malformed() -> + common_decode_malformed(fun noisy_b64/1, standard, fun(Data, _) -> base64:mime_decode(Data) end). -prop_mime_decode_to_string() -> +prop_mime_decode_2_malformed() -> + common_decode_malformed(fun noisy_b64/1, mode(), fun base64:mime_decode/2). + +prop_mime_decode_to_string_1() -> ?FORALL( {NormalizedB64, NoisyB64}, - noisy_b64(), + noisy_b64(standard), begin Dec = base64:mime_decode_to_string(NoisyB64), Enc = base64:encode(Dec), - is_bytelist(Dec) andalso b64_equals(NormalizedB64, Enc) + is_bytelist(Dec) andalso b64_equals(standard, NormalizedB64, Enc) end ). -prop_mime_decode_to_string_malformed() -> - common_decode_malformed(noisy_b64(), fun base64:mime_decode_to_string/1). +prop_mime_decode_to_string_2() -> + ?FORALL( + {{NormalizedB64, NoisyB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:mime_decode_to_string(NoisyB64, Mode), + Enc = base64:encode(Dec, Mode), + is_bytelist(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) + end + ). -common_decode_noisy(Fn) -> +prop_mime_decode_to_string_1_malformed() -> + common_decode_malformed(fun noisy_b64/1, standard, fun(Data, _) -> base64:mime_decode_to_string(Data) end). + +prop_mime_decode_to_string_2_malformed() -> + common_decode_malformed(fun noisy_b64/1, mode(), fun base64:mime_decode_to_string/2). + +common_decode_noisy(ModeGen, Fn) -> ?FORALL( - {_, NoisyB64}, - ?SUCHTHAT({NormalizedB64, NoisyB64}, noisy_b64(), NormalizedB64 =/= NoisyB64), + {{_, NoisyB64}, Mode}, + ?LET( + Mode, + ModeGen, + {?SUCHTHAT({NormalizedB64, NoisyB64}, noisy_b64(Mode), NormalizedB64 =/= NoisyB64), Mode} + ), try - Fn(NoisyB64) + Fn(NoisyB64, Mode) of _ -> false @@ -153,25 +257,30 @@ common_decode_noisy(Fn) -> end ). -common_decode_malformed(Gen, Fn) -> +common_decode_malformed(DataGen, ModeGen, Fn) -> ?FORALL( - MalformedB64, + {MalformedB64, Mode}, ?LET( - {{NormalizedB64, NoisyB64}, Malformings}, - { - Gen, - oneof( - [ - [b64_char()], - [b64_char(), b64_char()], - [b64_char(), b64_char(), b64_char()] - ] - ) - }, - {NormalizedB64, insert_noise(NoisyB64, Malformings)} + Mode, + ModeGen, + ?LET( + {{NormalizedB64, NoisyB64}, Malformings, InsertFn}, + { + DataGen(Mode), + oneof( + [ + [b64_char(Mode)], + [b64_char(Mode), b64_char(Mode)], + [b64_char(Mode), b64_char(Mode), b64_char(Mode)] + ] + ), + function1(boolean()) + }, + {{NormalizedB64, insert_noise(NoisyB64, Malformings, InsertFn)}, Mode} + ) ), try - Fn(MalformedB64) + Fn(MalformedB64, Mode) of _ -> false @@ -185,16 +294,20 @@ common_decode_malformed(Gen, Fn) -> %%% Generators %%% %%%%%%%%%%%%%%%%%% +%% Generate base64 encoding mode. +mode() -> + oneof([standard, urlsafe]). + %% Generate a single character from the base64 alphabet. -b64_char() -> - oneof(b64_chars()). +b64_char(Mode) -> + oneof(b64_chars(Mode)). %% Generate a string of characters from the base64 alphabet, %% including padding if needed. -b64_string() -> +b64_string(Mode) -> ?LET( {L, Filler}, - {list(b64_char()), b64_char()}, + {list(b64_char(Mode)), b64_char(Mode)}, case length(L) rem 4 of 0 -> L; 1 -> L ++ [Filler, $=, $=]; @@ -205,43 +318,43 @@ b64_string() -> %% Generate a binary of characters from the base64 alphabet, %% including padding if needed. -b64_binary() -> +b64_binary(Mode) -> ?LET( L, - b64_string(), + b64_string(Mode), list_to_binary(L) ). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed. -b64() -> - oneof([b64_string(), b64_binary()]). +b64(Mode) -> + oneof([b64_string(Mode), b64_binary(Mode)]). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed, with %% whitespaces inserted at random indexes. -wsped_b64() -> +wsped_b64(Mode) -> ?LET( - {B64, Wsps}, - {b64(), list(oneof([$\t, $\r, $\n, $\s]))}, - {B64, insert_noise(B64, Wsps)} + {B64, Wsps, InsertFn}, + {b64(Mode), list(oneof([$\t, $\r, $\n, $\s])), function1(boolean())}, + {B64, insert_noise(B64, Wsps, InsertFn)} ). %% Generate a single character outside of the base64 alphabet. %% As whitespaces are allowed but ignored in base64, this generator %% will produce no whitespaces, either. -non_b64_char() -> - oneof(lists:seq(16#00, 16#FF) -- b64_allowed_chars()). +non_b64_char(Mode) -> + oneof(lists:seq(16#00, 16#FF) -- b64_allowed_chars(Mode)). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed, with %% whitespaces and non-base64 ("invalid") characters %% inserted at random indexes. -noisy_b64() -> +noisy_b64(Mode) -> ?LET( - {{B64, WspedB64}, Noise}, - {wsped_b64(), non_empty(list(non_b64_char()))}, - {B64, insert_noise(WspedB64, Noise)} + {{B64, WspedB64}, Noise, InsertFn}, + {wsped_b64(Mode), non_empty(list(non_b64_char(Mode))), function1(boolean())}, + {B64, insert_noise(WspedB64, Noise, InsertFn)} ). %%%%%%%%%%%%%%% @@ -252,81 +365,92 @@ noisy_b64() -> %% "=" is not included, as it is special in that it %% may only appear at the end of a base64 encoded string %% for padding. -b64_chars() -> +b64_chars_common() -> lists:seq($0, $9) ++ lists:seq($a, $z) ++ - lists:seq($A, $Z) ++ - [$+, $/]. + lists:seq($A, $Z). + +b64_chars(standard) -> + b64_chars_common() ++ [$+, $/]; +b64_chars(urlsafe) -> + b64_chars_common() ++ [$-, $_]. %% In addition to the above, the whitespace characters %% HTAB, CR, LF and SP are allowed to appear in a base64 %% encoded string and should be ignored. -b64_allowed_chars() -> - [$\t, $\r, $\n, $\s | b64_chars()]. +b64_allowed_chars(Mode) -> + [$\t, $\r, $\n, $\s | b64_chars(Mode)]. %% Insert the given list of noise characters at random %% places into the given base64 string. -insert_noise(B64, []) -> +insert_noise(B64, Noise, InsertFn) -> + insert_noise(B64, Noise, InsertFn, 0). + +insert_noise(B64, [], _, _) -> B64; -insert_noise([], Noise) -> +insert_noise([], Noise, _, _) -> Noise; -insert_noise(<<>>, Noise) -> +insert_noise(<<>>, Noise, _, _) -> list_to_binary(Noise); -insert_noise([B|Bs] = B64, [N|Ns] = Noise) -> - case rand:uniform(2) of - 1 -> - [B|insert_noise(Bs, Noise)]; - 2 -> - [N|insert_noise(B64, Ns)] +insert_noise([B|Bs] = B64, [N|Ns] = Noise, InsertFn, Idx) -> + case InsertFn(Idx) of + true -> + [B|insert_noise(Bs, Noise, InsertFn, Idx + 1)]; + false -> + [N|insert_noise(B64, Ns, InsertFn, Idx + 1)] end; -insert_noise(<<B, Bs/binary>> = B64, [N|Ns] = Noise) -> - case rand:uniform(2) of - 1 -> - <<B, (insert_noise(Bs, Noise))/binary>>; - 2 -> - <<N, (insert_noise(B64, Ns))/binary>> +insert_noise(<<B, Bs/binary>> = B64, [N|Ns] = Noise, InsertFn, Idx) -> + case InsertFn(Idx) of + true -> + <<B, (insert_noise(Bs, Noise, InsertFn, Idx + 1))/binary>>; + false -> + <<N, (insert_noise(B64, Ns, InsertFn, Idx + 1))/binary>> end. %% Check if the given character is in the base64 alphabet. %% This does not include the padding character "=". -is_b64_char($+) -> +is_b64_char(standard, $+) -> + true; +is_b64_char(standard, $/) -> + true; +is_b64_char(urlsafe, $-) -> true; -is_b64_char($/) -> +is_b64_char(urlsafe, $_) -> true; -is_b64_char(C) when C >= $0, C =< $9 -> +is_b64_char(_, C) when C >= $0, C =< $9 -> true; -is_b64_char(C) when C >= $A, C =< $Z -> +is_b64_char(_, C) when C >= $A, C =< $Z -> true; -is_b64_char(C) when C >= $a, C =< $z -> +is_b64_char(_, C) when C >= $a, C =< $z -> true; -is_b64_char(_) -> +is_b64_char(_, _) -> false. %% Check if the given argument is a base64 binary, %% ie that it consists of quadruplets of characters %% from the base64 alphabet, whereas the last quadruplet %% may be padded with one or two "="s -is_b64_binary(B) -> - is_b64_binary(B, 0). +is_b64_binary(Mode, B) -> + is_b64_binary(Mode, B, 0). -is_b64_binary(<<>>, N) -> +is_b64_binary(_, <<>>, N) -> N rem 4 =:= 0; -is_b64_binary(<<$=>>, N) -> +is_b64_binary(_, <<$=>>, N) -> N rem 4 =:= 3; -is_b64_binary(<<$=, $=>>, N) -> +is_b64_binary(_, <<$=, $=>>, N) -> N rem 4 =:= 2; -is_b64_binary(<<C, More/binary>>, N) -> - case is_b64_char(C) of +is_b64_binary(Mode, <<C, More/binary>>, N) -> + case is_b64_char(Mode, C) of true -> - is_b64_binary(More, N + 1); + is_b64_binary(Mode, More, N + 1); false -> false end. %% Check if the given argument is a base64 string %% (see is_b64_binary/1) -is_b64_string(S) -> - is_b64_binary(list_to_binary(S)). +is_b64_string(Mode, S) -> + is_b64_binary(Mode, list_to_binary(S)). %% Check if the argument is a list of bytes. is_bytelist(L) -> @@ -349,23 +473,23 @@ str_equals(Str1, Str2) when is_binary(Str1), is_binary(Str2) -> %% Assumes that the given arguments are in a normalized form, %% ie that they consist only of characters from the base64 %% alphabet and possible padding ("="). -b64_equals(L, B) when is_list(L) -> - b64_equals(list_to_binary(L), B); -b64_equals(B, L) when is_list(L) -> - b64_equals(B, list_to_binary(L)); -b64_equals(B1, B2) when is_binary(B1), is_binary(B2) -> - b64_equals1(B1, B2). - -b64_equals1(<<Eq:4/bytes>>, <<Eq:4/bytes>>) -> - is_b64_binary(Eq); -b64_equals1(<<Eq:4/bytes, More1/binary>>, <<Eq:4/bytes, More2/binary>>) -> - case lists:all(fun is_b64_char/1, binary_to_list(Eq)) of +b64_equals(Mode, L, B) when is_list(L) -> + b64_equals(Mode, list_to_binary(L), B); +b64_equals(Mode, B, L) when is_list(L) -> + b64_equals(Mode, B, list_to_binary(L)); +b64_equals(Mode, B1, B2) when is_binary(B1), is_binary(B2) -> + b64_equals1(Mode, B1, B2). + +b64_equals1(Mode, <<Eq:4/bytes>>, <<Eq:4/bytes>>) -> + is_b64_binary(Mode, Eq); +b64_equals1(Mode, <<Eq:4/bytes, More1/binary>>, <<Eq:4/bytes, More2/binary>>) -> + case lists:all(fun(C) -> is_b64_char(Mode, C) end, binary_to_list(Eq)) of true -> - b64_equals1(More1, More2); + b64_equals1(Mode, More1, More2); false -> false end; -b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> +b64_equals1(Mode, <<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> %% If the encoded string ends with "==", there exist multiple %% possibilities for the character preceding the "==" as only the %% 3rd and 4th bits of the encoded byte represented by that @@ -374,7 +498,7 @@ b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> %% For example, all of the encoded strings "QQ==", "QR==", ..., "QZ==" %% decode to the string "A", since all the bytes represented by Q to Z %% are the same in the significant 3rd and 4th bit. - case is_b64_char(Eq) of + case is_b64_char(Mode, Eq) of true -> Normalize = fun (C) when C >= $A, C =< $P -> $A; @@ -383,20 +507,21 @@ b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> (C) when C >= $g, C =< $v -> $g; (C) when C >= $w, C =< $z -> $w; (C) when C >= $0, C =< $9 -> $w; - ($+) -> $w; - ($/) -> $w + ($+) when Mode =:= standard -> $w; + ($-) when Mode =:= urlsafe -> $w; + ($/) when Mode =:= standard -> $w; + ($_) when Mode =:= urlsafe -> $w end, Normalize(B1) =:= Normalize(B2); false -> false end; -b64_equals1(<<Eq:2/bytes, B1, $=>>, <<Eq:2/bytes, B2, $=>>) -> +b64_equals1(Mode, <<Eq1, Eq2, B1, $=>>, <<Eq1, Eq2, B2, $=>>) -> %% Similar to the above, but with the encoded string ending with a %% single "=" the 3rd to 6th bits of the encoded byte are significant, %% such that, for example, all the encoded strings "QUE=" to "QUH=" %% decode to the same string "AA". - <<Eq1, Eq2>> = Eq, - case is_b64_char(Eq1) andalso is_b64_char(Eq2) of + case is_b64_char(Mode, Eq1) andalso is_b64_char(Mode, Eq2) of true -> Normalize = fun (C) when C >= $A, C =< $D -> $A; @@ -416,14 +541,16 @@ b64_equals1(<<Eq:2/bytes, B1, $=>>, <<Eq:2/bytes, B2, $=>>) -> (C) when C >= $0, C =< $3 -> $0; (C) when C >= $4, C =< $7 -> $4; (C) when C >= $8, C =< $9 -> $8; - ($+) -> $8; - ($/) -> $8 + ($+) when Mode =:= standard -> $8; + ($-) when Mode =:= urlsafe -> $8; + ($/) when Mode =:= standard -> $8; + ($_) when Mode =:= urlsafe -> $8 end, Normalize(B1) =:= Normalize(B2); false -> false end; -b64_equals1(<<>>, <<>>) -> +b64_equals1(_, <<>>, <<>>) -> true; -b64_equals1(_, _) -> +b64_equals1(_, _, _) -> false. diff --git a/lib/stdlib/test/property_test/lists_prop.erl b/lib/stdlib/test/property_test/lists_prop.erl new file mode 100644 index 0000000000..6df00335c6 --- /dev/null +++ b/lib/stdlib/test/property_test/lists_prop.erl @@ -0,0 +1,1807 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(lists_prop). + +-compile([export_all, nowarn_export_all]). + +-proptest(eqc). +-proptest([triq, proper]). + +-ifndef(EQC). +-ifndef(PROPER). +-ifndef(TRIQ). +-define(EQC, true). +-endif. +-endif. +-endif. + +-ifdef(EQC). +-include_lib("eqc/include/eqc.hrl"). +-define(MOD_eqc,eqc). + +-else. +-ifdef(PROPER). +-include_lib("proper/include/proper.hrl"). +-define(MOD_eqc,proper). + +-else. +-ifdef(TRIQ). +-define(MOD_eqc,triq). +-include_lib("triq/include/triq.hrl"). + +-endif. +-endif. +-endif. + +-define(RANDOM_ATOMS, 1000). + +%%%%%%%%%%%%%%%%%% +%%% Properties %%% +%%%%%%%%%%%%%%%%%% + +%% all/2 +prop_all_true() -> + ?FORALL( + InList, + gen_list(), + lists:all(fun(_) -> true end, InList) + ). + +prop_all_false() -> + ?FORALL( + {InList, Elem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {F ++ [E|R], E} + ), + not lists:all(fun(T) -> T =/= Elem end, InList) + ). + +%% any/2 +prop_any_true() -> + ?FORALL( + {InList, Elem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {F ++ [E|R], E} + ), + lists:any(fun(T) -> T =:= Elem end, InList) + ). + +prop_any_false() -> + ?FORALL( + InList, + gen_list(), + not lists:any(fun(_) -> false end, InList) + ). + +%% append/1 +prop_append_1() -> + ?FORALL( + InLists, + list(gen_list()), + check_appended(InLists, lists:append(InLists)) + ). + +%% append/2 +prop_append_2() -> + ?FORALL( + {InList1, InList2}, + {gen_list(), gen_list()}, + lists:append(InList1, InList2) =:= InList1 ++ InList2 + ). + +%% concat/1 +prop_concat() -> + ?FORALL( + {InList, ExpString}, + gen_list_fold( + oneof([gen_atom(), number(), string()]), + fun + (A, Acc) when is_atom(A) -> Acc ++ atom_to_list(A); + (I, Acc) when is_integer(I) -> Acc ++ integer_to_list(I); + (F, Acc) when is_float(F) -> Acc ++ float_to_list(F); + (L, Acc) when is_list(L) -> Acc ++ L + end, + [] + ), + lists:concat(InList) =:= ExpString + ). + +%% delete/2 +prop_delete() -> + ?FORALL( + {InList, DelElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], E} + ), + begin + DeletedList = lists:delete(DelElem, InList), + length(DeletedList) =:= length(InList) - 1 andalso + check_deleted(DelElem, InList, DeletedList) + end + ). + +prop_delete_absent() -> + ?FORALL( + InList, + gen_list(), + lists:delete(make_ref(), InList) =:= InList + ). + +%% droplast/1 +prop_droplast() -> + ?FORALL( + InList, + gen_list(), + try + lists:droplast(InList) =:= lists:reverse(tl(lists:reverse(InList))) + catch + error:_ -> + InList =:= [] + end + ). + +%% dropwhile/2 +prop_dropwhile() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + Fn, + function1(boolean()), + ?LET( + {L, {_, DL}}, + gen_list_fold( + gen_any(), + fun(E, {Drop, Acc}) -> + case Drop andalso Fn(E) of + true -> {true, Acc}; + false -> {false, Acc ++ [E]} + end + end, + {true, []} + ), + {Fn, L, DL} + ) + ), + lists:dropwhile(Pred, InList) =:= ExpList + ). + +%% duplicate/2 +prop_duplicate() -> + ?FORALL( + {N, Term, ExpList}, + ?LET( + T, + gen_any(), + ?LET(L, list(T), {length(L), T, L}) + ), + lists:duplicate(N, Term) =:= ExpList + ). + +%% enumerate/1 +prop_enumerate_1() -> + ?FORALL( + {InList, ExpList}, + ?LET( + {L, {_, EL}}, + gen_list_fold( + gen_any(), + fun(T, {I, Acc}) -> + {I + 1, Acc ++ [{I, T}]} + end, + {1, []} + ), + {L, EL} + ), + lists:enumerate(InList) =:= ExpList + ). + +%% enumerate/2 +prop_enumerate_2() -> + ?FORALL( + {StartIndex, InList, ExpList}, + ?LET( + N, + integer(), + ?LET( + {L, {_, EL}}, + gen_list_fold( + gen_any(), + fun(T, {I, Acc}) -> + {I + 1, Acc ++ [{I, T}]} + end, + {N, []} + ), + {N, L, EL} + ) + ), + lists:enumerate(StartIndex, InList) =:= ExpList + ). + +%% filter/2 +prop_filter() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + P, + function1(boolean()), + ?LET( + {L, F}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + case P(T) of + true -> Acc ++ [T]; + false -> Acc + end + end, + [] + ), + {P, L, F} + ) + ), + lists:filter(Pred, InList) =:= ExpList + ). + +%% filtermap/2 +prop_filtermap() -> + ?FORALL( + {FilterMapFn, InList, ExpList}, + ?LET( + Fn, + function1(oneof([true, false, {true, gen_any()}])), + ?LET( + {L, FM}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + case Fn(T) of + false -> Acc; + true -> Acc ++ [T]; + {true, T1} -> Acc ++ [T1] + end + end, + [] + ), + {Fn, L, FM} + ) + ), + lists:filtermap(FilterMapFn, InList) =:= ExpList + ). + +%% flatlength/1 +prop_flatlength() -> + ?FORALL( + {DeepList, Len}, + gen_list_deepfold(fun(_, _, Cnt) -> Cnt + 1 end, 0), + lists:flatlength(DeepList) =:= Len + ). + +%% flatmap/2 +prop_flatmap() -> + ?FORALL( + {MapFn, InList, ExpList}, + ?LET( + Fn, + function1(gen_list()), + ?LET( + {L, FlatMapped}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + Acc ++ Fn(T) + end, + [] + ), + {Fn, L, FlatMapped} + ) + ), + lists:flatmap(MapFn, InList) =:= ExpList + ). + +%% flatten/1 +prop_flatten_1() -> + ?FORALL( + {DeepList, FlatList}, + gen_list_deepfold(fun(_, E, Acc) -> Acc ++ [E] end, []), + lists:flatten(DeepList) =:= FlatList + ). + +%% flatten/2 +prop_flatten_2() -> + ?FORALL( + {{DeepList, FlatList}, Tail}, + {gen_list_deepfold(fun(_, E, Acc) -> Acc ++ [E] end, []), gen_list()}, + lists:flatten(DeepList, Tail) =:= FlatList ++ Tail + ). + +%% foldl/3 +prop_foldl() -> + ?FORALL( + {FoldFn, InList, Acc0, Exp}, + ?LET( + {Fn, Acc0}, + {function2(gen_any()), gen_any()}, + ?LET( + {L, V}, + gen_list_fold(gen_any(), Fn, Acc0), + {Fn, L, Acc0, V} + ) + ), + lists:foldl(FoldFn, Acc0, InList) =:= Exp + ). + +%% foldr/3 +prop_foldr() -> + ?FORALL( + {FoldFn, InList, Acc0, Exp}, + ?LET( + {Fn, Acc0}, + {function2(gen_any()), gen_any()}, + ?LET( + {L, V}, + gen_list_fold(gen_any(), Fn, Acc0), + {Fn, lists:reverse(L), Acc0, V} + ) + ), + lists:foldr(FoldFn, Acc0, InList) =:= Exp + ). + +%% foreach/2 +prop_foreach() -> + ?FORALL( + InList, + gen_list(), + begin + Tag = make_ref(), + lists:foreach(fun(E) -> self() ! {Tag, E} end, InList), + [receive {Tag, T} -> T after 100 -> error(timeout) end || _ <- InList] =:= InList + end + ). + +%% join/2 +prop_join() -> + ?FORALL( + {Sep, InList}, + {gen_any(), gen_list()}, + check_joined(Sep, InList, lists:join(Sep, InList)) + ). + +%% keydelete/3 +prop_keydelete() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + DeletedL = lists:keydelete(Key, N, InList), + length(DeletedL) =:= length(InList) - 1 andalso + check_keydeleted(Key, N, InList, DeletedL) + end + ). + +prop_keydelete_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + lists:keydelete(make_ref(), N, InList) =:= InList + ). + +%% keyfind/3 +prop_keyfind() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + Found = lists:keyfind(Key, N, InList), + is_tuple(Found) andalso + tuple_size(Found) >= N andalso + element(N, Found) == Key + end + ). + +prop_keyfind_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keyfind(make_ref(), N, InList) + ). + +%% keymap/3 +prop_keymap() -> + ?FORALL( + {MapFn, N, InList, ExpList}, + ?LET( + Fn, + function([gen_any()], gen_any()), + ?LET( + N, + range(1, 5), + ?LET( + {L, M}, + gen_list_fold( + gen_tuple(N, N + 3), + fun(T, Acc) -> + Acc ++ [setelement(N, T, Fn(element(N, T)))] + end, + [] + ), + {Fn, N, L, M} + ) + ) + ), + lists:keymap(MapFn, N, InList) =:= ExpList + ). + +%% keymember/3 +prop_keymember() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + lists:keymember(Key, N, InList) + ). + +prop_keymember_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keymember(make_ref(), N, InList) + ). + +%% keymerge/3 +prop_keymerge() -> + ?FORALL( + {N, InList1, InList2}, + ?LET( + N, + range(1, 5), + ?LET( + {L1, L2}, + {list(gen_tuple(N, N+3)), list(gen_tuple(N, N+3))}, + {N, lists:sort(L1), lists:sort(L2)} + ) + ), + check_merged( + fun (E1, E2) -> element(N, E1) =< element(N, E2) end, + [InList1, InList2], + lists:keymerge(N, InList1, InList2) + ) + ). + +%% keyreplace/4 +prop_keyreplace() -> + ?FORALL( + {Key, N, InList, Replacement}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E0, E1}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3), gen_tuple()}, + {K, N, F ++ [E0|R], E1} + ) + ), + check_keyreplaced(Key, N, Replacement, InList, lists:keyreplace(Key, N, InList, Replacement)) + ). + +prop_keyreplace_absent() -> + ?FORALL( + {N, InList, Replacement}, + {pos_integer(), gen_list(), gen_tuple()}, + lists:keyreplace(make_ref(), N, InList, Replacement) =:= InList + ). + +%% keysearch/3 +prop_keysearch() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + {value, Found} = lists:keysearch(Key, N, InList), + is_tuple(Found) andalso + tuple_size(Found) >= N andalso + element(N, Found) == Key + end + ). + +prop_keysearch_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keysearch(make_ref(), N, InList) + ). + +%% keysort/2 +prop_keysort() -> + ?FORALL( + {N, InList}, + ?LET( + N, + range(1, 5), + {N, list(gen_tuple(N, N + 3))} + ), + begin + Sorted = lists:keysort(N, InList), + length(Sorted) =:= length(InList) andalso + check_sorted(fun(E1, E2) -> element(N, E1) =< element(N, E2) end, InList, Sorted) + end + ). + +%% keystore/4 +prop_keystore() -> + ?FORALL( + {Key, N, InList, ToStore}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E0, E1}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3), gen_tuple()}, + {K, N, F ++ [E0|R], E1} + ) + ), + check_keyreplaced(Key, N, ToStore, InList, lists:keystore(Key, N, InList, ToStore)) + ). + +prop_keystore_absent() -> + ?FORALL( + {N, InList, ToStore}, + {pos_integer(), gen_list(), gen_tuple()}, + lists:keystore(make_ref(), N, InList, ToStore) =:= InList ++ [ToStore] + ). + +%% keytake/3 +prop_keytake() -> + ?FORALL( + {Key, N, InList, ExpList, ExpElem}, + ?LET( + {K, N}, + {make_ref(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R], F ++ R, E} + ) + ), + lists:keytake(Key, N, InList) =:= {value, ExpElem, ExpList} + ). + +prop_keytake_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + lists:keytake(make_ref(), N, InList) =:= false + ). + +%% last/1 +prop_last() -> + ?FORALL( + InList, + gen_list(), + try + lists:last(InList) =:= hd(lists:reverse(InList)) + catch + error:_ -> + InList =:= [] + end + ). + +%% map/2 +prop_map() -> + ?FORALL( + {MapFn, InList, ExpList}, + ?LET( + Fn, + function1(gen_any()), + ?LET( + {L, M}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + Acc ++ [Fn(T)] + end, + [] + ), + {Fn, L, M} + ) + ), + lists:map(MapFn, InList) =:= ExpList + ). + +%% mapfoldl/3 +prop_mapfoldl() -> + ?FORALL( + {MapFoldFn, InList, Acc0, Exp}, + ?LET( + {MapFn, FoldFn, Acc0}, + {function1(gen_any()), function2(gen_any()), gen_any()}, + ?LET( + {L, MV}, + gen_list_fold( + gen_any(), + fun(T, {AccM, AccF}) -> + {AccM ++ [MapFn(T)], FoldFn(T, AccF)} + end, + {[], Acc0} + ), + {fun(T, Acc) -> {MapFn(T), FoldFn(T, Acc)} end, L, Acc0, MV} + ) + ), + lists:mapfoldl(MapFoldFn, Acc0, InList) =:= Exp + ). + +%% mapfoldr/3 +prop_mapfoldr() -> + ?FORALL( + {MapFoldFn, InList, Acc0, Exp}, + ?LET( + {MapFn, FoldFn, Acc0}, + {function1(gen_any()), function2(gen_any()), gen_any()}, + ?LET( + {L, MV}, + gen_list_fold( + gen_any(), + fun(T, {AccM, AccF}) -> + {[MapFn(T)|AccM], FoldFn(T, AccF)} + end, + {[], Acc0} + ), + {fun(T, Acc) -> {MapFn(T), FoldFn(T, Acc)} end, lists:reverse(L), Acc0, MV} + ) + ), + lists:mapfoldr(MapFoldFn, Acc0, InList) =:= Exp + ). + +%% max/1 +prop_max() -> + ?FORALL( + {InList, ExpMax}, + gen_list_fold(gen_any(), fun erlang:max/2), + try + lists:max(InList) == ExpMax + catch + error:_ -> + InList =:= [] + end + ). + +%% member/2 +prop_member() -> + ?FORALL( + {InList, Member}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], E} + ), + lists:member(Member, InList) + ). + +prop_member_absent() -> + ?FORALL( + InList, + gen_list(), + not lists:member(make_ref(), InList) + ). + +%% merge/1 +prop_merge_1() -> + ?FORALL( + InLists, + list(?LET(L, gen_list(), lists:sort(L))), + check_merged(fun erlang:'=<'/2, InLists, lists:merge(InLists)) + ). + +%% merge/2 +prop_merge_2() -> + ?FORALL( + {InList1, InList2}, + ?LET( + {L1, L2}, + {gen_list(), gen_list()}, + {lists:sort(L1), lists:sort(L2)} + ), + check_merged(fun erlang:'=<'/2, [InList1, InList2], lists:merge(InList1, InList2)) + ). + +%% merge/3 +prop_merge_3() -> + ?FORALL( + {SortFn, InList1, InList2}, + ?LET( + {Fn, L1, L2}, + {gen_ordering_fun(), gen_list(), gen_list()}, + {Fn, lists:sort(Fn, L1), lists:sort(Fn, L2)} + ), + check_merged(SortFn, [InList1, InList2], lists:merge(SortFn, InList1, InList2)) + ). + +%% merge3/3 +prop_merge3() -> + ?FORALL( + {InList1, InList2, InList3}, + ?LET( + {L1, L2, L3}, + {gen_list(), gen_list(), gen_list()}, + {lists:sort(L1), lists:sort(L2), lists:sort(L3)} + ), + check_merged(fun erlang:'=<'/2, [InList1, InList2, InList3], lists:merge3(InList1, InList2, InList3)) + ). + +%% min/1 +prop_min() -> + ?FORALL( + {InList, ExpMin}, + gen_list_fold(gen_any(), fun erlang:min/2), + try + lists:min(InList) == ExpMin + catch + error:_ -> + InList =:= [] + end + ). + +%% nth/2 +prop_nth() -> + ?FORALL( + {InList, N, ExpElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], length(F)+1, E} + ), + lists:nth(N, InList) =:= ExpElem + ). + +prop_nth_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:nth(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% nthtail/2 +prop_nthtail() -> + ?FORALL( + {InList, N, ExpTail}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, length(F), R} + ), + lists:nthtail(N, InList) =:= ExpTail + ). + +prop_nthtail_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:nthtail(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% partition/2 +prop_partition() -> + ?FORALL( + {Pred, InList}, + {function1(boolean()), gen_list()}, + begin + {Group1, Group2} = lists:partition(Pred, InList), + check_partitioned(Pred, InList, Group1, Group2) + end + ). + +%% prefix/2 +prop_prefix() -> + ?FORALL( + {InList, Prefix}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, F} + ), + lists:prefix(Prefix, InList) andalso + not lists:prefix([make_ref()|Prefix], InList) andalso + not lists:prefix(Prefix ++ [make_ref()], InList) andalso + (not lists:prefix(Prefix, [make_ref()|InList]) orelse Prefix =:= []) + ). + +%% reverse/1 +prop_reverse_1() -> + ?FORALL( + InList, + gen_list(), + check_reversed(InList, lists:reverse(InList)) andalso + lists:reverse(lists:reverse(InList)) =:= InList + ). + +%% reverse/2 +prop_reverse_2() -> + ?FORALL( + {InList, InTail}, + {gen_list(), gen_list()}, + check_reversed(InList, lists:reverse(InList, InTail), InTail) + ). + +%% search/2 +prop_search() -> + ?FORALL( + {Pred, InList, ExpElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {fun(T) -> T =:= E end, F ++ [E|R], E} + ), + lists:search(Pred, InList) =:= {value, ExpElem} + ). + +prop_search_absent() -> + ?FORALL( + InList, + gen_list(), + lists:search(fun(_) -> false end, InList) =:= false + ). + +%% seq/2 +prop_seq2() -> + ?FORALL( + {From, To}, + {integer(), integer()}, + try + lists:seq(From, To) + of + Seq -> + To >= From - 1 andalso + check_seq(Seq, From, To, 1) + catch + error:_ -> + To < From - 1 + end + ). + +%% seq/3 +prop_seq3() -> + ?FORALL( + {From, To, Step}, + {integer(), integer(), integer()}, + try + lists:seq(From, To, Step) + of + Seq when Step > 0 -> + To >= From - Step andalso + check_seq(Seq, From, To, Step); + Seq when Step < 0 -> + To =< From - Step andalso + check_seq(Seq, From, To, Step); + Seq when Step =:= 0 -> + From =:= To andalso + check_seq(Seq, From, To, Step) + catch + error:_ when Step > 0 -> + To < From - Step; + error:_ when Step < 0 -> + To > From - Step; + error:_ when Step =:= 0 -> + From =/= To + end + ). + +%% sort/1 +prop_sort_1() -> + ?FORALL( + InList, + gen_list(), + begin + Sorted = lists:sort(InList), + length(Sorted) =:= length(InList) andalso + check_sorted(InList, Sorted) + end + ). + +%% sort/2 +prop_sort_2() -> + ?FORALL( + {SortFn, InList}, + {gen_ordering_fun(), gen_list()}, + begin + Sorted = lists:sort(SortFn, InList), + length(Sorted) =:= length(InList) andalso + check_sorted(SortFn, InList, Sorted) + end + ). + +%% split/2 +prop_split() -> + ?FORALL( + {N, InList, ExpList1, ExpList2}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {length(F), F ++ R, F, R} + ), + lists:split(N, InList) =:= {ExpList1, ExpList2} + ). + +prop_split_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:split(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% splitwith/2 +prop_splitwith() -> + ?FORALL( + {Pred, InList}, + {function1(boolean()), gen_list()}, + begin + {Part1, Part2} = lists:splitwith(Pred, InList), + check_splitwithed(Pred, InList, Part1, Part2) + end + ). + +%% sublist/2 +prop_sublist_2() -> + ?FORALL( + {Len, InList, ExpList}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {length(F), F ++ R, F} + ), + lists:sublist(InList, Len) =:= ExpList + ). + +%% sublist/3 +prop_sublist_3() -> + ?FORALL( + {Start, Len, InList, ExpList}, + ?LET( + {F, M, R}, + {gen_list(), gen_list(), gen_list()}, + {length(F)+1, length(M), F ++ M ++ R, M} + ), + lists:sublist(InList, Start, Len) =:= ExpList + ). + +%% subtract/2 +prop_subtract() -> + ?FORALL( + {InList, SubtractList}, + ?LET( + {L, B, S}, + {gen_list(), gen_list(), gen_list()}, + {L ++ B, S ++ B} + ), + lists:subtract(InList, SubtractList) =:= InList -- SubtractList + ). + +%% suffix/2 +prop_suffix() -> + ?FORALL( + {InList, Suffix}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, R} + ), + lists:suffix(Suffix, InList) andalso + not lists:suffix([make_ref()|Suffix], InList) andalso + not lists:suffix(Suffix ++ [make_ref()], InList) andalso + (not lists:suffix(Suffix, InList ++ [make_ref()]) orelse Suffix =:= []) + ). + +%% sum/1 +prop_sum() -> + ?FORALL( + {InList, ExpSum}, + gen_list_fold(number(), fun erlang:'+'/2, 0), + lists:sum(InList) =:= ExpSum + ). + +%% takewhile/2 +prop_takewhile() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + Fn, + function1(boolean()), + ?LET( + {L, {_, TL}}, + gen_list_fold( + gen_any(), + fun(E, {Take, Acc}) -> + case Take andalso Fn(E) of + true -> {true, Acc ++ [E]}; + false -> {false, Acc} + end + end, + {true, []} + ), + {Fn, L, TL} + ) + ), + lists:takewhile(Pred, InList) =:= ExpList + ). + +%% ukeymerge/3 +prop_ukeymerge() -> + ?FORALL( + {N, InList1, InList2}, + ?LET( + N, + range(1, 5), + ?LET( + {L1, L2}, + {list(gen_tuple(N, N+3)), list(gen_tuple(N, N+3))}, + {N, lists:ukeysort(N, L1), lists:ukeysort(N, L2)} + ) + ), + check_umerged( + fun(E1, E2) -> element(N, E1) =< element(N, E2) end, + [InList1, InList2], + lists:ukeymerge(N, InList1, InList2) + ) + ). + +%% ukeysort/2 +prop_ukeysort() -> + ?FORALL( + {N, InList}, + ?LET( + N, + range(1, 5), + {N, list(gen_tuple(N, N + 3))} + ), + begin + Sorted = lists:ukeysort(N, InList), + length(Sorted) =< length(InList) andalso + check_usorted(fun(E1, E2) -> element(N, E1) =< element(N, E2) end, InList, Sorted) + end + ). + +%% umerge/1 +prop_umerge_1() -> + ?FORALL( + InLists, + list(?LET(L, gen_list(), lists:usort(L))), + check_umerged(InLists, lists:umerge(InLists)) + ). + +%% umerge/2 +prop_umerge_2() -> + ?FORALL( + {InList1, InList2}, + ?LET( + {L1, L2}, + {gen_list(), gen_list()}, + {lists:usort(L1), lists:usort(L2)} + ), + check_umerged([InList1, InList2], lists:umerge(InList1, InList2)) + ). + +%% umerge/3 +prop_umerge_3() -> + ?FORALL( + {SortFn, InList1, InList2}, + ?LET( + {Fn, L1, L2}, + {gen_ordering_fun(), gen_list(), gen_list()}, + {Fn, lists:usort(Fn, L1), lists:usort(Fn, L2)} + ), + check_umerged(SortFn, [InList1, InList2], lists:umerge(SortFn, InList1, InList2)) + ). + +%% umerge3/3 +prop_umerge3() -> + ?FORALL( + {InList1, InList2, InList3}, + ?LET( + {L1, L2, L3}, + {gen_list(), gen_list(), gen_list()}, + {lists:usort(L1), lists:usort(L2), lists:usort(L3)} + ), + check_umerged([InList1, InList2, InList3], lists:umerge3(InList1, InList2, InList3)) + ). + +%% uniq/1 +prop_uniq_1() -> + ?FORALL( + InList, + ?LET( + {L, M}, + {gen_list(), gen_list()}, + ?LET( + S, + vector(length(L) + 2 * length(M), integer()), + [E || {_, E} <- lists:sort(lists:zip(S, L ++ M ++ M))] + ) + ), + check_uniqed(InList, lists:uniq(InList)) + ). + +%% uniq/2 +prop_uniq_2() -> + ?FORALL( + {UniqFn, InList}, + {function1(oneof([a, b, c])), gen_list()}, + check_uniqed(UniqFn, InList, lists:uniq(UniqFn, InList)) + ). + +%% unzip/1 +prop_unzip() -> + ?FORALL( + {InList, {ExpList1, ExpList2}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2}) -> + {L1 ++ [T1], L2 ++ [T2]} + end, + {[], []} + ), + lists:unzip(InList) =:= {ExpList1, ExpList2} + ). + +%% unzip3/1 +prop_unzip3() -> + ?FORALL( + {InList, {ExpList1, ExpList2, ExpList3}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3]} + end, + {[], [], []} + ), + lists:unzip3(InList) =:= {ExpList1, ExpList2, ExpList3} + ). + +%% usort/1 +prop_usort_1() -> + ?FORALL( + InList, + gen_list(), + begin + Sorted = lists:usort(InList), + length(Sorted) =< length(InList) andalso + check_usorted(InList, Sorted) + end + ). + +%% usort/2 +prop_usort_2() -> + ?FORALL( + {SortFn, InList}, + {gen_ordering_fun(), gen_list()}, + begin + Sorted = lists:usort(SortFn, InList), + length(Sorted) =< length(InList) andalso + check_usorted(SortFn, InList, Sorted) + end + ). + +%% zip/2 +prop_zip() -> + ?FORALL( + {ExpList, {InList1, InList2}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2}) -> + {L1 ++ [T1], L2 ++ [T2]} + end, + {[], []} + ), + lists:zip(InList1, InList2) =:= ExpList + ). + +%% zip3/3 +prop_zip3() -> + ?FORALL( + {ExpList, {InList1, InList2, InList3}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3]} + end, + {[], [], []} + ), + lists:zip3(InList1, InList2, InList3) =:= ExpList + ). + +%% zipwith/3 +prop_zipwith() -> + ?FORALL( + {ZipFn, InList1, InList2, ExpList}, + ?LET( + Fn, + function2(gen_any()), + ?LET( + {_, {L1, L2, Z}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2, Z}) -> + {L1 ++ [T1], L2 ++ [T2], Z ++ [Fn(T1, T2)]} + end, + {[], [], []} + ), + {Fn, L1, L2, Z} + ) + ), + lists:zipwith(ZipFn, InList1, InList2) =:= ExpList + ). + +%% zipwith3/4 +prop_zipwith3() -> + ?FORALL( + {ZipFn, InList1, InList2, InList3, ExpList}, + ?LET( + Fn, + function3(gen_any()), + ?LET( + {_, {L1, L2, L3, Z}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3, Z}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3], Z ++ [Fn(T1, T2, T3)]} + end, + {[], [], [], []} + ), + {Fn, L1, L2, L3, Z} + ) + ), + lists:zipwith3(ZipFn, InList1, InList2, InList3) =:= ExpList + ). + +%%%%%%%%%%%%%%%%%% +%%% Generators %%% +%%%%%%%%%%%%%%%%%% + +%% Generator for lists of the given type, folding the given function +%% over values on the top level as they are generated. The first generated +%% value serves as the initial accumulator. +gen_list_fold(Gen, FoldFn) -> + ?SIZED( + Size, + ?LET( + T, + Gen, + if + Size =< 1 -> + {[], T}; + true -> + gen_list_fold(max(0, Size - 1), Gen, [T], FoldFn, T) + end + ) + ). + +%% Generator for lists of the given type, folding the given function +%% over values on the top level as they are generated. +gen_list_fold(Gen, FoldFn, Acc0) -> + ?SIZED( + Size, + gen_list_fold(max(0, Size - 1), Gen, [], FoldFn, Acc0) + ). + +gen_list_fold(0, _Gen, L, _FoldFn, Acc) -> + {L, Acc}; +gen_list_fold(N, Gen, L, FoldFn, Acc) -> + ?LET( + E, + Gen, + gen_list_fold(N - 1, Gen, L ++ [E], FoldFn, FoldFn(E, Acc)) + ). + +%% Generator for key tuples of the given size, +%% with the given key in the given (ie, last) position. +gen_keytuple(Key, Size) -> + gen_keytuple(Key, Size, Size). + +%% Generator for key tuples of the given minimum and maximum +%% sizes, with the given key in the given minimum position. +gen_keytuple(Key, MinSize, MaxSize) -> + ?LET( + Tuple, + gen_tuple(MinSize, MaxSize), + setelement(MinSize, Tuple, Key) + ). + +%% Generator for tuples of random size. +gen_tuple() -> + ?LET( + N, + non_neg_integer(), + gen_tuple(N) + ). + +%% Generator for tuples of the given size. +gen_tuple(Size) -> + ?LET( + V, + vector(Size, gen_any()), + list_to_tuple(V) + ). + +%% Generator for tuples of the given minimum and +%% maximum sizes. +gen_tuple(MinSize, MaxSize) -> + ?LET( + N, + range(MinSize, MaxSize), + ?LET( + V, + vector(N, gen_any()), + list_to_tuple(V) + ) + ). + +%% Generator for lists of anything. +gen_list() -> + list(gen_any()). + +%% Generator for lists of anything, folding the given function +%% over values on all levels of list-nesting as they are generated. +gen_list_deepfold(FoldFn, Acc0) -> + ?SIZED( + Size, + ?LET( + {_, L, Acc}, + gen_list_deepfold(max(0, Size - 1), 0, [], FoldFn, Acc0), + {L, Acc} + ) + ). + +gen_list_deepfold(N, _Level, L, _FoldFn, Acc) when N =< 0 -> + {N, lists:reverse(L), Acc}; +gen_list_deepfold(N, Level, L, FoldFn, Acc) -> + ?LET( + X, + frequency([ + {4, {term, gen_any_simple()}}, + {1, deeplist}, + {1, tuple}, + {2, stop} + ]), + case X of + deeplist -> + ?LET( + {N1, L1, Acc1}, + gen_list_deepfold(N, Level + 1, [], FoldFn, Acc), + gen_list_deepfold(N1, Level, [L1|L], FoldFn, Acc1) + ); + tuple -> + ?LET( + {N1, L1, _}, + gen_list_deepfold(N, Level + 1, [], fun(_, _, _) -> undefined end, undefined), + begin + E = list_to_tuple(L1), + gen_list_deepfold(N1, Level, [E|L], FoldFn, FoldFn(Level, E, Acc)) + end + ); + stop -> + {N, lists:reverse(L), Acc}; + {term, E} -> + gen_list_deepfold(N - 1, Level, [E|L], FoldFn, FoldFn(Level, E, Acc)) + end + ). + +%% Generator for simple and composite (lists and tuples) types. +gen_any() -> + frequency( + [ + {4, gen_any_simple()}, + {1, ?LET({L, _}, gen_list_deepfold(fun(_, _, Acc) -> Acc end, undefined), L)}, + {1, ?LET({L, _}, gen_list_deepfold(fun(_, _, Acc) -> Acc end, undefined), list_to_tuple(L))} + ] + ). + +%% Generator for simple types: +%% - atoms +%% - integers +%% - floats +%% - bitstrings +gen_any_simple() -> + oneof([gen_atom(), integer(), float(), bitstring()]). + +%% Generator for interesting atoms: +%% - well-known atoms like `ok', `undefined', `infinity'... +%% - randomly generated "weird" atoms +gen_atom() -> + oneof( + [ + oneof([ok, error, true, false, undefined, infinity]), + oneof(['', '"', '\'', '(', ')', '()', '[', '[', '[]', '{', '}', '{}']), + gen_random_atom() + ] + ). + +%% Generator for a limited set of random atoms. The number of +%% atoms that will be generated is set in `?RANDOM_ATOMS'. +gen_random_atom() -> + ?LAZY( + ?LET( + N, + range(1, ?RANDOM_ATOMS), + try + persistent_term:get({?MODULE, random_atoms}) + of + Atoms -> + maps:get(N, Atoms) + catch + error:badarg -> + ?LET( + AtomsList, + vector(?RANDOM_ATOMS, ?SIZED(Size, resize(Size * 100, atom()))), + begin + Fn = fun + F(_, [], Acc) -> + Acc; + F(Index, [A|As], Acc) -> + F(Index + 1, As, Acc#{Index => A}) + end, + Atoms = Fn(1, AtomsList, #{}), + persistent_term:put({?MODULE, random_atoms}, Atoms), + maps:get(N, Atoms) + end + ) + end + ) + ). + +%% Generator for ordering functions, to be used for sorting and merging. +%% The generated ordering functions are designed to fulfill the requirements given +%% at the top of the `lists' documentation, namely to be antisymmetric, transitive, +%% and total. Further, the chances that two terms compare equal, less or greater +%% are equal. +gen_ordering_fun() -> + ?LET( + F, + function1(range(1, 3)), + fun(T1, T2) -> + F(T1) =< F(T2) + end + ). + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% + +%% -------------------------------------------------------------------- +check_appended([], []) -> + true; +check_appended([[]|Ls], AL) -> + check_appended(Ls, AL); +check_appended([L], AL) -> + L =:= AL; +check_appended([[E1|L]|Ls], [E2|AL]) -> + E1 =:= E2 andalso + check_appended([L|Ls], AL); +check_appended(_Ls, _AL) -> + false. + +%% -------------------------------------------------------------------- +check_deleted(E, [E|L], DL) -> + L =:= DL; +check_deleted(E, [_|L], [_|DL]) -> + check_deleted(E, L, DL); +check_deleted(_E, [], []) -> + true; +check_deleted(_E, _L, _DL) -> + false. + +%% -------------------------------------------------------------------- +check_joined(Sep, [E|L], [E, Sep|JL]) -> + check_joined(Sep, L, JL); +check_joined(_Sep, [E], [E]) -> + true; +check_joined(_Sep, [], []) -> + true; +check_joined(_Sep, _L, _JL) -> + false. + +%% -------------------------------------------------------------------- +check_keydeleted(K, N, [E|L], KDL) when element(N, E) == K -> + L =:= KDL; +check_keydeleted(K, N, [_|L], [_|KDL]) -> + check_keydeleted(K, N, L, KDL); +check_keydeleted(_K, _N, _L, _KDL) -> + false. + +%% -------------------------------------------------------------------- +check_keyreplaced(K, N, R, [E1|L], [E2|KRL]) when element(N, E1) == K -> + E2 =:= R andalso L =:= KRL; +check_keyreplaced(K, N, R, [_|L], [_|KRL]) -> + check_keyreplaced(K, N, R, L, KRL); +check_keyreplaced(_K, _N, _R, _L, _KRL) -> + false. + +%% -------------------------------------------------------------------- +check_merged(Ls, ML) -> + check_merged(fun erlang:'=<'/2, Ls, ML). + +check_merged(Fn, [[]|Ls], ML) -> + check_merged(Fn, Ls, ML); +check_merged(_Fn, [], ML) -> + ML =:= []; +check_merged(_Fn, [L], ML) -> + ML =:= L; +check_merged(Fn, Ls, [E|ML]) -> + case find_in_heads(Fn, E, Ls) of + {true, Ls1} -> + check_merged(Fn, Ls1, ML); + false -> + false + end; +check_merged(_Fn, _Ls, _ML) -> + false. + +find_in_heads(Fn, E, Ls) -> + find_in_heads(Fn, E, Ls, []). + +find_in_heads(Fn, E, [[]|Ls], Seen) -> + find_in_heads(Fn, E, Ls, Seen); +find_in_heads(Fn, E, [[E1|LRest]=L|Ls], Seen) -> + case Fn(E, E1) andalso Fn(E1, E) of + true -> + {true, lists:reverse(Seen, [LRest|Ls])}; + false -> + find_in_heads(Fn, E, Ls, [L|Seen]) + end; +find_in_heads(_Fn, _E, _Ls, _Seen) -> + false. + +%% -------------------------------------------------------------------- +check_partitioned(Pred, [E|L], P1, P2) -> + case {Pred(E), P1, P2} of + {true, [E|Rest], _} -> + check_partitioned(Pred, L, Rest, P2); + {false, _, [E|Rest]} -> + check_partitioned(Pred, L, P1, Rest); + _ -> + false + end; +check_partitioned(_Pred, [], [], []) -> + true; +check_partitioned(_Pred, _L, _P1, _P2) -> + false. + +%% -------------------------------------------------------------------- +check_reversed(L1, L2) -> + check_reversed(L1, L2, []). + +check_reversed(L1, L2, Tail) -> + check_reversed1(L1, L2) =:= Tail. + +check_reversed1([], L2) -> + L2; +check_reversed1([E|L1], L2) -> + case check_reversed1(L1, L2) of + [E|L2Rest] -> L2Rest; + _ -> false + end. + +%% -------------------------------------------------------------------- +check_seq([F|Seq], F, T, S) -> + check_seq(Seq, F + S, T, S); +check_seq([], F, T, S) when S >= 0 -> + F >= T; +check_seq([], F, T, S) when S < 0 -> + F =< T; +check_seq(_Seq, _F, _T, _S) -> + false. + +%% -------------------------------------------------------------------- +check_sorted(L, Sorted) -> + check_sorted(fun erlang:'=<'/2, L, Sorted). + +check_sorted(SortFun, L, Sorted) -> + ExpElems = count_elems(L), + check_sorted(SortFun, Sorted, ExpElems, #{}). + +check_sorted(_SortFun, [], ExpElems, FoundElems) -> + ExpElems =:= FoundElems; +check_sorted(SortFun, [E], ExpElems, FoundElems) -> + maps:is_key(E, ExpElems) andalso + check_sorted(SortFun, [], ExpElems, maps:update_with(E, fun(Cnt) -> Cnt + 1 end, 1, FoundElems)); +check_sorted(SortFun, [E1|[E2|_]=L], ExpElems, FoundElems) -> + SortFun(E1, E2) andalso + maps:is_key(E1, ExpElems) andalso + check_sorted(SortFun, L, ExpElems, maps:update_with(E1, fun(Cnt) -> Cnt + 1 end, 1, FoundElems)); +check_sorted(_SortFun, _L, _ExpElems, _FoundElems) -> + false. + +count_elems(L) -> + count_elems(L, #{}). + +count_elems([E|Es], Acc) -> + count_elems(Es, maps:update_with(E, fun(Cnt) -> Cnt + 1 end, 1, Acc)); +count_elems([], Acc) -> + Acc. + +%% -------------------------------------------------------------------- +check_splitwithed(Pred, [E|L], [E|P1], P2) -> + Pred(E) andalso + check_splitwithed(Pred, L, P1, P2); +check_splitwithed(Pred, [E|_]=L, [], P2) -> + not Pred(E) andalso L =:= P2; +check_splitwithed(_Pred, [], [], []) -> + true; +check_splitwithed(_Pred, _L, _P1, _P2) -> + false. + +%% -------------------------------------------------------------------- +check_umerged(Ls, ML) -> + check_umerged(fun erlang:'=<'/2, Ls, ML). + +check_umerged(Fn, [[]|Ls], ML) -> + check_umerged(Fn, Ls, ML); +check_umerged(_Fn, [L], ML) -> + ML =:= L; +check_umerged(_Fn, [], ML) -> + ML =:= []; +check_umerged(Fn, Ls, [E|ML]) -> + case find_and_remove_from_heads(Fn, E, Ls) of + {true, Ls1} -> + check_umerged(Fn, Ls1, ML); + false -> + false + end; +check_umerged(_Fn, _Ls, _ML) -> + false. + +find_and_remove_from_heads(Fn, E, Ls) -> + find_and_remove_from_heads(false, Fn, E, Ls, []). + +find_and_remove_from_heads(Found, Fn, E, [[]|Ls], Seen) -> + find_and_remove_from_heads(Found, Fn, E, Ls, Seen); +find_and_remove_from_heads(false, _Fn, _E, [], _Seen) -> + false; +find_and_remove_from_heads(true, _Fn, _E, [], Seen) -> + {true, lists:reverse(Seen)}; +find_and_remove_from_heads(Found, Fn, E, [[E1|LRest]=L|Ls], Seen) -> + case Fn(E, E1) andalso Fn(E1, E) of + true -> + find_and_remove_from_heads(true, Fn, E, Ls, [LRest|Seen]); + false -> + find_and_remove_from_heads(Found, Fn, E, Ls, [L|Seen]) + end. + +%% -------------------------------------------------------------------- +check_uniqed(L, UL) -> + check_uniqed(fun(X) -> X end, L, UL). + +check_uniqed(Fn, L, UL) -> + check_uniqed1(Fn, L, UL, sets:new([{version, 2}])). + +check_uniqed1(Fn, [E|L], [], Seen) -> + sets:is_element(Fn(E), Seen) andalso + check_uniqed1(Fn, L, [], Seen); +check_uniqed1(Fn, [E1|L], [E2|URest]=U, Seen) -> + X1 = Fn(E1), + X2 = Fn(E2), + case sets:is_element(X1, Seen) of + true -> + X1 =/= X2 andalso + check_uniqed1(Fn, L, U, Seen); + false -> + X1 =:= X2 andalso + check_uniqed1(Fn, L, URest, sets:add_element(X1, Seen)) + end; +check_uniqed1(_Fn, [], [], _Seen) -> + true; +check_uniqed1(_Fn, _L, _UL, _Seen) -> + false. + +%% -------------------------------------------------------------------- +check_usorted(L, Sorted) -> + check_usorted(fun erlang:'=<'/2, L, Sorted). + +check_usorted(SortFun, L, Sorted) -> + ExpElems = ucount_elems(SortFun, L), + check_sorted(SortFun, Sorted, ExpElems, #{}). + +ucount_elems(SortFun, L) -> + ucount_elems(SortFun, L, #{}). + +ucount_elems(SortFun, [E|Es], Acc) -> + K = ufind_key(SortFun, E, maps:keys(Acc)), + ucount_elems(SortFun, Es, maps:put(K, 1, Acc)); +ucount_elems(_SortFun, [], Acc) -> + Acc. + +ufind_key(SortFun, E, [K|Keys]) -> + case SortFun(E, K) andalso SortFun(K, E) of + true -> + K; + false -> + ufind_key(SortFun, E, Keys) + end; +ufind_key(_SortFun, E, []) -> + E. diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index 09a65d8fdd..fc6e977942 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, pcre/1,compile_options/1, run_options/1,combined_options/1,replace_autogen/1, - global_capture/1,replace_input_types/1,replace_return/1, + global_capture/1,replace_input_types/1,replace_with_fun/1,replace_return/1, split_autogen/1,split_options/1,split_specials/1, error_handling/1,pcre_cve_2008_2371/1,re_version/1, pcre_compile_workspace_overflow/1,re_infinite_loop/1, @@ -42,7 +42,7 @@ suite() -> all() -> [pcre, compile_options, run_options, combined_options, replace_autogen, global_capture, replace_input_types, - replace_return, split_autogen, split_options, + replace_with_fun, replace_return, split_autogen, split_options, split_specials, error_handling, pcre_cve_2008_2371, pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, @@ -365,6 +365,16 @@ replace_input_types(Config) when is_list(Config) -> <<"a",208,128,"cd">> = re:replace(<<"abcd">>,"b","\x{400}",[{return,binary},unicode]), ok. +%% Test replace with a replacement function. +replace_with_fun(Config) when is_list(Config) -> + <<"ABCD">> = re:replace("abcd", ".", fun(<<C>>, []) -> <<(C - $a + $A)>> end, [global, {return, binary}]), + <<"AbCd">> = re:replace("abcd", ".", fun(<<C>>, []) when (C - $a) rem 2 =:= 0 -> <<(C - $a + $A)>>; (C, []) -> C end, [global, {return, binary}]), + <<"b-ad-c">> = re:replace("abcd", "(.)(.)", fun(_, [A, B]) -> <<B/binary, $-, A/binary>> end, [global, {return, binary}]), + <<"#ab-B#cd">> = re:replace("abcd", ".(.)", fun(Whole, [<<C>>]) -> <<$#, Whole/binary, $-, (C - $a + $A), $#>> end, [{return, binary}]), + <<"#ab#cd">> = re:replace("abcd", ".(x)?(.)", fun(Whole, [<<>>, _]) -> <<$#, Whole/binary, $#>> end, [{return, binary}]), + <<"#ab#cd">> = re:replace("abcd", ".(.)(x)?", fun(Whole, [_]) -> <<$#, Whole/binary, $#>> end, [{return, binary}]), + ok. + %% Test return options of replace together with global searching. replace_return(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch re:replace("na","(a","")), diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index b38dee47e7..142f8ad445 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -18,24 +18,25 @@ %% %CopyrightEnd% %% -module(shell_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). - -export([forget/1, records/1, known_bugs/1, otp_5226/1, otp_5327/1, otp_5435/1, otp_5195/1, otp_5915/1, otp_5916/1, bs_match_misc_SUITE/1, bs_match_int_SUITE/1, bs_match_tail_SUITE/1, bs_match_bin_SUITE/1, bs_construct_SUITE/1, - refman_bit_syntax/1, - progex_bit_syntax/1, progex_records/1, + refman_bit_syntax/1, + progex_bit_syntax/1, progex_records/1, progex_lc/1, progex_funs/1, otp_5990/1, otp_6166/1, otp_6554/1, otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1, - otp_14285/1, otp_14296/1, typed_records/1]). + otp_14285/1, otp_14296/1, typed_records/1, types/1]). --export([ start_restricted_from_shell/1, +-export([ start_restricted_from_shell/1, start_restricted_on_command_line/1,restricted_local/1]). +-export([ start_interactive/1, whereis/1 ]). + %% Internal export. -export([otp_5435_2/0, prompt1/1, prompt2/1, prompt3/1, prompt4/1, prompt5/1]). @@ -73,19 +74,21 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,10}}]. -all() -> +all() -> [forget, known_bugs, otp_5226, otp_5327, - otp_5435, otp_5195, otp_5915, otp_5916, {group, bits}, + otp_5435, otp_5195, otp_5915, otp_5916, + start_interactive, whereis, {group, bits}, {group, refman}, {group, progex}, {group, tickets}, - {group, restricted}, {group, records}]. + {group, restricted}, {group, records}, {group, definitions}]. -groups() -> +groups() -> [{restricted, [], [start_restricted_from_shell, start_restricted_on_command_line, restricted_local]}, {bits, [], [bs_match_misc_SUITE, bs_match_tail_SUITE, bs_match_bin_SUITE, bs_construct_SUITE]}, + {definitions, [], [types]}, {records, [], [records, typed_records]}, {refman, [], [refman_bit_syntax]}, @@ -300,7 +303,7 @@ restricted_local(Config) when is_list(Config) -> application:get_env(stdlib, restricted_shell), true = purge_and_delete(user_default), ok. - + %% f/0 and f/1. forget(Config) when is_list(Config) -> @@ -319,15 +322,73 @@ forget(Config) when is_list(Config) -> comm_err(<<"f(a).">>), ok. +%% type definition support +types(Config) when is_list(Config) -> + %% type + [ok] = scan(<<"-type baz() :: integer().">>), + %% record + [ok] = scan(<<"-record(foo, {bar :: baz()}).">>), + %% spec + [ok] = scan(<<"-spec foo(Bar) -> Baz when + Bar :: string(), + Baz :: integer().">>), + shell_attribute_test(Config), + ok. +shell_attribute_test(Config) -> + Path = filename:join([proplists:get_value(priv_dir, Config), + "shell_history", "function_def"]), + rtnode:run( + [{putline, "foo(Bar) -> Bar."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{function,{shell_default,foo,1}}\\E"}, + {putline, "foo(1)."}, + {expect, "1"}, + {putline, "shell_default:foo(2)."}, + {expect, "2"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-record(hej, {a = 0 :: integer()})."}, + {expect, "ok"}, + {putline, "rl()."}, + {expect, "\\Q-record(hej,{a = 0 :: integer()}).\\E"}, + {putline, "#hej{a=1}."}, + {expect, "\\Q#hej{a = 1}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-spec foo(Bar) -> Bar when Bar :: integer()."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{function_type,{shell_default,foo,1}}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-type my_type() :: boolean() | integer()."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{type,my_type}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + ok. + %% Test of the record support. OTP-5063. records(Config) when is_list(Config) -> %% rd/2 [{attribute,_,record,{bar,_}},ok] = - scan(<<"rd(foo,{bar}), + scan(<<"rd(foo,{bar}), rd(bar,{foo = (#foo{})#foo.bar}), rl(bar).">>), "variable 'R' is unbound" = % used to work (before OTP-5878, R11B) - exit_string(<<"rd(foo,{bar}), + exit_string(<<"rd(foo,{bar}), R = #foo{}, rd(bar,{foo = R#foo.bar}).">>), "exception error: no function clause matching call to rd/2" = @@ -401,7 +462,7 @@ records(Config) when is_list(Config) -> [{attribute,A1,record,{test1,_}},ok] = scan(RR5), RR6 = "rr(\"" ++ Test ++ "\", '_', {d,test2}), rl([test1,test2]).", [{attribute,A1,record,{test2,_}},ok] = scan(RR6), - RR7 = "rr(\"" ++ Test ++ + RR7 = "rr(\"" ++ Test ++ "\", '_', [{d,test1},{d,test2,17}]), rl([test1,test2]).", [{attribute,A1,record,{test1,_}},{attribute,A1,record,{test2,_}},ok] = scan(RR7), @@ -503,7 +564,7 @@ records(Config) when is_list(Config) -> [ok] = scan(<<"rd(a,{}), is_record({a},a) andalso true, b().">>), - + %% nested record defs "#b{a = #a{}}.\n" = t(<<"rd(a,{}), rd(b, {a = #a{}}), #b{}.">>), @@ -672,7 +733,7 @@ otp_5435_2() -> %% application being in the path. %% OTP-5876. [{attribute,_,record,{bar,_}},ok] = - scan(<<"rd(foo,{bar}), + scan(<<"rd(foo,{bar}), rd(bar,{foo = (#foo{})#foo.bar}), rl(bar).">>), ok. @@ -685,7 +746,7 @@ otp_5195(Config) when is_list(Config) -> %% An experimental shell used to translate error tuples: %% "(qlc) \"1: generated variable 'X' must not be used in " - %% "list expression\".\n" = + %% "list expression\".\n" = %% t(<<"qlc:q([X || X <- [{a}], Y <- [X]]).">>), %% Same as last one (if the shell does not translate error tuples): [{error,qlc,{{1,31},qlc,{used_generator_variable,'X'}}}] = @@ -1204,7 +1265,7 @@ bs_match_int_SUITE(Config) when is_list(Config) -> Int -> ok; Other -> io:format(\"Bin = ~p,\", [Bin]), - io:format(\"SkipBef = ~p, N = ~p\", + io:format(\"SkipBef = ~p, N = ~p\", [SkipBef,N]), io:format(\"Expected ~p, got ~p\", [Int,Other]) @@ -1311,8 +1372,8 @@ ok = evaluate(C, []). %% OTP-5327. Adopted from emulator/test/bs_match_bin_SUITE.erl. bs_match_bin_SUITE(Config) when is_list(Config) -> - ByteSplitBinary = - <<"ByteSplit = + ByteSplitBinary = + <<"ByteSplit = fun(L, B, Pos, Fun) when Pos >= 0 -> Sz1 = Pos, Sz2 = size(B) - Pos, @@ -1343,7 +1404,7 @@ ok = evaluate(ByteSplitBinary, []), BitSplitBinary = <<"Mkbin = fun(L) when list(L) -> list_to_binary(L) end, - MakeInt = + MakeInt = fun(List, 0, Acc, _F) -> Acc; ([H|T], N, Acc, F) -> F(T, N-1, Acc bsl 1 bor H, F) end, @@ -1452,7 +1513,7 @@ bs_construct_SUITE(Config) when is_list(Config) -> ?FAIL(<<<<23,56,0,2>>:(-16)/binary>>) "," ?FAIL(<<<<23,56,0,2>>:(2.5)/binary>>) "," ?FAIL(<<<<23,56,0,2>>:(anka)>>) " - end, + end, TestF(), NotUsed1 = fun(I, BinString) -> <<I:32,BinString/binary>>, ok end, @@ -1512,7 +1573,7 @@ ok = evaluate(C1, []), C2 = <<" I = fun(X) -> X end, - Fail = fun() -> + Fail = fun() -> I_minus_777 = I(-777), I_minus_2047 = I(-2047), @@ -1634,8 +1695,8 @@ progex_bit_syntax(Config) when is_list(Config) -> Fun4 = fun(Dgram) -> DgramSize = byte_size(Dgram), - case Dgram of - <<?IP_VERSION:4, HLen:4, SrvcType:8, TotLen:16, + case Dgram of + <<?IP_VERSION:4, HLen:4, SrvcType:8, TotLen:16, ID:16, Flgs:3, FragOff:13, TTL:8, Proto:8, HdrChkSum:16, SrcIP:32, DestIP:32, @@ -1736,7 +1797,7 @@ triples_to_bin1(T) -> triples_to_bin1([{X,Y,Z} | T], Acc) -> triples_to_bin1(T, <<Acc/binary, X:32, Y:32, Z:32>>); % inefficient -triples_to_bin1([], Acc) -> +triples_to_bin1([], Acc) -> Acc. triples_to_bin2(T) -> @@ -1744,12 +1805,12 @@ triples_to_bin2(T) -> triples_to_bin2([{X,Y,Z} | T], Acc) -> triples_to_bin2(T, [<<X:32, Y:32, Z:32>> | Acc]); -triples_to_bin2([], Acc) -> +triples_to_bin2([], Acc) -> list_to_binary(lists:reverse(Acc)). %% Record examples from Programming Examples. OTP-5237. progex_records(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(recs). -record(person, {name = \"\", phone = [], address}). -record(name, {first = \"Robert\", last = \"Ericsson\"}). @@ -1786,7 +1847,7 @@ t() -> c), P3 = #person{name=\"Joe\", phone=[0,0,7], address=\"A street\"}, - #person{name = Name} = P3, + #person{name = Name} = P3, \"Joe\" = Name, \"Robert\" = demo(), @@ -1854,7 +1915,7 @@ Test1_shell = Find), P3 = #person{name=\"Joe\", phone=[0,0,7], address=\"A street\"}, - #person{name = Name} = P3, + #person{name = Name} = P3, \"Joe\" = Name, Demo = fun() -> @@ -1884,7 +1945,7 @@ print(#person{name = Name, age = Age, io:format(\"Name: ~s, Age: ~w, Phone: ~w ~n\" \"Dictionary: ~w.~n\", [Name, Age, Phone, Dict]). - birthday(P) when record(P, person) -> + birthday(P) when record(P, person) -> P#person{age = P#person.age + 1}. register_two_hackers() -> @@ -1902,7 +1963,7 @@ ok. %% List comprehension examples from Programming Examples. OTP-5237. progex_lc(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(lc). -export([t/0]). @@ -2036,7 +2097,7 @@ ok. %% Funs examples from Programming Examples. OTP-5237. progex_funs(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(funs). -export([t/0]). @@ -2273,7 +2334,7 @@ Test2_shell = \"ERLANG\" = Upcase_word(\"Erlang\"), [\"I\",\"LIKE\",\"ERLANG\"] = lists:map(Upcase_word, L), {[\"I\",\"LIKE\",\"ERLANG\"],11} = - lists:mapfoldl(fun(Word, Sum) -> + lists:mapfoldl(fun(Word, Sum) -> {Upcase_word(Word), Sum + length(Word)} end, 0, L), [500,12,45] = lists:filter(Big, [500,12,2,45,6,7]), @@ -2318,10 +2379,10 @@ otp_6166(Config) when is_list(Config) -> -record(r6, {f = #r5{}}). % r6 > r0 -record(r0, {f = #r5{}, g = #r5{}}). % r0 < r5">>, ok = file:write_file(Test2, Contents2), - - RR12 = "[r1,r2,r3,r4,r5] = rr(\"" ++ Test1 ++ "\"), - [r0,r1,r2,r3,r4,r5,r6] = rr(\"" ++ Test2 ++ "\"), - R0 = #r0{}, R6 = #r6{}, + + RR12 = "[r1,r2,r3,r4,r5] = rr(\"" ++ Test1 ++ "\"), + [r0,r1,r2,r3,r4,r5,r6] = rr(\"" ++ Test2 ++ "\"), + R0 = #r0{}, R6 = #r6{}, true = is_record(R0, r0), true = is_record(R6, r6), ok. ", @@ -2438,7 +2499,7 @@ otp_6554(Config) when is_list(Config) -> "exception error: undefined function math:sqrt/2" = comm_err(<<"math:sqrt(2, 2).">>), "exception error: limit of number of arguments to interpreted function " - "exceeded" = + "exceeded" = comm_err(<<"fun(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U) ->" " a end().">>), "exception error: bad filter a" = @@ -2536,7 +2597,7 @@ otp_6554(Config) when is_list(Config) -> t(<<"results(2). 1. v(2). h().">>), application:unset_env(stdlib, shell_saved_results), "1\nfoo\n17\nB = foo\nC = 17\nF = fun() ->\n foo" - "\n end.\nok.\n" = + "\n end.\nok.\n" = t(<<"begin F = fun() -> foo end, 1 end. B = F(). C = 17. b().">>), "3: command not found" = comm_err(<<"#{v(3) => v}.">>), @@ -2562,9 +2623,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184, X} - end + end end), receive up -> ok end, erlang:raise(throw, thrown, []).">>), @@ -2574,9 +2635,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184, X} - end + end end), receive up -> ok end, erlang:raise(exit, fini, []).">>), @@ -2586,9 +2647,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184,X} - end + end end), receive up -> ok end, erlang:raise(error, bad, []).">>), @@ -2713,7 +2774,7 @@ exit_term(B) -> -endif. error_string(B) -> - "** exception error:" ++ Reply = t(B), + "** exception error:" ++ Reply = t(B), caught_string(Reply). exit_string(B) -> @@ -2820,7 +2881,7 @@ otp_10302(Config) when is_list(Config) -> {ok, Es} = erl_parse:parse_exprs(Ts), B = erl_eval:new_bindings(), erl_eval:exprs(Es, B).">>, - + "ok.\n** exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n" " called as <<\"ª\">> / <<\"ª\">>.\n" = t({Node,Test7}), @@ -3014,11 +3075,198 @@ otp_14296(Config) when is_list(Config) -> {error, {_,_,"bad term"}} = TF("1, 2"), ok. +start_interactive(_Config) -> + start_interactive_shell([]), + start_interactive_shell(["-env","TERM","dumb"]). + +start_interactive_shell(ExtraArgs) -> + + %% Basic test case + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive() end}, + {expect, "1>"}, + {expect, "2>"}, + {eval, fun() -> {error,already_started} = shell:start_interactive(), ok end} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that custom MFA works + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive({shell,start,[]}) end}, + {expect, "1>"}, + {expect, "2>"} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that we can start noshell and then a shell + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive(noshell) end}, + {eval, fun() -> io:format(user,"~ts",[io:get_line(user, "")]) end}, + {expect, "test\\."}, + {eval, fun() -> shell:start_interactive() end}, + {expect, "1>"}, + {putline, "test."}, + {expect, "2>"} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that we can start various remote shell combos + [ begin + {ok, Peer, Node} = ?CT_PEER(), + SNode = atom_to_list(Node), + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive(Arg(Node)) end}, + {expect, "\\Q("++SNode++")\\E2>"} + ] ++ quit_hosting_node(), + [],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + peer:stop(Peer) + end || Arg <- [fun(Node) -> {Node, {shell,start,[]}} end, + fun(Node) -> {remote, atom_to_list(Node)} end, + fun(Node) -> {remote, hd(string:split(atom_to_list(Node),"@"))} end, + fun(Node) -> {remote, atom_to_list(Node), {shell,start,[]}} end + ]], + + %% Test that errors work as they should + {ok, Peer, Node} = ?CT_PEER(), + rtnode:run( + [{expect, "eval_test"}, + {eval, fun() -> + {error,noconnection} = shell:start_interactive( + {remote,"invalid_node"}), + {error,noconnection} = shell:start_interactive( + {remote,"invalid_node", + {invalid_module, start, []}}), + {error,nofile} = shell:start_interactive( + {remote,atom_to_list(Node), + {invalid_module, start, []}}), + shell:start_interactive({remote, atom_to_list(Node)}) + end}, + {expect, "1> $"} + ] ++ quit_hosting_node(), + [],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + peer:stop(Peer), + + ok. + +whereis(_Config) -> + Proxy = spawn_link( + fun() -> + (fun F(P) -> + receive + {set,NewPid} -> + F(NewPid); + {get,From} -> + From ! P, + F(P) + end + end)(undefined) + end), + + %% Test that shell:whereis() works with JCL in newshell + rtnode:run( + [{expect,"1> $"}, + {putline,"shell:whereis()."}, + {expect,"2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {set,shell:whereis()}, + ok + end}, + {putline,"\^g"}, + {expect, "--> $"}, + {putline, "s"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, "\r\nEshell"}, + {putline,"shell:whereis()."}, + {expect,"2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {get, self()}, + receive PrevPid -> PrevPid end, + io:format("~p =:= ~p~n",[PrevPid, shell:whereis()]), + false = PrevPid =:= shell:whereis(), + ok + end}, + {putline,"\^g"}, + {expect, "--> $"}, + {putline, "c 1"}, + {expect, "\r\n"}, + {putline, ""}, + {expect, "2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {get, self()}, + receive PrevPid -> PrevPid end, + true = PrevPid =:= shell:whereis(), + ok + end}]), + + %% Test that shell:whereis() works in oldshell + rtnode:run( + [{expect,"1>"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + true = is_pid(shell:whereis()), + ok + end}], + [],"",["-env","TERM","dumb"]), + + %% Test that noinput and noshell gives undefined shell process + rtnode:run( + [{eval,fun() -> + group_leader(erlang:whereis(user),self()), + undefined = shell:whereis(), + ok + end}], + [],"",["-noinput"]), + rtnode:run( + [{eval,fun() -> + group_leader(erlang:whereis(user),self()), + undefined = shell:whereis(), + ok + end}], + [],"",["-noshell"]), + + %% Test that remsh gives the correct shell process + {ok, Peer, Node} = ?CT_PEER(), + NodeStr = lists:flatten(io_lib:format("~w",[Node])), + rtnode:run( + [{expect, "1>"}, + {putline,"shell:whereis()."}, + {expect,"\n<0[.]"}, + {expect, "2>"}, + {eval, fun() -> + group_leader(erlang:whereis(user),self()), + true = Node =:= node(shell:whereis()), + ok + end}] ++ quit_hosting_node(), + peer:random_name(?FUNCTION_NAME), " ", "-remsh " ++ NodeStr ++ + " -pa " ++ filename:dirname(code:which(?MODULE))), + + peer:stop(Peer), + + ok. + +quit_hosting_node() -> + [{putline, "\^g"}, + {expect, "--> $"}, + {putline, "s"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, ["Eshell"]}, + {expect, ["1> $"]}]. + term_to_string(T) -> lists:flatten(io_lib:format("~w", [T])). scan(B) -> - F = fun(Ts) -> + F = fun(Ts) -> case erl_parse:parse_term(Ts) of {ok,Term} -> Term; @@ -3059,18 +3307,18 @@ t1(Parent, {Bin,Enc}, F) -> S = #state{bin = Bin, unic = Enc, reply = [], leader = group_leader()}, group_leader(self(), self()), _Shell = F(), - try + try server_loop(S) catch exit:R -> Parent ! {self(), R}; throw:{?MODULE,LoopReply,latin1} -> - L0 = binary_to_list(list_to_binary(LoopReply)), - [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), - Parent ! {self(), dotify(L1)}; + L0 = binary_to_list(list_to_binary(LoopReply)), + [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), + Parent ! {self(), dotify(L1)}; throw:{?MODULE,LoopReply,_Uni} -> - Tmp = unicode:characters_to_binary(LoopReply), - L0 = unicode:characters_to_list(Tmp), - [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), - Parent ! {self(), dotify(L1)} + Tmp = unicode:characters_to_binary(LoopReply), + L0 = unicode:characters_to_list(Tmp), + [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), + Parent ! {self(), dotify(L1)} after group_leader(S#state.leader, self()) end. @@ -3102,20 +3350,20 @@ start_new_shell(Node) -> %% This is a very minimal implementation of the IO protocol... server_loop(S) -> - receive + receive {io_request, From, ReplyAs, Request} when is_pid(From) -> - server_loop(do_io_request(Request, From, S, ReplyAs)); - NotExpected -> + server_loop(do_io_request(Request, From, S, ReplyAs)); + NotExpected -> exit(NotExpected) end. - + do_io_request(Req, From, S, ReplyAs) -> case io_requests([Req], [], S) of {_Status,{eof,_},S1} -> - io_reply(From, ReplyAs, {error,terminated}), - throw({?MODULE,S1#state.reply,S1#state.unic}); - {_Status,Reply,S1} -> - io_reply(From, ReplyAs, Reply), + io_reply(From, ReplyAs, {error,terminated}), + throw({?MODULE,S1#state.reply,S1#state.unic}); + {_Status,Reply,S1} -> + io_reply(From, ReplyAs, Reply), S1 end. @@ -3133,7 +3381,7 @@ io_requests([R | Rs], Cont, S) -> end; io_requests([], [Rs|Cont], S) -> io_requests(Rs, Cont, S); -io_requests([], [], S) -> +io_requests([], [], S) -> {ok,ok,S}. io_request({setopts, Opts}, S) -> @@ -3165,7 +3413,7 @@ io_request({put_chars,unicode,Chars0}, S) -> {ok,ok,S#state{reply = [S#state.reply | Chars]}}; io_request({put_chars,Enc,Mod,Func,Args}, S) -> case catch apply(Mod, Func, Args) of - Chars when is_list(Chars) -> + Chars when is_list(Chars) -> io_request({put_chars,Enc,Chars}, S) end; io_request({get_until,Enc,_Prompt,Mod,Func,ExtraArgs}, S) -> @@ -3178,7 +3426,7 @@ get_until_loop(M, F, As, S, {more,Cont}, Enc) -> Bin = S#state.bin, case byte_size(Bin) of 0 -> - get_until_loop(M, F, As, S, + get_until_loop(M, F, As, S, catch apply(M, F, [Cont,eof|As]), Enc); _ when S#state.unic =:= latin1 -> get_until_loop(M, F, As, S#state{bin = <<>>}, @@ -3208,7 +3456,7 @@ run_file(Config, Module, Test) -> ok = file:write_file(FileName, Test), ok = compile_file(Config, FileName, Test, []), code:purge(Module), - {module, Module} = code:load_abs(LoadBeamFile), + {module, Module} = code:load_abs(LoadBeamFile), ok = Module:t(), file:delete(FileName), file:delete(BeamFile), diff --git a/lib/stdlib/test/shell_docs_SUITE.erl b/lib/stdlib/test/shell_docs_SUITE.erl index 028e2c0aba..b7d85204d8 100644 --- a/lib/stdlib/test/shell_docs_SUITE.erl +++ b/lib/stdlib/test/shell_docs_SUITE.erl @@ -255,14 +255,15 @@ render_non_native(_Config) -> beam_language = not_erlang, format = <<"text/asciidoc">>, module_doc = #{<<"en">> => <<"This is\n\npure text">>}, - docs= [] + docs = [] }, <<"\n\tnot_an_erlang_module\n\n" " This is\n" " \n" " pure text\n">> = - unicode:characters_to_binary(shell_docs:render(not_an_erlang_module, Docs, #{})), + unicode:characters_to_binary( + shell_docs:render(not_an_erlang_module, Docs, #{ ansi => false })), ok. diff --git a/lib/stdlib/test/timer_simple_SUITE.erl b/lib/stdlib/test/timer_simple_SUITE.erl index 98a8dd408d..9031a2d242 100644 --- a/lib/stdlib/test/timer_simple_SUITE.erl +++ b/lib/stdlib/test/timer_simple_SUITE.erl @@ -51,7 +51,11 @@ kill_after2/1, kill_after3/1, apply_interval1/1, + apply_interval2/1, apply_interval_invalid_args/1, + apply_repeatedly1/1, + apply_repeatedly2/1, + apply_repeatedly_invalid_args/1, send_interval1/1, send_interval2/1, send_interval3/1, @@ -95,6 +99,7 @@ all() -> {group, exit_after}, {group, kill_after}, {group, apply_interval}, + {group, apply_repeatedly}, {group, send_interval}, {group, cancel}, {group, sleep}, @@ -152,10 +157,20 @@ groups() -> [], [ apply_interval1, + apply_interval2, apply_interval_invalid_args ] }, { + apply_repeatedly, + [], + [ + apply_repeatedly1, + apply_repeatedly2, + apply_repeatedly_invalid_args + ] + }, + { send_interval, [], [ @@ -406,6 +421,23 @@ apply_interval1(Config) when is_list(Config) -> {ok, cancel} = timer:cancel(Ref), nor = get_mess(1000, Msg). +%% Test apply_interval with the execution time of the action +%% longer than the timer interval. The timer should not wait for +%% the action to complete, ie start another action while the +%% previously started action is still running. +apply_interval2(Config) when is_list(Config) -> + Msg = make_ref(), + Self = self(), + {ok, Ref} = timer:apply_interval(500, erlang, apply, + [fun() -> + Self ! Msg, + receive after 1000 -> ok end + end, []]), + receive after 1800 -> ok end, + {ok, cancel} = timer:cancel(Ref), + ok = get_mess(1000, Msg, 3), + nor = get_mess(1000, Msg). + %% Test that apply_interval rejects invalid arguments. apply_interval_invalid_args(Config) when is_list(Config) -> {error, badarg} = timer:apply_interval(-1, foo, bar, []), @@ -414,6 +446,44 @@ apply_interval_invalid_args(Config) when is_list(Config) -> {error, badarg} = timer:apply_interval(0, foo, bar, baz), ok. +%% Test of apply_repeatedly by sending messages. Receive +%% 3 messages, cancel the timer, and check that we do +%% not get any more messages. In a case like this, ie where +%% the execution time of the action is shorter than the timer +%% interval, this should behave the same as apply_interval. +apply_repeatedly1(Config) when is_list(Config) -> + Msg = make_ref(), + {ok, Ref} = timer:apply_repeatedly(1000, ?MODULE, send, + [self(), Msg]), + ok = get_mess(1500, Msg, 3), + {ok, cancel} = timer:cancel(Ref), + nor = get_mess(1000, Msg). + +%% Test apply_repeatedly with the execution time of the action +%% longer than the timer interval. The timer should wait for +%% the action to complete, ie not start another action until it +%% has completed. +apply_repeatedly2(Config) when is_list(Config) -> + Msg = make_ref(), + Self = self(), + {ok, Ref} = timer:apply_repeatedly(1, erlang, apply, + [fun() -> + Self ! Msg, + receive after 1000 -> ok end + end, []]), + receive after 2500 -> ok end, + {ok, cancel} = timer:cancel(Ref), + ok = get_mess(1000, Msg, 3), + nor = get_mess(1000, Msg). + +%% Test that apply_repeatedly rejects invalid arguments. +apply_repeatedly_invalid_args(Config) when is_list(Config) -> + {error, badarg} = timer:apply_repeatedly(-1, foo, bar, []), + {error, badarg} = timer:apply_repeatedly(0, "foo", bar, []), + {error, badarg} = timer:apply_repeatedly(0, foo, "bar", []), + {error, badarg} = timer:apply_repeatedly(0, foo, bar, baz), + ok. + %% Test of send_interval/2. Receive 5 messages, cancel the timer, and %% check that we do not get any more messages. send_interval1(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index e4f3e5f379..7be6baa548 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -27,6 +27,7 @@ nfd/1, nfc/1, nfkd/1, nfkc/1, whitespace/1, get/1, + lookup/1, count/1]). -export([debug/0, id/1, bin_split/1, uc_loaded_size/0, @@ -45,6 +46,7 @@ all() -> nfd, nfc, nfkd, nfkc, whitespace, get, + lookup, count ]. @@ -90,7 +92,7 @@ casefold(_) -> whitespace(_) -> WS = unicode_util:whitespace(), WS = lists:filter(fun unicode_util:is_whitespace/1, WS), - %% TODO add more tests + false = unicode_util:is_whitespace($A), ok. cp(_) -> @@ -101,6 +103,15 @@ cp(_) -> "hejsan" = fetch(["hej"|<<"san">>], Get), {error, <<128>>} = Get(<<128>>), {error, [<<128>>, 0]} = Get([<<128>>, 0]), + + {'EXIT', _} = catch Get([-1]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([foo, $a]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([[], -1]), + {'EXIT', _} = catch Get([[-1], $a]), + {'EXIT', _} = catch Get([[-1, $a], $a]), + ok. gc(Config) -> @@ -113,6 +124,15 @@ gc(Config) -> {error, <<128>>} = Get(<<128>>), {error, [<<128>>, 0]} = Get([<<128>>, 0]), + {'EXIT', _} = catch Get([-1]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([foo, $a]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([[], -1]), + {'EXIT', _} = catch Get([[-1], $a]), + {'EXIT', _} = catch Get([[-1, $a], $a]), + {'EXIT', _} = catch Get([<<$a>>, [-1, $a], $a]), %% Current impl + 0 = fold(fun verify_gc/3, 0, DataDir ++ "/GraphemeBreakTest.txt"), ok. @@ -324,6 +344,29 @@ verify_nfkc(Data0, LineNo, _Acc) -> get(_) -> add_get_tests. +lookup(Config) -> + DataDir = proplists:get_value(data_dir, Config), + {ok, Bin} = file:read_file(filename:join(DataDir, "unicode_table.bin")), + 0 = check_category(0, binary_to_term(Bin), 0), + ok. + +check_category(Id, [{Id, {_, _, _, What}}|Rest], Es) -> + case maps:get(category, unicode_util:lookup(Id)) of + What -> check_category(Id+1, Rest, Es); + _Err -> + io:format("~w Exp: ~w Got ~w~n",[Id, What, _Err]), exit(_Err), + check_category(Id+1, Rest, Es+1) + end; +check_category(Id, [{Next,_}|_] = Rest, Es) -> + case maps:get(category, unicode_util:lookup(Id)) of + {other, not_assigned} -> check_category(max(Id+1,Next-1), Rest, Es); + Err -> io:format("~w Exp: {other, not_assigned} Got ~w~n",[Id,Err]), + check_category(max(Id+1,Next-1), Rest, Es+1) + end; +check_category(_Id, [], Es) -> + Es. + + count(Config) -> Parent = self(), Exec = fun() -> diff --git a/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin b/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin Binary files differnew file mode 100644 index 0000000000..65b88fd273 --- /dev/null +++ b/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index 1709acc523..f44095a732 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -280,6 +280,8 @@ zip_api(Config) when is_list(Config) -> Name1 = hd(Names), {ok, Data1} = file:read_file(Name1), {ok, {Name1, Data1}} = zip:zip_get(Name1, ZipSrv), + Data1Crc = erlang:crc32(Data1), + {ok, Data1Crc} = zip:zip_get_crc32(Name1, ZipSrv), %% Get all files FilesDatas = lists:map(fun(Name) -> {ok, B} = file:read_file(Name), diff --git a/lib/stdlib/uc_spec/EastAsianWidth.txt b/lib/stdlib/uc_spec/EastAsianWidth.txt new file mode 100644 index 0000000000..e04f705178 --- /dev/null +++ b/lib/stdlib/uc_spec/EastAsianWidth.txt @@ -0,0 +1,2587 @@ +# EastAsianWidth-14.0.0.txt +# Date: 2021-07-06, 09:58:53 GMT [KW, LI] +# © 2021 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# East_Asian_Width Property +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# The format is two fields separated by a semicolon. +# Field 0: Unicode code point value or range of code point values +# Field 1: East_Asian_Width property, consisting of one of the following values: +# "A", "F", "H", "N", "Na", "W" +# - All code points, assigned or unassigned, that are not listed +# explicitly are given the value "N". +# - The unassigned code points in the following blocks default to "W": +# CJK Unified Ideographs Extension A: U+3400..U+4DBF +# CJK Unified Ideographs: U+4E00..U+9FFF +# CJK Compatibility Ideographs: U+F900..U+FAFF +# - All undesignated code points in Planes 2 and 3, whether inside or +# outside of allocated blocks, default to "W": +# Plane 2: U+20000..U+2FFFD +# Plane 3: U+30000..U+3FFFD +# +# Character ranges are specified as for other property files in the +# Unicode Character Database. +# +# For legacy reasons, there are no spaces before or after the semicolon +# which separates the two fields. The comments following the number sign +# "#" list the General_Category property value or the L& alias of the +# derived value LC, the Unicode character name or names, and, in lines +# with ranges of code points, the code point count in square brackets. +# +# For more information, see UAX #11: East Asian Width, +# at https://www.unicode.org/reports/tr11/ +# +# @missing: 0000..10FFFF; N +0000..001F;N # Cc [32] <control-0000>..<control-001F> +0020;Na # Zs SPACE +0021..0023;Na # Po [3] EXCLAMATION MARK..NUMBER SIGN +0024;Na # Sc DOLLAR SIGN +0025..0027;Na # Po [3] PERCENT SIGN..APOSTROPHE +0028;Na # Ps LEFT PARENTHESIS +0029;Na # Pe RIGHT PARENTHESIS +002A;Na # Po ASTERISK +002B;Na # Sm PLUS SIGN +002C;Na # Po COMMA +002D;Na # Pd HYPHEN-MINUS +002E..002F;Na # Po [2] FULL STOP..SOLIDUS +0030..0039;Na # Nd [10] DIGIT ZERO..DIGIT NINE +003A..003B;Na # Po [2] COLON..SEMICOLON +003C..003E;Na # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +003F..0040;Na # Po [2] QUESTION MARK..COMMERCIAL AT +0041..005A;Na # Lu [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005B;Na # Ps LEFT SQUARE BRACKET +005C;Na # Po REVERSE SOLIDUS +005D;Na # Pe RIGHT SQUARE BRACKET +005E;Na # Sk CIRCUMFLEX ACCENT +005F;Na # Pc LOW LINE +0060;Na # Sk GRAVE ACCENT +0061..007A;Na # Ll [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +007B;Na # Ps LEFT CURLY BRACKET +007C;Na # Sm VERTICAL LINE +007D;Na # Pe RIGHT CURLY BRACKET +007E;Na # Sm TILDE +007F;N # Cc <control-007F> +0080..009F;N # Cc [32] <control-0080>..<control-009F> +00A0;N # Zs NO-BREAK SPACE +00A1;A # Po INVERTED EXCLAMATION MARK +00A2..00A3;Na # Sc [2] CENT SIGN..POUND SIGN +00A4;A # Sc CURRENCY SIGN +00A5;Na # Sc YEN SIGN +00A6;Na # So BROKEN BAR +00A7;A # Po SECTION SIGN +00A8;A # Sk DIAERESIS +00A9;N # So COPYRIGHT SIGN +00AA;A # Lo FEMININE ORDINAL INDICATOR +00AB;N # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00AC;Na # Sm NOT SIGN +00AD;A # Cf SOFT HYPHEN +00AE;A # So REGISTERED SIGN +00AF;Na # Sk MACRON +00B0;A # So DEGREE SIGN +00B1;A # Sm PLUS-MINUS SIGN +00B2..00B3;A # No [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE +00B4;A # Sk ACUTE ACCENT +00B5;N # Ll MICRO SIGN +00B6..00B7;A # Po [2] PILCROW SIGN..MIDDLE DOT +00B8;A # Sk CEDILLA +00B9;A # No SUPERSCRIPT ONE +00BA;A # Lo MASCULINE ORDINAL INDICATOR +00BB;N # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BC..00BE;A # No [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS +00BF;A # Po INVERTED QUESTION MARK +00C0..00C5;N # Lu [6] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER A WITH RING ABOVE +00C6;A # Lu LATIN CAPITAL LETTER AE +00C7..00CF;N # Lu [9] LATIN CAPITAL LETTER C WITH CEDILLA..LATIN CAPITAL LETTER I WITH DIAERESIS +00D0;A # Lu LATIN CAPITAL LETTER ETH +00D1..00D6;N # Lu [6] LATIN CAPITAL LETTER N WITH TILDE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D7;A # Sm MULTIPLICATION SIGN +00D8;A # Lu LATIN CAPITAL LETTER O WITH STROKE +00D9..00DD;N # Lu [5] LATIN CAPITAL LETTER U WITH GRAVE..LATIN CAPITAL LETTER Y WITH ACUTE +00DE..00E1;A # L& [4] LATIN CAPITAL LETTER THORN..LATIN SMALL LETTER A WITH ACUTE +00E2..00E5;N # Ll [4] LATIN SMALL LETTER A WITH CIRCUMFLEX..LATIN SMALL LETTER A WITH RING ABOVE +00E6;A # Ll LATIN SMALL LETTER AE +00E7;N # Ll LATIN SMALL LETTER C WITH CEDILLA +00E8..00EA;A # Ll [3] LATIN SMALL LETTER E WITH GRAVE..LATIN SMALL LETTER E WITH CIRCUMFLEX +00EB;N # Ll LATIN SMALL LETTER E WITH DIAERESIS +00EC..00ED;A # Ll [2] LATIN SMALL LETTER I WITH GRAVE..LATIN SMALL LETTER I WITH ACUTE +00EE..00EF;N # Ll [2] LATIN SMALL LETTER I WITH CIRCUMFLEX..LATIN SMALL LETTER I WITH DIAERESIS +00F0;A # Ll LATIN SMALL LETTER ETH +00F1;N # Ll LATIN SMALL LETTER N WITH TILDE +00F2..00F3;A # Ll [2] LATIN SMALL LETTER O WITH GRAVE..LATIN SMALL LETTER O WITH ACUTE +00F4..00F6;N # Ll [3] LATIN SMALL LETTER O WITH CIRCUMFLEX..LATIN SMALL LETTER O WITH DIAERESIS +00F7;A # Sm DIVISION SIGN +00F8..00FA;A # Ll [3] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER U WITH ACUTE +00FB;N # Ll LATIN SMALL LETTER U WITH CIRCUMFLEX +00FC;A # Ll LATIN SMALL LETTER U WITH DIAERESIS +00FD;N # Ll LATIN SMALL LETTER Y WITH ACUTE +00FE;A # Ll LATIN SMALL LETTER THORN +00FF;N # Ll LATIN SMALL LETTER Y WITH DIAERESIS +0100;N # Lu LATIN CAPITAL LETTER A WITH MACRON +0101;A # Ll LATIN SMALL LETTER A WITH MACRON +0102..0110;N # L& [15] LATIN CAPITAL LETTER A WITH BREVE..LATIN CAPITAL LETTER D WITH STROKE +0111;A # Ll LATIN SMALL LETTER D WITH STROKE +0112;N # Lu LATIN CAPITAL LETTER E WITH MACRON +0113;A # Ll LATIN SMALL LETTER E WITH MACRON +0114..011A;N # L& [7] LATIN CAPITAL LETTER E WITH BREVE..LATIN CAPITAL LETTER E WITH CARON +011B;A # Ll LATIN SMALL LETTER E WITH CARON +011C..0125;N # L& [10] LATIN CAPITAL LETTER G WITH CIRCUMFLEX..LATIN SMALL LETTER H WITH CIRCUMFLEX +0126..0127;A # L& [2] LATIN CAPITAL LETTER H WITH STROKE..LATIN SMALL LETTER H WITH STROKE +0128..012A;N # L& [3] LATIN CAPITAL LETTER I WITH TILDE..LATIN CAPITAL LETTER I WITH MACRON +012B;A # Ll LATIN SMALL LETTER I WITH MACRON +012C..0130;N # L& [5] LATIN CAPITAL LETTER I WITH BREVE..LATIN CAPITAL LETTER I WITH DOT ABOVE +0131..0133;A # L& [3] LATIN SMALL LETTER DOTLESS I..LATIN SMALL LIGATURE IJ +0134..0137;N # L& [4] LATIN CAPITAL LETTER J WITH CIRCUMFLEX..LATIN SMALL LETTER K WITH CEDILLA +0138;A # Ll LATIN SMALL LETTER KRA +0139..013E;N # L& [6] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER L WITH CARON +013F..0142;A # L& [4] LATIN CAPITAL LETTER L WITH MIDDLE DOT..LATIN SMALL LETTER L WITH STROKE +0143;N # Lu LATIN CAPITAL LETTER N WITH ACUTE +0144;A # Ll LATIN SMALL LETTER N WITH ACUTE +0145..0147;N # L& [3] LATIN CAPITAL LETTER N WITH CEDILLA..LATIN CAPITAL LETTER N WITH CARON +0148..014B;A # L& [4] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER ENG +014C;N # Lu LATIN CAPITAL LETTER O WITH MACRON +014D;A # Ll LATIN SMALL LETTER O WITH MACRON +014E..0151;N # L& [4] LATIN CAPITAL LETTER O WITH BREVE..LATIN SMALL LETTER O WITH DOUBLE ACUTE +0152..0153;A # L& [2] LATIN CAPITAL LIGATURE OE..LATIN SMALL LIGATURE OE +0154..0165;N # L& [18] LATIN CAPITAL LETTER R WITH ACUTE..LATIN SMALL LETTER T WITH CARON +0166..0167;A # L& [2] LATIN CAPITAL LETTER T WITH STROKE..LATIN SMALL LETTER T WITH STROKE +0168..016A;N # L& [3] LATIN CAPITAL LETTER U WITH TILDE..LATIN CAPITAL LETTER U WITH MACRON +016B;A # Ll LATIN SMALL LETTER U WITH MACRON +016C..017F;N # L& [20] LATIN CAPITAL LETTER U WITH BREVE..LATIN SMALL LETTER LONG S +0180..01BA;N # L& [59] LATIN SMALL LETTER B WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB;N # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF;N # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3;N # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..01CD;N # L& [10] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER A WITH CARON +01CE;A # Ll LATIN SMALL LETTER A WITH CARON +01CF;N # Lu LATIN CAPITAL LETTER I WITH CARON +01D0;A # Ll LATIN SMALL LETTER I WITH CARON +01D1;N # Lu LATIN CAPITAL LETTER O WITH CARON +01D2;A # Ll LATIN SMALL LETTER O WITH CARON +01D3;N # Lu LATIN CAPITAL LETTER U WITH CARON +01D4;A # Ll LATIN SMALL LETTER U WITH CARON +01D5;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D6;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D7;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D8;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01D9;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DA;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DB;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DC;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE +01DD..024F;N # L& [115] LATIN SMALL LETTER TURNED E..LATIN SMALL LETTER Y WITH STROKE +0250;N # Ll LATIN SMALL LETTER TURNED A +0251;A # Ll LATIN SMALL LETTER ALPHA +0252..0260;N # Ll [15] LATIN SMALL LETTER TURNED ALPHA..LATIN SMALL LETTER G WITH HOOK +0261;A # Ll LATIN SMALL LETTER SCRIPT G +0262..0293;N # Ll [50] LATIN LETTER SMALL CAPITAL G..LATIN SMALL LETTER EZH WITH CURL +0294;N # Lo LATIN LETTER GLOTTAL STOP +0295..02AF;N # Ll [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1;N # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C3;N # Sk [2] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER RIGHT ARROWHEAD +02C4;A # Sk MODIFIER LETTER UP ARROWHEAD +02C5;N # Sk MODIFIER LETTER DOWN ARROWHEAD +02C6;N # Lm MODIFIER LETTER CIRCUMFLEX ACCENT +02C7;A # Lm CARON +02C8;N # Lm MODIFIER LETTER VERTICAL LINE +02C9..02CB;A # Lm [3] MODIFIER LETTER MACRON..MODIFIER LETTER GRAVE ACCENT +02CC;N # Lm MODIFIER LETTER LOW VERTICAL LINE +02CD;A # Lm MODIFIER LETTER LOW MACRON +02CE..02CF;N # Lm [2] MODIFIER LETTER LOW GRAVE ACCENT..MODIFIER LETTER LOW ACUTE ACCENT +02D0;A # Lm MODIFIER LETTER TRIANGULAR COLON +02D1;N # Lm MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02D7;N # Sk [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN +02D8..02DB;A # Sk [4] BREVE..OGONEK +02DC;N # Sk SMALL TILDE +02DD;A # Sk DOUBLE ACUTE ACCENT +02DE;N # Sk MODIFIER LETTER RHOTIC HOOK +02DF;A # Sk MODIFIER LETTER CROSS ACCENT +02E0..02E4;N # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB;N # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC;N # Lm MODIFIER LETTER VOICING +02ED;N # Sk MODIFIER LETTER UNASPIRATED +02EE;N # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF;N # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0300..036F;A # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0370..0373;N # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374;N # Lm GREEK NUMERAL SIGN +0375;N # Sk GREEK LOWER NUMERAL SIGN +0376..0377;N # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A;N # Lm GREEK YPOGEGRAMMENI +037B..037D;N # Ll [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +037E;N # Po GREEK QUESTION MARK +037F;N # Lu GREEK CAPITAL LETTER YOT +0384..0385;N # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0386;N # Lu GREEK CAPITAL LETTER ALPHA WITH TONOS +0387;N # Po GREEK ANO TELEIA +0388..038A;N # Lu [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C;N # Lu GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..0390;N # L& [3] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391..03A1;A # Lu [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03A9;A # Lu [7] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER OMEGA +03AA..03B0;N # L& [7] GREEK CAPITAL LETTER IOTA WITH DIALYTIKA..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03B1..03C1;A # Ll [17] GREEK SMALL LETTER ALPHA..GREEK SMALL LETTER RHO +03C2;N # Ll GREEK SMALL LETTER FINAL SIGMA +03C3..03C9;A # Ll [7] GREEK SMALL LETTER SIGMA..GREEK SMALL LETTER OMEGA +03CA..03F5;N # L& [44] GREEK SMALL LETTER IOTA WITH DIALYTIKA..GREEK LUNATE EPSILON SYMBOL +03F6;N # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +03F7..03FF;N # L& [9] GREEK CAPITAL LETTER SHO..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400;N # Lu CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401;A # Lu CYRILLIC CAPITAL LETTER IO +0402..040F;N # Lu [14] CYRILLIC CAPITAL LETTER DJE..CYRILLIC CAPITAL LETTER DZHE +0410..044F;A # L& [64] CYRILLIC CAPITAL LETTER A..CYRILLIC SMALL LETTER YA +0450;N # Ll CYRILLIC SMALL LETTER IE WITH GRAVE +0451;A # Ll CYRILLIC SMALL LETTER IO +0452..0481;N # L& [48] CYRILLIC SMALL LETTER DJE..CYRILLIC SMALL LETTER KOPPA +0482;N # So CYRILLIC THOUSANDS SIGN +0483..0487;N # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0488..0489;N # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +048A..04FF;N # L& [118] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER HA WITH STROKE +0500..052F;N # L& [48] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER EL WITH DESCENDER +0531..0556;N # Lu [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559;N # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +055A..055F;N # Po [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK +0560..0588;N # Ll [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE +0589;N # Po ARMENIAN FULL STOP +058A;N # Pd ARMENIAN HYPHEN +058D..058E;N # So [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN +058F;N # Sc ARMENIAN DRAM SIGN +0591..05BD;N # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BE;N # Pd HEBREW PUNCTUATION MAQAF +05BF;N # Mn HEBREW POINT RAFE +05C0;N # Po HEBREW PUNCTUATION PASEQ +05C1..05C2;N # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C3;N # Po HEBREW PUNCTUATION SOF PASUQ +05C4..05C5;N # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C6;N # Po HEBREW PUNCTUATION NUN HAFUKHA +05C7;N # Mn HEBREW POINT QAMATS QATAN +05D0..05EA;N # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05EF..05F2;N # Lo [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD +05F3..05F4;N # Po [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM +0600..0605;N # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE +0606..0608;N # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +0609..060A;N # Po [2] ARABIC-INDIC PER MILLE SIGN..ARABIC-INDIC PER TEN THOUSAND SIGN +060B;N # Sc AFGHANI SIGN +060C..060D;N # Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR +060E..060F;N # So [2] ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA +0610..061A;N # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +061B;N # Po ARABIC SEMICOLON +061C;N # Cf ARABIC LETTER MARK +061D..061F;N # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +0620..063F;N # Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640;N # Lm ARABIC TATWEEL +0641..064A;N # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..065F;N # Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW +0660..0669;N # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066A..066D;N # Po [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR +066E..066F;N # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670;N # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3;N # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D4;N # Po ARABIC FULL STOP +06D5;N # Lo ARABIC LETTER AE +06D6..06DC;N # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DD;N # Cf ARABIC END OF AYAH +06DE;N # So ARABIC START OF RUB EL HIZB +06DF..06E4;N # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6;N # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8;N # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06E9;N # So ARABIC PLACE OF SAJDAH +06EA..06ED;N # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06EE..06EF;N # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9;N # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC;N # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FD..06FE;N # So [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN +06FF;N # Lo ARABIC LETTER HEH WITH INVERTED V +0700..070D;N # Po [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS +070F;N # Cf SYRIAC ABBREVIATION MARK +0710;N # Lo SYRIAC LETTER ALAPH +0711;N # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072F;N # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +0730..074A;N # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +074D..074F;N # Lo [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE +0750..077F;N # Lo [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE +0780..07A5;N # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0;N # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1;N # Lo THAANA LETTER NAA +07C0..07C9;N # Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE +07CA..07EA;N # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07EB..07F3;N # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5;N # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07F6;N # So NKO SYMBOL OO DENNEN +07F7..07F9;N # Po [3] NKO SYMBOL GBAKURUNEN..NKO EXCLAMATION MARK +07FA;N # Lm NKO LAJANYALAN +07FD;N # Mn NKO DANTAYALAN +07FE..07FF;N # Sc [2] NKO DOROME SIGN..NKO TAMAN SIGN +0800..0815;N # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +0816..0819;N # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081A;N # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823;N # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824;N # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827;N # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828;N # Lm SAMARITAN MODIFIER LETTER I +0829..082D;N # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0830..083E;N # Po [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +0840..0858;N # Lo [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN +0859..085B;N # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK +085E;N # Po MANDAIC PUNCTUATION +0860..086A;N # Lo [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA +0870..0887;N # Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT +0888;N # Sk ARABIC RAISED ROUND DOT +0889..088E;N # Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL +0890..0891;N # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +0898..089F;N # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08A0..08C8;N # Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF +08C9;N # Lm ARABIC SMALL FARSI YEH +08CA..08E1;N # Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA +08E2;N # Cf ARABIC DISPUTED END OF AYAH +08E3..08FF;N # Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA +0900..0902;N # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903;N # Mc DEVANAGARI SIGN VISARGA +0904..0939;N # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093A;N # Mn DEVANAGARI VOWEL SIGN OE +093B;N # Mc DEVANAGARI VOWEL SIGN OOE +093C;N # Mn DEVANAGARI SIGN NUKTA +093D;N # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940;N # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948;N # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C;N # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094D;N # Mn DEVANAGARI SIGN VIRAMA +094E..094F;N # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW +0950;N # Lo DEVANAGARI OM +0951..0957;N # Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE +0958..0961;N # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963;N # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0964..0965;N # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0966..096F;N # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0970;N # Po DEVANAGARI ABBREVIATION SIGN +0971;N # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972..097F;N # Lo [14] DEVANAGARI LETTER CANDRA A..DEVANAGARI LETTER BBA +0980;N # Lo BENGALI ANJI +0981;N # Mn BENGALI SIGN CANDRABINDU +0982..0983;N # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C;N # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990;N # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8;N # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0;N # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2;N # Lo BENGALI LETTER LA +09B6..09B9;N # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC;N # Mn BENGALI SIGN NUKTA +09BD;N # Lo BENGALI SIGN AVAGRAHA +09BE..09C0;N # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4;N # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8;N # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC;N # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CD;N # Mn BENGALI SIGN VIRAMA +09CE;N # Lo BENGALI LETTER KHANDA TA +09D7;N # Mc BENGALI AU LENGTH MARK +09DC..09DD;N # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1;N # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3;N # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09E6..09EF;N # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1;N # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +09F2..09F3;N # Sc [2] BENGALI RUPEE MARK..BENGALI RUPEE SIGN +09F4..09F9;N # No [6] BENGALI CURRENCY NUMERATOR ONE..BENGALI CURRENCY DENOMINATOR SIXTEEN +09FA;N # So BENGALI ISSHAR +09FB;N # Sc BENGALI GANDA MARK +09FC;N # Lo BENGALI LETTER VEDIC ANUSVARA +09FD;N # Po BENGALI ABBREVIATION SIGN +09FE;N # Mn BENGALI SANDHI MARK +0A01..0A02;N # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03;N # Mc GURMUKHI SIGN VISARGA +0A05..0A0A;N # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10;N # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28;N # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30;N # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33;N # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36;N # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39;N # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C;N # Mn GURMUKHI SIGN NUKTA +0A3E..0A40;N # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42;N # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48;N # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D;N # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51;N # Mn GURMUKHI SIGN UDAAT +0A59..0A5C;N # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E;N # Lo GURMUKHI LETTER FA +0A66..0A6F;N # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A70..0A71;N # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74;N # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A75;N # Mn GURMUKHI SIGN YAKASH +0A76;N # Po GURMUKHI ABBREVIATION SIGN +0A81..0A82;N # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83;N # Mc GUJARATI SIGN VISARGA +0A85..0A8D;N # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91;N # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8;N # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0;N # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3;N # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9;N # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC;N # Mn GUJARATI SIGN NUKTA +0ABD;N # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0;N # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5;N # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8;N # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9;N # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC;N # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0ACD;N # Mn GUJARATI SIGN VIRAMA +0AD0;N # Lo GUJARATI OM +0AE0..0AE1;N # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE2..0AE3;N # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AE6..0AEF;N # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0AF0;N # Po GUJARATI ABBREVIATION SIGN +0AF1;N # Sc GUJARATI RUPEE SIGN +0AF9;N # Lo GUJARATI LETTER ZHA +0AFA..0AFF;N # Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B01;N # Mn ORIYA SIGN CANDRABINDU +0B02..0B03;N # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C;N # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10;N # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28;N # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30;N # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33;N # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39;N # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3C;N # Mn ORIYA SIGN NUKTA +0B3D;N # Lo ORIYA SIGN AVAGRAHA +0B3E;N # Mc ORIYA VOWEL SIGN AA +0B3F;N # Mn ORIYA VOWEL SIGN I +0B40;N # Mc ORIYA VOWEL SIGN II +0B41..0B44;N # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48;N # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C;N # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B4D;N # Mn ORIYA SIGN VIRAMA +0B55..0B56;N # Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK +0B57;N # Mc ORIYA AU LENGTH MARK +0B5C..0B5D;N # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61;N # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B62..0B63;N # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B66..0B6F;N # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B70;N # So ORIYA ISSHAR +0B71;N # Lo ORIYA LETTER WA +0B72..0B77;N # No [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS +0B82;N # Mn TAMIL SIGN ANUSVARA +0B83;N # Lo TAMIL SIGN VISARGA +0B85..0B8A;N # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90;N # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95;N # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A;N # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C;N # Lo TAMIL LETTER JA +0B9E..0B9F;N # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4;N # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA;N # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9;N # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBE..0BBF;N # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0;N # Mn TAMIL VOWEL SIGN II +0BC1..0BC2;N # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8;N # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC;N # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BCD;N # Mn TAMIL SIGN VIRAMA +0BD0;N # Lo TAMIL OM +0BD7;N # Mc TAMIL AU LENGTH MARK +0BE6..0BEF;N # Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE +0BF0..0BF2;N # No [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND +0BF3..0BF8;N # So [6] TAMIL DAY SIGN..TAMIL AS ABOVE SIGN +0BF9;N # Sc TAMIL RUPEE SIGN +0BFA;N # So TAMIL NUMBER SIGN +0C00;N # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C01..0C03;N # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C04;N # Mn TELUGU SIGN COMBINING ANUSVARA ABOVE +0C05..0C0C;N # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10;N # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28;N # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C39;N # Lo [16] TELUGU LETTER PA..TELUGU LETTER HA +0C3C;N # Mn TELUGU SIGN NUKTA +0C3D;N # Lo TELUGU SIGN AVAGRAHA +0C3E..0C40;N # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44;N # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48;N # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D;N # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56;N # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C58..0C5A;N # Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA +0C5D;N # Lo TELUGU LETTER NAKAARA POLLU +0C60..0C61;N # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C62..0C63;N # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C66..0C6F;N # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C77;N # Po TELUGU SIGN SIDDHAM +0C78..0C7E;N # No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR +0C7F;N # So TELUGU SIGN TUUMU +0C80;N # Lo KANNADA SIGN SPACING CANDRABINDU +0C81;N # Mn KANNADA SIGN CANDRABINDU +0C82..0C83;N # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C84;N # Po KANNADA SIGN SIDDHAM +0C85..0C8C;N # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90;N # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8;N # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3;N # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9;N # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBC;N # Mn KANNADA SIGN NUKTA +0CBD;N # Lo KANNADA SIGN AVAGRAHA +0CBE;N # Mc KANNADA VOWEL SIGN AA +0CBF;N # Mn KANNADA VOWEL SIGN I +0CC0..0CC4;N # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6;N # Mn KANNADA VOWEL SIGN E +0CC7..0CC8;N # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB;N # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC..0CCD;N # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CD5..0CD6;N # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDD..0CDE;N # Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA +0CE0..0CE1;N # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3;N # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CE6..0CEF;N # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0CF1..0CF2;N # Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA +0D00..0D01;N # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU +0D02..0D03;N # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D04..0D0C;N # Lo [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L +0D0E..0D10;N # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D3A;N # Lo [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA +0D3B..0D3C;N # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D3D;N # Lo MALAYALAM SIGN AVAGRAHA +0D3E..0D40;N # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44;N # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48;N # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C;N # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D4D;N # Mn MALAYALAM SIGN VIRAMA +0D4E;N # Lo MALAYALAM LETTER DOT REPH +0D4F;N # So MALAYALAM SIGN PARA +0D54..0D56;N # Lo [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL +0D57;N # Mc MALAYALAM AU LENGTH MARK +0D58..0D5E;N # No [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH +0D5F..0D61;N # Lo [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL +0D62..0D63;N # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D66..0D6F;N # Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D70..0D78;N # No [9] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE SIXTEENTHS +0D79;N # So MALAYALAM DATE MARK +0D7A..0D7F;N # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D81;N # Mn SINHALA SIGN CANDRABINDU +0D82..0D83;N # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96;N # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1;N # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB;N # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD;N # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6;N # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA;N # Mn SINHALA SIGN AL-LAKUNA +0DCF..0DD1;N # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4;N # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6;N # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF;N # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DE6..0DEF;N # Nd [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE +0DF2..0DF3;N # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0DF4;N # Po SINHALA PUNCTUATION KUNDDALIYA +0E01..0E30;N # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31;N # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33;N # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A;N # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E3F;N # Sc THAI CURRENCY SYMBOL BAHT +0E40..0E45;N # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46;N # Lm THAI CHARACTER MAIYAMOK +0E47..0E4E;N # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0E4F;N # Po THAI CHARACTER FONGMAN +0E50..0E59;N # Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E5A..0E5B;N # Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0E81..0E82;N # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84;N # Lo LAO LETTER KHO TAM +0E86..0E8A;N # Lo [5] LAO LETTER PALI GHA..LAO LETTER SO TAM +0E8C..0EA3;N # Lo [24] LAO LETTER PALI JHA..LAO LETTER LO LING +0EA5;N # Lo LAO LETTER LO LOOT +0EA7..0EB0;N # Lo [10] LAO LETTER WO..LAO VOWEL SIGN A +0EB1;N # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3;N # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EBC;N # Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO +0EBD;N # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4;N # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6;N # Lm LAO KO LA +0EC8..0ECD;N # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0ED0..0ED9;N # Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDC..0EDF;N # Lo [4] LAO HO NO..LAO LETTER KHMU NYO +0F00;N # Lo TIBETAN SYLLABLE OM +0F01..0F03;N # So [3] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA +0F04..0F12;N # Po [15] TIBETAN MARK INITIAL YIG MGO MDUN MA..TIBETAN MARK RGYA GRAM SHAD +0F13;N # So TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN +0F14;N # Po TIBETAN MARK GTER TSHEG +0F15..0F17;N # So [3] TIBETAN LOGOTYPE SIGN CHAD RTAGS..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS +0F18..0F19;N # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F1A..0F1F;N # So [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG +0F20..0F29;N # Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F2A..0F33;N # No [10] TIBETAN DIGIT HALF ONE..TIBETAN DIGIT HALF ZERO +0F34;N # So TIBETAN MARK BSDUS RTAGS +0F35;N # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F36;N # So TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN +0F37;N # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F38;N # So TIBETAN MARK CHE MGO +0F39;N # Mn TIBETAN MARK TSA -PHRU +0F3A;N # Ps TIBETAN MARK GUG RTAGS GYON +0F3B;N # Pe TIBETAN MARK GUG RTAGS GYAS +0F3C;N # Ps TIBETAN MARK ANG KHANG GYON +0F3D;N # Pe TIBETAN MARK ANG KHANG GYAS +0F3E..0F3F;N # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F40..0F47;N # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C;N # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F71..0F7E;N # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F;N # Mc TIBETAN SIGN RNAM BCAD +0F80..0F84;N # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F85;N # Po TIBETAN MARK PALUTA +0F86..0F87;N # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F88..0F8C;N # Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN +0F8D..0F97;N # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA +0F99..0FBC;N # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FBE..0FC5;N # So [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE +0FC6;N # Mn TIBETAN SYMBOL PADMA GDAN +0FC7..0FCC;N # So [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL +0FCE..0FCF;N # So [2] TIBETAN SIGN RDEL NAG RDEL DKAR..TIBETAN SIGN RDEL NAG GSUM +0FD0..0FD4;N # Po [5] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA +0FD5..0FD8;N # So [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS +0FD9..0FDA;N # Po [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS +1000..102A;N # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C;N # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030;N # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031;N # Mc MYANMAR VOWEL SIGN E +1032..1037;N # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1038;N # Mc MYANMAR SIGN VISARGA +1039..103A;N # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103B..103C;N # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E;N # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +103F;N # Lo MYANMAR LETTER GREAT SA +1040..1049;N # Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +104A..104F;N # Po [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE +1050..1055;N # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057;N # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059;N # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105A..105D;N # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +105E..1060;N # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1061;N # Lo MYANMAR LETTER SGAW KAREN SHA +1062..1064;N # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1065..1066;N # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..106D;N # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +106E..1070;N # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1071..1074;N # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1075..1081;N # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1082;N # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084;N # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086;N # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C;N # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D;N # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108E;N # Lo MYANMAR LETTER RUMAI PALAUNG FA +108F;N # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +1090..1099;N # Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109C;N # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D;N # Mn MYANMAR VOWEL SIGN AITON AI +109E..109F;N # So [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION +10A0..10C5;N # Lu [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10C7;N # Lu GEORGIAN CAPITAL LETTER YN +10CD;N # Lu GEORGIAN CAPITAL LETTER AEN +10D0..10FA;N # Ll [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FB;N # Po GEORGIAN PARAGRAPH SEPARATOR +10FC;N # Lm MODIFIER LETTER GEORGIAN NAR +10FD..10FF;N # Ll [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN +1100..115F;W # Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER +1160..11FF;N # Lo [160] HANGUL JUNGSEONG FILLER..HANGUL JONGSEONG SSANGNIEUN +1200..1248;N # Lo [73] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE QWA +124A..124D;N # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256;N # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258;N # Lo ETHIOPIC SYLLABLE QHWA +125A..125D;N # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288;N # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D;N # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0;N # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5;N # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE;N # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0;N # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5;N # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6;N # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310;N # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315;N # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A;N # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +135D..135F;N # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK +1360..1368;N # Po [9] ETHIOPIC SECTION MARK..ETHIOPIC PARAGRAPH SEPARATOR +1369..137C;N # No [20] ETHIOPIC DIGIT ONE..ETHIOPIC NUMBER TEN THOUSAND +1380..138F;N # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +1390..1399;N # So [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT +13A0..13F5;N # Lu [86] CHEROKEE LETTER A..CHEROKEE LETTER MV +13F8..13FD;N # Ll [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV +1400;N # Pd CANADIAN SYLLABICS HYPHEN +1401..166C;N # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166D;N # So CANADIAN SYLLABICS CHI SIGN +166E;N # Po CANADIAN SYLLABICS FULL STOP +166F..167F;N # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1680;N # Zs OGHAM SPACE MARK +1681..169A;N # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +169B;N # Ps OGHAM FEATHER MARK +169C;N # Pe OGHAM REVERSED FEATHER MARK +16A0..16EA;N # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EB..16ED;N # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +16EE..16F0;N # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +16F1..16F8;N # Lo [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC +1700..1711;N # Lo [18] TAGALOG LETTER A..TAGALOG LETTER HA +1712..1714;N # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1715;N # Mc TAGALOG SIGN PAMUDPOD +171F;N # Lo TAGALOG LETTER ARCHAIC RA +1720..1731;N # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733;N # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1734;N # Mc HANUNOO SIGN PAMUDPOD +1735..1736;N # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1740..1751;N # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753;N # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C;N # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770;N # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773;N # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3;N # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B4..17B5;N # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +17B6;N # Mc KHMER VOWEL SIGN AA +17B7..17BD;N # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5;N # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6;N # Mn KHMER SIGN NIKAHIT +17C7..17C8;N # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17C9..17D3;N # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17D4..17D6;N # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17D7;N # Lm KHMER SIGN LEK TOO +17D8..17DA;N # Po [3] KHMER SIGN BEYYAL..KHMER SIGN KOOMUUT +17DB;N # Sc KHMER CURRENCY SYMBOL RIEL +17DC;N # Lo KHMER SIGN AVAKRAHASANYA +17DD;N # Mn KHMER SIGN ATTHACAN +17E0..17E9;N # Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +17F0..17F9;N # No [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON +1800..1805;N # Po [6] MONGOLIAN BIRGA..MONGOLIAN FOUR DOTS +1806;N # Pd MONGOLIAN TODO SOFT HYPHEN +1807..180A;N # Po [4] MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER..MONGOLIAN NIRUGU +180B..180D;N # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180E;N # Cf MONGOLIAN VOWEL SEPARATOR +180F;N # Mn MONGOLIAN FREE VARIATION SELECTOR FOUR +1810..1819;N # Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1842;N # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843;N # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1878;N # Lo [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS +1880..1884;N # Lo [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA +1885..1886;N # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +1887..18A8;N # Lo [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9;N # Mn MONGOLIAN LETTER ALI GALI DAGALGA +18AA;N # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5;N # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191E;N # Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA +1920..1922;N # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926;N # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928;N # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B;N # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931;N # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932;N # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938;N # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1939..193B;N # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1940;N # So LIMBU SIGN LOO +1944..1945;N # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1946..194F;N # Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE +1950..196D;N # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974;N # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB;N # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C9;N # Lo [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 +19D0..19D9;N # Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE +19DA;N # No NEW TAI LUE THAM DIGIT ONE +19DE..19DF;N # So [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV +19E0..19FF;N # So [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC +1A00..1A16;N # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A17..1A18;N # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1A;N # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O +1A1B;N # Mn BUGINESE VOWEL SIGN AE +1A1E..1A1F;N # Po [2] BUGINESE PALLAWA..BUGINESE END OF SECTION +1A20..1A54;N # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55;N # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56;N # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57;N # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E;N # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60;N # Mn TAI THAM SIGN SAKOT +1A61;N # Mc TAI THAM VOWEL SIGN A +1A62;N # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64;N # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C;N # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72;N # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A7C;N # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F;N # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1A80..1A89;N # Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE +1A90..1A99;N # Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA0..1AA6;N # Po [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA +1AA7;N # Lm TAI THAM SIGN MAI YAMOK +1AA8..1AAD;N # Po [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG +1AB0..1ABD;N # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1ABE;N # Me COMBINING PARENTHESES OVERLAY +1ABF..1ACE;N # Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T +1B00..1B03;N # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04;N # Mc BALINESE SIGN BISAH +1B05..1B33;N # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B34;N # Mn BALINESE SIGN REREKAN +1B35;N # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A;N # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B;N # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C;N # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41;N # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42;N # Mn BALINESE VOWEL SIGN PEPET +1B43..1B44;N # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1B45..1B4C;N # Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA +1B50..1B59;N # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B5A..1B60;N # Po [7] BALINESE PANTI..BALINESE PAMENENG +1B61..1B6A;N # So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE +1B6B..1B73;N # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B74..1B7C;N # So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING +1B7D..1B7E;N # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1B80..1B81;N # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82;N # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0;N # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1;N # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5;N # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7;N # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9;N # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAA;N # Mc SUNDANESE SIGN PAMAAEH +1BAB..1BAD;N # Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BAE..1BAF;N # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1BB0..1BB9;N # Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE +1BBA..1BBF;N # Lo [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M +1BC0..1BE5;N # Lo [38] BATAK LETTER A..BATAK LETTER U +1BE6;N # Mn BATAK SIGN TOMPI +1BE7;N # Mc BATAK VOWEL SIGN E +1BE8..1BE9;N # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE +1BEA..1BEC;N # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O +1BED;N # Mn BATAK VOWEL SIGN KARO O +1BEE;N # Mc BATAK VOWEL SIGN U +1BEF..1BF1;N # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H +1BF2..1BF3;N # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN +1BFC..1BFF;N # Po [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT +1C00..1C23;N # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B;N # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33;N # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35;N # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36..1C37;N # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C3B..1C3F;N # Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C40..1C49;N # Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C4F;N # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C50..1C59;N # Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE +1C5A..1C77;N # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D;N # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1C7E..1C7F;N # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +1C80..1C88;N # Ll [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1C90..1CBA;N # Lu [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD..1CBF;N # Lu [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1CC0..1CC7;N # Po [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA +1CD0..1CD2;N # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD3;N # Po VEDIC SIGN NIHSHVASA +1CD4..1CE0;N # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1;N # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8;N # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CE9..1CEC;N # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CED;N # Mn VEDIC SIGN TIRYAK +1CEE..1CF3;N # Lo [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA +1CF4;N # Mn VEDIC TONE CANDRA ABOVE +1CF5..1CF6;N # Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA +1CF7;N # Mc VEDIC SIGN ATIKRAMA +1CF8..1CF9;N # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +1CFA;N # Lo VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA +1D00..1D2B;N # Ll [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D6A;N # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1D6B..1D77;N # Ll [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G +1D78;N # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D7F;N # Ll [7] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER UPSILON WITH STROKE +1D80..1D9A;N # Ll [27] LATIN SMALL LETTER B WITH PALATAL HOOK..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF;N # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1DC0..1DFF;N # Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E00..1EFF;N # L& [256] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER Y WITH LOOP +1F00..1F15;N # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D;N # Lu [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45;N # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D;N # Lu [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57;N # Ll [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D;N # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4;N # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC;N # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBD;N # Sk GREEK KORONIS +1FBE;N # Ll GREEK PROSGEGRAMMENI +1FBF..1FC1;N # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FC2..1FC4;N # Ll [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC;N # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCD..1FCF;N # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FD0..1FD3;N # Ll [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB;N # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD..1FDF;N # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FE0..1FEC;N # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FED..1FEF;N # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FF2..1FF4;N # Ll [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC;N # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFD..1FFE;N # Sk [2] GREEK OXIA..GREEK DASIA +2000..200A;N # Zs [11] EN QUAD..HAIR SPACE +200B..200F;N # Cf [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK +2010;A # Pd HYPHEN +2011..2012;N # Pd [2] NON-BREAKING HYPHEN..FIGURE DASH +2013..2015;A # Pd [3] EN DASH..HORIZONTAL BAR +2016;A # Po DOUBLE VERTICAL LINE +2017;N # Po DOUBLE LOW LINE +2018;A # Pi LEFT SINGLE QUOTATION MARK +2019;A # Pf RIGHT SINGLE QUOTATION MARK +201A;N # Ps SINGLE LOW-9 QUOTATION MARK +201B;N # Pi SINGLE HIGH-REVERSED-9 QUOTATION MARK +201C;A # Pi LEFT DOUBLE QUOTATION MARK +201D;A # Pf RIGHT DOUBLE QUOTATION MARK +201E;N # Ps DOUBLE LOW-9 QUOTATION MARK +201F;N # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2020..2022;A # Po [3] DAGGER..BULLET +2023;N # Po TRIANGULAR BULLET +2024..2027;A # Po [4] ONE DOT LEADER..HYPHENATION POINT +2028;N # Zl LINE SEPARATOR +2029;N # Zp PARAGRAPH SEPARATOR +202A..202E;N # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +202F;N # Zs NARROW NO-BREAK SPACE +2030;A # Po PER MILLE SIGN +2031;N # Po PER TEN THOUSAND SIGN +2032..2033;A # Po [2] PRIME..DOUBLE PRIME +2034;N # Po TRIPLE PRIME +2035;A # Po REVERSED PRIME +2036..2038;N # Po [3] REVERSED DOUBLE PRIME..CARET +2039;N # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A;N # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +203B;A # Po REFERENCE MARK +203C..203D;N # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +203E;A # Po OVERLINE +203F..2040;N # Pc [2] UNDERTIE..CHARACTER TIE +2041..2043;N # Po [3] CARET INSERTION POINT..HYPHEN BULLET +2044;N # Sm FRACTION SLASH +2045;N # Ps LEFT SQUARE BRACKET WITH QUILL +2046;N # Pe RIGHT SQUARE BRACKET WITH QUILL +2047..2051;N # Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY +2052;N # Sm COMMERCIAL MINUS SIGN +2053;N # Po SWUNG DASH +2054;N # Pc INVERTED UNDERTIE +2055..205E;N # Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS +205F;N # Zs MEDIUM MATHEMATICAL SPACE +2060..2064;N # Cf [5] WORD JOINER..INVISIBLE PLUS +2066..206F;N # Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES +2070;N # No SUPERSCRIPT ZERO +2071;N # Lm SUPERSCRIPT LATIN SMALL LETTER I +2074;A # No SUPERSCRIPT FOUR +2075..2079;N # No [5] SUPERSCRIPT FIVE..SUPERSCRIPT NINE +207A..207C;N # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D;N # Ps SUPERSCRIPT LEFT PARENTHESIS +207E;N # Pe SUPERSCRIPT RIGHT PARENTHESIS +207F;A # Lm SUPERSCRIPT LATIN SMALL LETTER N +2080;N # No SUBSCRIPT ZERO +2081..2084;A # No [4] SUBSCRIPT ONE..SUBSCRIPT FOUR +2085..2089;N # No [5] SUBSCRIPT FIVE..SUBSCRIPT NINE +208A..208C;N # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D;N # Ps SUBSCRIPT LEFT PARENTHESIS +208E;N # Pe SUBSCRIPT RIGHT PARENTHESIS +2090..209C;N # Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T +20A0..20A8;N # Sc [9] EURO-CURRENCY SIGN..RUPEE SIGN +20A9;H # Sc WON SIGN +20AA..20AB;N # Sc [2] NEW SHEQEL SIGN..DONG SIGN +20AC;A # Sc EURO SIGN +20AD..20C0;N # Sc [20] KIP SIGN..SOM SIGN +20D0..20DC;N # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20DD..20E0;N # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E1;N # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E2..20E4;N # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE +20E5..20F0;N # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2100..2101;N # So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT +2102;N # Lu DOUBLE-STRUCK CAPITAL C +2103;A # So DEGREE CELSIUS +2104;N # So CENTRE LINE SYMBOL +2105;A # So CARE OF +2106;N # So CADA UNA +2107;N # Lu EULER CONSTANT +2108;N # So SCRUPLE +2109;A # So DEGREE FAHRENHEIT +210A..2112;N # L& [9] SCRIPT SMALL G..SCRIPT CAPITAL L +2113;A # Ll SCRIPT SMALL L +2114;N # So L B BAR SYMBOL +2115;N # Lu DOUBLE-STRUCK CAPITAL N +2116;A # So NUMERO SIGN +2117;N # So SOUND RECORDING COPYRIGHT +2118;N # Sm SCRIPT CAPITAL P +2119..211D;N # Lu [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +211E..2120;N # So [3] PRESCRIPTION TAKE..SERVICE MARK +2121..2122;A # So [2] TELEPHONE SIGN..TRADE MARK SIGN +2123;N # So VERSICLE +2124;N # Lu DOUBLE-STRUCK CAPITAL Z +2125;N # So OUNCE SIGN +2126;A # Lu OHM SIGN +2127;N # So INVERTED OHM SIGN +2128;N # Lu BLACK-LETTER CAPITAL Z +2129;N # So TURNED GREEK SMALL LETTER IOTA +212A;N # Lu KELVIN SIGN +212B;A # Lu ANGSTROM SIGN +212C..212D;N # Lu [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212E;N # So ESTIMATED SYMBOL +212F..2134;N # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138;N # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139;N # Ll INFORMATION SOURCE +213A..213B;N # So [2] ROTATED CAPITAL Q..FACSIMILE SIGN +213C..213F;N # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144;N # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149;N # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214A;N # So PROPERTY LINE +214B;N # Sm TURNED AMPERSAND +214C..214D;N # So [2] PER SIGN..AKTIESELSKAB +214E;N # Ll TURNED SMALL F +214F;N # So SYMBOL FOR SAMARITAN SOURCE +2150..2152;N # No [3] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE TENTH +2153..2154;A # No [2] VULGAR FRACTION ONE THIRD..VULGAR FRACTION TWO THIRDS +2155..215A;N # No [6] VULGAR FRACTION ONE FIFTH..VULGAR FRACTION FIVE SIXTHS +215B..215E;A # No [4] VULGAR FRACTION ONE EIGHTH..VULGAR FRACTION SEVEN EIGHTHS +215F;N # No FRACTION NUMERATOR ONE +2160..216B;A # Nl [12] ROMAN NUMERAL ONE..ROMAN NUMERAL TWELVE +216C..216F;N # Nl [4] ROMAN NUMERAL FIFTY..ROMAN NUMERAL ONE THOUSAND +2170..2179;A # Nl [10] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL TEN +217A..2182;N # Nl [9] SMALL ROMAN NUMERAL ELEVEN..ROMAN NUMERAL TEN THOUSAND +2183..2184;N # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188;N # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2189;A # No VULGAR FRACTION ZERO THIRDS +218A..218B;N # So [2] TURNED DIGIT TWO..TURNED DIGIT THREE +2190..2194;A # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199;A # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B;N # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F;N # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0;N # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2;N # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3;N # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5;N # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6;N # Sm RIGHTWARDS ARROW FROM BAR +21A7..21AD;N # So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW +21AE;N # Sm LEFT RIGHT ARROW WITH STROKE +21AF..21B7;N # So [9] DOWNWARDS ZIGZAG ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21B8..21B9;A # So [2] NORTH WEST ARROW TO LONG BAR..LEFTWARDS ARROW TO BAR OVER RIGHTWARDS ARROW TO BAR +21BA..21CD;N # So [20] ANTICLOCKWISE OPEN CIRCLE ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF;N # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1;N # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2;A # Sm RIGHTWARDS DOUBLE ARROW +21D3;N # So DOWNWARDS DOUBLE ARROW +21D4;A # Sm LEFT RIGHT DOUBLE ARROW +21D5..21E6;N # So [18] UP DOWN DOUBLE ARROW..LEFTWARDS WHITE ARROW +21E7;A # So UPWARDS WHITE ARROW +21E8..21F3;N # So [12] RIGHTWARDS WHITE ARROW..UP DOWN WHITE ARROW +21F4..21FF;N # Sm [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW +2200;A # Sm FOR ALL +2201;N # Sm COMPLEMENT +2202..2203;A # Sm [2] PARTIAL DIFFERENTIAL..THERE EXISTS +2204..2206;N # Sm [3] THERE DOES NOT EXIST..INCREMENT +2207..2208;A # Sm [2] NABLA..ELEMENT OF +2209..220A;N # Sm [2] NOT AN ELEMENT OF..SMALL ELEMENT OF +220B;A # Sm CONTAINS AS MEMBER +220C..220E;N # Sm [3] DOES NOT CONTAIN AS MEMBER..END OF PROOF +220F;A # Sm N-ARY PRODUCT +2210;N # Sm N-ARY COPRODUCT +2211;A # Sm N-ARY SUMMATION +2212..2214;N # Sm [3] MINUS SIGN..DOT PLUS +2215;A # Sm DIVISION SLASH +2216..2219;N # Sm [4] SET MINUS..BULLET OPERATOR +221A;A # Sm SQUARE ROOT +221B..221C;N # Sm [2] CUBE ROOT..FOURTH ROOT +221D..2220;A # Sm [4] PROPORTIONAL TO..ANGLE +2221..2222;N # Sm [2] MEASURED ANGLE..SPHERICAL ANGLE +2223;A # Sm DIVIDES +2224;N # Sm DOES NOT DIVIDE +2225;A # Sm PARALLEL TO +2226;N # Sm NOT PARALLEL TO +2227..222C;A # Sm [6] LOGICAL AND..DOUBLE INTEGRAL +222D;N # Sm TRIPLE INTEGRAL +222E;A # Sm CONTOUR INTEGRAL +222F..2233;N # Sm [5] SURFACE INTEGRAL..ANTICLOCKWISE CONTOUR INTEGRAL +2234..2237;A # Sm [4] THEREFORE..PROPORTION +2238..223B;N # Sm [4] DOT MINUS..HOMOTHETIC +223C..223D;A # Sm [2] TILDE OPERATOR..REVERSED TILDE +223E..2247;N # Sm [10] INVERTED LAZY S..NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO +2248;A # Sm ALMOST EQUAL TO +2249..224B;N # Sm [3] NOT ALMOST EQUAL TO..TRIPLE TILDE +224C;A # Sm ALL EQUAL TO +224D..2251;N # Sm [5] EQUIVALENT TO..GEOMETRICALLY EQUAL TO +2252;A # Sm APPROXIMATELY EQUAL TO OR THE IMAGE OF +2253..225F;N # Sm [13] IMAGE OF OR APPROXIMATELY EQUAL TO..QUESTIONED EQUAL TO +2260..2261;A # Sm [2] NOT EQUAL TO..IDENTICAL TO +2262..2263;N # Sm [2] NOT IDENTICAL TO..STRICTLY EQUIVALENT TO +2264..2267;A # Sm [4] LESS-THAN OR EQUAL TO..GREATER-THAN OVER EQUAL TO +2268..2269;N # Sm [2] LESS-THAN BUT NOT EQUAL TO..GREATER-THAN BUT NOT EQUAL TO +226A..226B;A # Sm [2] MUCH LESS-THAN..MUCH GREATER-THAN +226C..226D;N # Sm [2] BETWEEN..NOT EQUIVALENT TO +226E..226F;A # Sm [2] NOT LESS-THAN..NOT GREATER-THAN +2270..2281;N # Sm [18] NEITHER LESS-THAN NOR EQUAL TO..DOES NOT SUCCEED +2282..2283;A # Sm [2] SUBSET OF..SUPERSET OF +2284..2285;N # Sm [2] NOT A SUBSET OF..NOT A SUPERSET OF +2286..2287;A # Sm [2] SUBSET OF OR EQUAL TO..SUPERSET OF OR EQUAL TO +2288..2294;N # Sm [13] NEITHER A SUBSET OF NOR EQUAL TO..SQUARE CUP +2295;A # Sm CIRCLED PLUS +2296..2298;N # Sm [3] CIRCLED MINUS..CIRCLED DIVISION SLASH +2299;A # Sm CIRCLED DOT OPERATOR +229A..22A4;N # Sm [11] CIRCLED RING OPERATOR..DOWN TACK +22A5;A # Sm UP TACK +22A6..22BE;N # Sm [25] ASSERTION..RIGHT ANGLE WITH ARC +22BF;A # Sm RIGHT TRIANGLE +22C0..22FF;N # Sm [64] N-ARY LOGICAL AND..Z NOTATION BAG MEMBERSHIP +2300..2307;N # So [8] DIAMETER SIGN..WAVY LINE +2308;N # Ps LEFT CEILING +2309;N # Pe RIGHT CEILING +230A;N # Ps LEFT FLOOR +230B;N # Pe RIGHT FLOOR +230C..2311;N # So [6] BOTTOM RIGHT CROP..SQUARE LOZENGE +2312;A # So ARC +2313..2319;N # So [7] SEGMENT..TURNED NOT SIGN +231A..231B;W # So [2] WATCH..HOURGLASS +231C..231F;N # So [4] TOP LEFT CORNER..BOTTOM RIGHT CORNER +2320..2321;N # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +2322..2328;N # So [7] FROWN..KEYBOARD +2329;W # Ps LEFT-POINTING ANGLE BRACKET +232A;W # Pe RIGHT-POINTING ANGLE BRACKET +232B..237B;N # So [81] ERASE TO THE LEFT..NOT CHECK MARK +237C;N # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A;N # So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23B3;N # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23DB;N # So [40] TOP SQUARE BRACKET..FUSE +23DC..23E1;N # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2..23E8;N # So [7] WHITE TRAPEZIUM..DECIMAL EXPONENT SYMBOL +23E9..23EC;W # So [4] BLACK RIGHT-POINTING DOUBLE TRIANGLE..BLACK DOWN-POINTING DOUBLE TRIANGLE +23ED..23EF;N # So [3] BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR..BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR +23F0;W # So ALARM CLOCK +23F1..23F2;N # So [2] STOPWATCH..TIMER CLOCK +23F3;W # So HOURGLASS WITH FLOWING SAND +23F4..23FF;N # So [12] BLACK MEDIUM LEFT-POINTING TRIANGLE..OBSERVER EYE SYMBOL +2400..2426;N # So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO +2440..244A;N # So [11] OCR HOOK..OCR DOUBLE BACKSLASH +2460..249B;A # No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP +249C..24E9;A # So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +24EA;N # No CIRCLED DIGIT ZERO +24EB..24FF;A # No [21] NEGATIVE CIRCLED NUMBER ELEVEN..NEGATIVE CIRCLED DIGIT ZERO +2500..254B;A # So [76] BOX DRAWINGS LIGHT HORIZONTAL..BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL +254C..254F;N # So [4] BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL..BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL +2550..2573;A # So [36] BOX DRAWINGS DOUBLE HORIZONTAL..BOX DRAWINGS LIGHT DIAGONAL CROSS +2574..257F;N # So [12] BOX DRAWINGS LIGHT LEFT..BOX DRAWINGS HEAVY UP AND LIGHT DOWN +2580..258F;A # So [16] UPPER HALF BLOCK..LEFT ONE EIGHTH BLOCK +2590..2591;N # So [2] RIGHT HALF BLOCK..LIGHT SHADE +2592..2595;A # So [4] MEDIUM SHADE..RIGHT ONE EIGHTH BLOCK +2596..259F;N # So [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT +25A0..25A1;A # So [2] BLACK SQUARE..WHITE SQUARE +25A2;N # So WHITE SQUARE WITH ROUNDED CORNERS +25A3..25A9;A # So [7] WHITE SQUARE CONTAINING BLACK SMALL SQUARE..SQUARE WITH DIAGONAL CROSSHATCH FILL +25AA..25B1;N # So [8] BLACK SMALL SQUARE..WHITE PARALLELOGRAM +25B2..25B3;A # So [2] BLACK UP-POINTING TRIANGLE..WHITE UP-POINTING TRIANGLE +25B4..25B5;N # So [2] BLACK UP-POINTING SMALL TRIANGLE..WHITE UP-POINTING SMALL TRIANGLE +25B6;A # So BLACK RIGHT-POINTING TRIANGLE +25B7;A # Sm WHITE RIGHT-POINTING TRIANGLE +25B8..25BB;N # So [4] BLACK RIGHT-POINTING SMALL TRIANGLE..WHITE RIGHT-POINTING POINTER +25BC..25BD;A # So [2] BLACK DOWN-POINTING TRIANGLE..WHITE DOWN-POINTING TRIANGLE +25BE..25BF;N # So [2] BLACK DOWN-POINTING SMALL TRIANGLE..WHITE DOWN-POINTING SMALL TRIANGLE +25C0;A # So BLACK LEFT-POINTING TRIANGLE +25C1;A # Sm WHITE LEFT-POINTING TRIANGLE +25C2..25C5;N # So [4] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE LEFT-POINTING POINTER +25C6..25C8;A # So [3] BLACK DIAMOND..WHITE DIAMOND CONTAINING BLACK SMALL DIAMOND +25C9..25CA;N # So [2] FISHEYE..LOZENGE +25CB;A # So WHITE CIRCLE +25CC..25CD;N # So [2] DOTTED CIRCLE..CIRCLE WITH VERTICAL FILL +25CE..25D1;A # So [4] BULLSEYE..CIRCLE WITH RIGHT HALF BLACK +25D2..25E1;N # So [16] CIRCLE WITH LOWER HALF BLACK..LOWER HALF CIRCLE +25E2..25E5;A # So [4] BLACK LOWER RIGHT TRIANGLE..BLACK UPPER RIGHT TRIANGLE +25E6..25EE;N # So [9] WHITE BULLET..UP-POINTING TRIANGLE WITH RIGHT HALF BLACK +25EF;A # So LARGE CIRCLE +25F0..25F7;N # So [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FC;N # Sm [5] UPPER LEFT TRIANGLE..BLACK MEDIUM SQUARE +25FD..25FE;W # Sm [2] WHITE MEDIUM SMALL SQUARE..BLACK MEDIUM SMALL SQUARE +25FF;N # Sm LOWER RIGHT TRIANGLE +2600..2604;N # So [5] BLACK SUN WITH RAYS..COMET +2605..2606;A # So [2] BLACK STAR..WHITE STAR +2607..2608;N # So [2] LIGHTNING..THUNDERSTORM +2609;A # So SUN +260A..260D;N # So [4] ASCENDING NODE..OPPOSITION +260E..260F;A # So [2] BLACK TELEPHONE..WHITE TELEPHONE +2610..2613;N # So [4] BALLOT BOX..SALTIRE +2614..2615;W # So [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE +2616..261B;N # So [6] WHITE SHOGI PIECE..BLACK RIGHT POINTING INDEX +261C;A # So WHITE LEFT POINTING INDEX +261D;N # So WHITE UP POINTING INDEX +261E;A # So WHITE RIGHT POINTING INDEX +261F..263F;N # So [33] WHITE DOWN POINTING INDEX..MERCURY +2640;A # So FEMALE SIGN +2641;N # So EARTH +2642;A # So MALE SIGN +2643..2647;N # So [5] JUPITER..PLUTO +2648..2653;W # So [12] ARIES..PISCES +2654..265F;N # So [12] WHITE CHESS KING..BLACK CHESS PAWN +2660..2661;A # So [2] BLACK SPADE SUIT..WHITE HEART SUIT +2662;N # So WHITE DIAMOND SUIT +2663..2665;A # So [3] BLACK CLUB SUIT..BLACK HEART SUIT +2666;N # So BLACK DIAMOND SUIT +2667..266A;A # So [4] WHITE CLUB SUIT..EIGHTH NOTE +266B;N # So BEAMED EIGHTH NOTES +266C..266D;A # So [2] BEAMED SIXTEENTH NOTES..MUSIC FLAT SIGN +266E;N # So MUSIC NATURAL SIGN +266F;A # Sm MUSIC SHARP SIGN +2670..267E;N # So [15] WEST SYRIAC CROSS..PERMANENT PAPER SIGN +267F;W # So WHEELCHAIR SYMBOL +2680..2692;N # So [19] DIE FACE-1..HAMMER AND PICK +2693;W # So ANCHOR +2694..269D;N # So [10] CROSSED SWORDS..OUTLINED WHITE STAR +269E..269F;A # So [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0;N # So WARNING SIGN +26A1;W # So HIGH VOLTAGE SIGN +26A2..26A9;N # So [8] DOUBLED FEMALE SIGN..HORIZONTAL MALE WITH STROKE SIGN +26AA..26AB;W # So [2] MEDIUM WHITE CIRCLE..MEDIUM BLACK CIRCLE +26AC..26BC;N # So [17] MEDIUM SMALL WHITE CIRCLE..SESQUIQUADRATE +26BD..26BE;W # So [2] SOCCER BALL..BASEBALL +26BF;A # So SQUARED KEY +26C0..26C3;N # So [4] WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26C5;W # So [2] SNOWMAN WITHOUT SNOW..SUN BEHIND CLOUD +26C6..26CD;A # So [8] RAIN..DISABLED CAR +26CE;W # So OPHIUCHUS +26CF..26D3;A # So [5] PICK..CHAINS +26D4;W # So NO ENTRY +26D5..26E1;A # So [13] ALTERNATE ONE-WAY LEFT WAY TRAFFIC..RESTRICTED LEFT ENTRY-2 +26E2;N # So ASTRONOMICAL SYMBOL FOR URANUS +26E3;A # So HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7;N # So [4] PENTAGRAM..INVERTED PENTAGRAM +26E8..26E9;A # So [2] BLACK CROSS ON SHIELD..SHINTO SHRINE +26EA;W # So CHURCH +26EB..26F1;A # So [7] CASTLE..UMBRELLA ON GROUND +26F2..26F3;W # So [2] FOUNTAIN..FLAG IN HOLE +26F4;A # So FERRY +26F5;W # So SAILBOAT +26F6..26F9;A # So [4] SQUARE FOUR CORNERS..PERSON WITH BALL +26FA;W # So TENT +26FB..26FC;A # So [2] JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL +26FD;W # So FUEL PUMP +26FE..26FF;A # So [2] CUP ON BLACK SQUARE..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700..2704;N # So [5] BLACK SAFETY SCISSORS..WHITE SCISSORS +2705;W # So WHITE HEAVY CHECK MARK +2706..2709;N # So [4] TELEPHONE LOCATION SIGN..ENVELOPE +270A..270B;W # So [2] RAISED FIST..RAISED HAND +270C..2727;N # So [28] VICTORY HAND..WHITE FOUR POINTED STAR +2728;W # So SPARKLES +2729..273C;N # So [20] STRESS OUTLINED WHITE STAR..OPEN CENTRE TEARDROP-SPOKED ASTERISK +273D;A # So HEAVY TEARDROP-SPOKED ASTERISK +273E..274B;N # So [14] SIX PETALLED BLACK AND WHITE FLORETTE..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK +274C;W # So CROSS MARK +274D;N # So SHADOWED WHITE CIRCLE +274E;W # So NEGATIVE SQUARED CROSS MARK +274F..2752;N # So [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE +2753..2755;W # So [3] BLACK QUESTION MARK ORNAMENT..WHITE EXCLAMATION MARK ORNAMENT +2756;N # So BLACK DIAMOND MINUS WHITE X +2757;W # So HEAVY EXCLAMATION MARK SYMBOL +2758..2767;N # So [16] LIGHT VERTICAL BAR..ROTATED FLORAL HEART BULLET +2768;N # Ps MEDIUM LEFT PARENTHESIS ORNAMENT +2769;N # Pe MEDIUM RIGHT PARENTHESIS ORNAMENT +276A;N # Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT +276B;N # Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT +276C;N # Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT +276D;N # Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT +276E;N # Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT +276F;N # Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT +2770;N # Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT +2771;N # Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT +2772;N # Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT +2773;N # Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT +2774;N # Ps MEDIUM LEFT CURLY BRACKET ORNAMENT +2775;N # Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT +2776..277F;A # No [10] DINGBAT NEGATIVE CIRCLED DIGIT ONE..DINGBAT NEGATIVE CIRCLED NUMBER TEN +2780..2793;N # No [20] DINGBAT CIRCLED SANS-SERIF DIGIT ONE..DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN +2794;N # So HEAVY WIDE-HEADED RIGHTWARDS ARROW +2795..2797;W # So [3] HEAVY PLUS SIGN..HEAVY DIVISION SIGN +2798..27AF;N # So [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW +27B0;W # So CURLY LOOP +27B1..27BE;N # So [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW +27BF;W # So DOUBLE CURLY LOOP +27C0..27C4;N # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5;N # Ps LEFT S-SHAPED BAG DELIMITER +27C6;N # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5;N # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6;Na # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7;Na # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8;Na # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9;Na # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA;Na # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB;Na # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC;Na # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED;Na # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE;N # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF;N # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF;N # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2800..28FF;N # So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +2900..297F;N # Sm [128] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..DOWN FISH TAIL +2980..2982;N # Sm [3] TRIPLE VERTICAL BAR DELIMITER..Z NOTATION TYPE COLON +2983;N # Ps LEFT WHITE CURLY BRACKET +2984;N # Pe RIGHT WHITE CURLY BRACKET +2985;Na # Ps LEFT WHITE PARENTHESIS +2986;Na # Pe RIGHT WHITE PARENTHESIS +2987;N # Ps Z NOTATION LEFT IMAGE BRACKET +2988;N # Pe Z NOTATION RIGHT IMAGE BRACKET +2989;N # Ps Z NOTATION LEFT BINDING BRACKET +298A;N # Pe Z NOTATION RIGHT BINDING BRACKET +298B;N # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C;N # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D;N # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E;N # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F;N # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990;N # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991;N # Ps LEFT ANGLE BRACKET WITH DOT +2992;N # Pe RIGHT ANGLE BRACKET WITH DOT +2993;N # Ps LEFT ARC LESS-THAN BRACKET +2994;N # Pe RIGHT ARC GREATER-THAN BRACKET +2995;N # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996;N # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997;N # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998;N # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7;N # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8;N # Ps LEFT WIGGLY FENCE +29D9;N # Pe RIGHT WIGGLY FENCE +29DA;N # Ps LEFT DOUBLE WIGGLY FENCE +29DB;N # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB;N # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC;N # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD;N # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..29FF;N # Sm [2] TINY..MINY +2A00..2AFF;N # Sm [256] N-ARY CIRCLED DOT OPERATOR..N-ARY WHITE VERTICAL BAR +2B00..2B1A;N # So [27] NORTH EAST WHITE ARROW..DOTTED SQUARE +2B1B..2B1C;W # So [2] BLACK LARGE SQUARE..WHITE LARGE SQUARE +2B1D..2B2F;N # So [19] BLACK VERY SMALL SQUARE..WHITE VERTICAL ELLIPSE +2B30..2B44;N # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B45..2B46;N # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW +2B47..2B4C;N # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B4F;N # So [3] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..SHORT BACKSLANTED SOUTH ARROW +2B50;W # So WHITE MEDIUM STAR +2B51..2B54;N # So [4] BLACK SMALL STAR..WHITE RIGHT-POINTING PENTAGON +2B55;W # So HEAVY LARGE CIRCLE +2B56..2B59;A # So [4] HEAVY OVAL WITH OVAL INSIDE..HEAVY CIRCLED SALTIRE +2B5A..2B73;N # So [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B76..2B95;N # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW +2B97..2BFF;N # So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL +2C00..2C5F;N # L& [96] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC SMALL LETTER CAUDATE CHRIVI +2C60..2C7B;N # L& [28] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN LETTER SMALL CAPITAL TURNED E +2C7C..2C7D;N # Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +2C7E..2C7F;N # Lu [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80..2CE4;N # L& [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI +2CE5..2CEA;N # So [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA +2CEB..2CEE;N # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2CEF..2CF1;N # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2CF2..2CF3;N # L& [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI +2CF9..2CFC;N # Po [4] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN VERSE DIVIDER +2CFD;N # No COPTIC FRACTION ONE HALF +2CFE..2CFF;N # Po [2] COPTIC FULL STOP..COPTIC MORPHOLOGICAL DIVIDER +2D00..2D25;N # Ll [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D27;N # Ll GEORGIAN SMALL LETTER YN +2D2D;N # Ll GEORGIAN SMALL LETTER AEN +2D30..2D67;N # Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO +2D6F;N # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D70;N # Po TIFINAGH SEPARATOR MARK +2D7F;N # Mn TIFINAGH CONSONANT JOINER +2D80..2D96;N # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6;N # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE;N # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6;N # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE;N # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6;N # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE;N # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6;N # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE;N # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2DE0..2DFF;N # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +2E00..2E01;N # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER +2E02;N # Pi LEFT SUBSTITUTION BRACKET +2E03;N # Pf RIGHT SUBSTITUTION BRACKET +2E04;N # Pi LEFT DOTTED SUBSTITUTION BRACKET +2E05;N # Pf RIGHT DOTTED SUBSTITUTION BRACKET +2E06..2E08;N # Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER +2E09;N # Pi LEFT TRANSPOSITION BRACKET +2E0A;N # Pf RIGHT TRANSPOSITION BRACKET +2E0B;N # Po RAISED SQUARE +2E0C;N # Pi LEFT RAISED OMISSION BRACKET +2E0D;N # Pf RIGHT RAISED OMISSION BRACKET +2E0E..2E16;N # Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E17;N # Pd DOUBLE OBLIQUE HYPHEN +2E18..2E19;N # Po [2] INVERTED INTERROBANG..PALM BRANCH +2E1A;N # Pd HYPHEN WITH DIAERESIS +2E1B;N # Po TILDE WITH RING ABOVE +2E1C;N # Pi LEFT LOW PARAPHRASE BRACKET +2E1D;N # Pf RIGHT LOW PARAPHRASE BRACKET +2E1E..2E1F;N # Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW +2E20;N # Pi LEFT VERTICAL BAR WITH QUILL +2E21;N # Pf RIGHT VERTICAL BAR WITH QUILL +2E22;N # Ps TOP LEFT HALF BRACKET +2E23;N # Pe TOP RIGHT HALF BRACKET +2E24;N # Ps BOTTOM LEFT HALF BRACKET +2E25;N # Pe BOTTOM RIGHT HALF BRACKET +2E26;N # Ps LEFT SIDEWAYS U BRACKET +2E27;N # Pe RIGHT SIDEWAYS U BRACKET +2E28;N # Ps LEFT DOUBLE PARENTHESIS +2E29;N # Pe RIGHT DOUBLE PARENTHESIS +2E2A..2E2E;N # Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK +2E2F;N # Lm VERTICAL TILDE +2E30..2E39;N # Po [10] RING POINT..TOP HALF SECTION SIGN +2E3A..2E3B;N # Pd [2] TWO-EM DASH..THREE-EM DASH +2E3C..2E3F;N # Po [4] STENOGRAPHIC FULL STOP..CAPITULUM +2E40;N # Pd DOUBLE HYPHEN +2E41;N # Po REVERSED COMMA +2E42;N # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E4F;N # Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER +2E50..2E51;N # So [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR +2E52..2E54;N # Po [3] TIRONIAN SIGN CAPITAL ET..MEDIEVAL QUESTION MARK +2E55;N # Ps LEFT SQUARE BRACKET WITH STROKE +2E56;N # Pe RIGHT SQUARE BRACKET WITH STROKE +2E57;N # Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE +2E58;N # Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE +2E59;N # Ps TOP HALF LEFT PARENTHESIS +2E5A;N # Pe TOP HALF RIGHT PARENTHESIS +2E5B;N # Ps BOTTOM HALF LEFT PARENTHESIS +2E5C;N # Pe BOTTOM HALF RIGHT PARENTHESIS +2E5D;N # Pd OBLIQUE HYPHEN +2E80..2E99;W # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2EF3;W # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5;W # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE +2FF0..2FFB;W # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +3000;F # Zs IDEOGRAPHIC SPACE +3001..3003;W # Po [3] IDEOGRAPHIC COMMA..DITTO MARK +3004;W # So JAPANESE INDUSTRIAL STANDARD SYMBOL +3005;W # Lm IDEOGRAPHIC ITERATION MARK +3006;W # Lo IDEOGRAPHIC CLOSING MARK +3007;W # Nl IDEOGRAPHIC NUMBER ZERO +3008;W # Ps LEFT ANGLE BRACKET +3009;W # Pe RIGHT ANGLE BRACKET +300A;W # Ps LEFT DOUBLE ANGLE BRACKET +300B;W # Pe RIGHT DOUBLE ANGLE BRACKET +300C;W # Ps LEFT CORNER BRACKET +300D;W # Pe RIGHT CORNER BRACKET +300E;W # Ps LEFT WHITE CORNER BRACKET +300F;W # Pe RIGHT WHITE CORNER BRACKET +3010;W # Ps LEFT BLACK LENTICULAR BRACKET +3011;W # Pe RIGHT BLACK LENTICULAR BRACKET +3012..3013;W # So [2] POSTAL MARK..GETA MARK +3014;W # Ps LEFT TORTOISE SHELL BRACKET +3015;W # Pe RIGHT TORTOISE SHELL BRACKET +3016;W # Ps LEFT WHITE LENTICULAR BRACKET +3017;W # Pe RIGHT WHITE LENTICULAR BRACKET +3018;W # Ps LEFT WHITE TORTOISE SHELL BRACKET +3019;W # Pe RIGHT WHITE TORTOISE SHELL BRACKET +301A;W # Ps LEFT WHITE SQUARE BRACKET +301B;W # Pe RIGHT WHITE SQUARE BRACKET +301C;W # Pd WAVE DASH +301D;W # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F;W # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +3020;W # So POSTAL MARK FACE +3021..3029;W # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +302A..302D;W # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK +302E..302F;W # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +3030;W # Pd WAVY DASH +3031..3035;W # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3036..3037;W # So [2] CIRCLED POSTAL MARK..IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL +3038..303A;W # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B;W # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C;W # Lo MASU MARK +303D;W # Po PART ALTERNATION MARK +303E;W # So IDEOGRAPHIC VARIATION INDICATOR +303F;N # So IDEOGRAPHIC HALF FILL SPACE +3041..3096;W # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +3099..309A;W # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C;W # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E;W # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F;W # Lo HIRAGANA DIGRAPH YORI +30A0;W # Pd KATAKANA-HIRAGANA DOUBLE HYPHEN +30A1..30FA;W # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FB;W # Po KATAKANA MIDDLE DOT +30FC..30FE;W # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF;W # Lo KATAKANA DIGRAPH KOTO +3105..312F;W # Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN +3131..318E;W # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +3190..3191;W # So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK +3192..3195;W # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK +3196..319F;W # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK +31A0..31BF;W # Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH +31C0..31E3;W # So [36] CJK STROKE T..CJK STROKE Q +31F0..31FF;W # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3200..321E;W # So [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU +3220..3229;W # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN +322A..3247;W # So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO +3248..324F;A # No [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE +3250;W # So PARTNERSHIP SIGN +3251..325F;W # No [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE +3260..327F;W # So [32] CIRCLED HANGUL KIYEOK..KOREAN STANDARD SYMBOL +3280..3289;W # No [10] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN +328A..32B0;W # So [39] CIRCLED IDEOGRAPH MOON..CIRCLED IDEOGRAPH NIGHT +32B1..32BF;W # No [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY +32C0..32FF;W # So [64] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..SQUARE ERA NAME REIWA +3300..33FF;W # So [256] SQUARE APAATO..SQUARE GAL +3400..4DBF;W # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4DC0..4DFF;N # So [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION +4E00..9FFF;W # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +A000..A014;W # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015;W # Lm YI SYLLABLE WU +A016..A48C;W # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A490..A4C6;W # So [55] YI RADICAL QOT..YI RADICAL KE +A4D0..A4F7;N # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD;N # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A4FE..A4FF;N # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A500..A60B;N # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C;N # Lm VAI SYLLABLE LENGTHENER +A60D..A60F;N # Po [3] VAI COMMA..VAI QUESTION MARK +A610..A61F;N # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A620..A629;N # Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE +A62A..A62B;N # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A66D;N # L& [46] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E;N # Lo CYRILLIC LETTER MULTIOCULAR O +A66F;N # Mn COMBINING CYRILLIC VZMET +A670..A672;N # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN +A673;N # Po SLAVONIC ASTERISK +A674..A67D;N # Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK +A67E;N # Po CYRILLIC KAVYKA +A67F;N # Lm CYRILLIC PAYEROK +A680..A69B;N # L& [28] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER CROSSED O +A69C..A69D;N # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A69E..A69F;N # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E +A6A0..A6E5;N # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF;N # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A6F0..A6F1;N # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A6F2..A6F7;N # Po [6] BAMUM NJAEMLI..BAMUM QUESTION MARK +A700..A716;N # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F;N # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721;N # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A722..A76F;N # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770;N # Lm MODIFIER LETTER US +A771..A787;N # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788;N # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A;N # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A78B..A78E;N # L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT +A78F;N # Lo LATIN LETTER SINOLOGICAL DOT +A790..A7CA;N # L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY +A7D0..A7D1;N # L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G +A7D3;N # Ll LATIN SMALL LETTER DOUBLE THORN +A7D5..A7D9;N # L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S +A7F2..A7F4;N # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A7F5..A7F6;N # L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H +A7F7;N # Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I +A7F8..A7F9;N # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +A7FA;N # Ll LATIN LETTER SMALL CAPITAL TURNED M +A7FB..A7FF;N # Lo [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M +A800..A801;N # Lo [2] SYLOTI NAGRI LETTER A..SYLOTI NAGRI LETTER I +A802;N # Mn SYLOTI NAGRI SIGN DVISVARA +A803..A805;N # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A806;N # Mn SYLOTI NAGRI SIGN HASANTA +A807..A80A;N # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80B;N # Mn SYLOTI NAGRI SIGN ANUSVARA +A80C..A822;N # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824;N # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826;N # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827;N # Mc SYLOTI NAGRI VOWEL SIGN OO +A828..A82B;N # So [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 +A82C;N # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA +A830..A835;N # No [6] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS +A836..A837;N # So [2] NORTH INDIC QUARTER MARK..NORTH INDIC PLACEHOLDER MARK +A838;N # Sc NORTH INDIC RUPEE MARK +A839;N # So NORTH INDIC QUANTITY MARK +A840..A873;N # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A874..A877;N # Po [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD +A880..A881;N # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3;N # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3;N # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C4..A8C5;N # Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU +A8CE..A8CF;N # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A8D0..A8D9;N # Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A8E0..A8F1;N # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A8F2..A8F7;N # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8F8..A8FA;N # Po [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET +A8FB;N # Lo DEVANAGARI HEADSTROKE +A8FC;N # Po DEVANAGARI SIGN SIDDHAM +A8FD..A8FE;N # Lo [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY +A8FF;N # Mn DEVANAGARI VOWEL SIGN AY +A900..A909;N # Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE +A90A..A925;N # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A926..A92D;N # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A92E..A92F;N # Po [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA +A930..A946;N # Lo [23] REJANG LETTER KA..REJANG LETTER A +A947..A951;N # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952..A953;N # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA +A95F;N # Po REJANG SECTION MARK +A960..A97C;W # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A980..A982;N # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983;N # Mc JAVANESE SIGN WIGNYAN +A984..A9B2;N # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B3;N # Mn JAVANESE SIGN CECAK TELU +A9B4..A9B5;N # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9;N # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB;N # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC..A9BD;N # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET +A9BE..A9C0;N # Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON +A9C1..A9CD;N # Po [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH +A9CF;N # Lm JAVANESE PANGRANGKEP +A9D0..A9D9;N # Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +A9DE..A9DF;N # Po [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN +A9E0..A9E4;N # Lo [5] MYANMAR LETTER SHAN GHA..MYANMAR LETTER SHAN BHA +A9E5;N # Mn MYANMAR SIGN SHAN SAW +A9E6;N # Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION +A9E7..A9EF;N # Lo [9] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING NNA +A9F0..A9F9;N # Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE +A9FA..A9FE;N # Lo [5] MYANMAR LETTER TAI LAING LLA..MYANMAR LETTER TAI LAING BHA +AA00..AA28;N # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA29..AA2E;N # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30;N # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32;N # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34;N # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36;N # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA40..AA42;N # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA43;N # Mn CHAM CONSONANT SIGN FINAL NG +AA44..AA4B;N # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4C;N # Mn CHAM CONSONANT SIGN FINAL M +AA4D;N # Mc CHAM CONSONANT SIGN FINAL H +AA50..AA59;N # Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA5C..AA5F;N # Po [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA +AA60..AA6F;N # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70;N # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76;N # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA77..AA79;N # So [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO +AA7A;N # Lo MYANMAR LETTER AITON RA +AA7B;N # Mc MYANMAR SIGN PAO KAREN TONE +AA7C;N # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D;N # Mc MYANMAR SIGN TAI LAING TONE-5 +AA7E..AA7F;N # Lo [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA +AA80..AAAF;N # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB0;N # Mn TAI VIET MAI KANG +AAB1;N # Lo TAI VIET VOWEL AA +AAB2..AAB4;N # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB5..AAB6;N # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB7..AAB8;N # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AAB9..AABD;N # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AABE..AABF;N # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC0;N # Lo TAI VIET TONE MAI NUENG +AAC1;N # Mn TAI VIET TONE MAI THO +AAC2;N # Lo TAI VIET TONE MAI SONG +AADB..AADC;N # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD;N # Lm TAI VIET SYMBOL SAM +AADE..AADF;N # Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI +AAE0..AAEA;N # Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA +AAEB;N # Mc MEETEI MAYEK VOWEL SIGN II +AAEC..AAED;N # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI +AAEE..AAEF;N # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU +AAF0..AAF1;N # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +AAF2;N # Lo MEETEI MAYEK ANJI +AAF3..AAF4;N # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK +AAF5;N # Mc MEETEI MAYEK VOWEL SIGN VISARGA +AAF6;N # Mn MEETEI MAYEK VIRAMA +AB01..AB06;N # Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO +AB09..AB0E;N # Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO +AB11..AB16;N # Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO +AB20..AB26;N # Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO +AB28..AB2E;N # Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO +AB30..AB5A;N # Ll [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG +AB5B;N # Sk MODIFIER BREVE WITH INVERTED BREVE +AB5C..AB5F;N # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB60..AB68;N # Ll [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE +AB69;N # Lm MODIFIER LETTER SMALL TURNED W +AB6A..AB6B;N # Sk [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +AB70..ABBF;N # Ll [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA +ABC0..ABE2;N # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4;N # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5;N # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7;N # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8;N # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA;N # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +ABEB;N # Po MEETEI MAYEK CHEIKHEI +ABEC;N # Mc MEETEI MAYEK LUM IYEK +ABED;N # Mn MEETEI MAYEK APUN IYEK +ABF0..ABF9;N # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +AC00..D7A3;W # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6;N # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB;N # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +D800..DB7F;N # Cs [896] <surrogate-D800>..<surrogate-DB7F> +DB80..DBFF;N # Cs [128] <surrogate-DB80>..<surrogate-DBFF> +DC00..DFFF;N # Cs [1024] <surrogate-DC00>..<surrogate-DFFF> +E000..F8FF;A # Co [6400] <private-use-E000>..<private-use-F8FF> +F900..FA6D;W # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA6E..FA6F;W # Cn [2] <reserved-FA6E>..<reserved-FA6F> +FA70..FAD9;W # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FADA..FAFF;W # Cn [38] <reserved-FADA>..<reserved-FAFF> +FB00..FB06;N # Ll [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17;N # Ll [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D;N # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E;N # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28;N # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB29;N # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FB2A..FB36;N # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C;N # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E;N # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41;N # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44;N # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FB4F;N # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATURE ALEF LAMED +FB50..FBB1;N # Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBB2..FBC2;N # Sk [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE +FBD3..FD3D;N # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD3E;N # Pe ORNATE LEFT PARENTHESIS +FD3F;N # Ps ORNATE RIGHT PARENTHESIS +FD40..FD4F;N # So [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH +FD50..FD8F;N # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7;N # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDCF;N # So ARABIC LIGATURE SALAAMUHU ALAYNAA +FDF0..FDFB;N # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FDFC;N # Sc RIAL SIGN +FDFD..FDFF;N # So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL +FE00..FE0F;A # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE10..FE16;W # Po [7] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL QUESTION MARK +FE17;W # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET +FE18;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET +FE19;W # Po PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS +FE20..FE2F;N # Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF +FE30;W # Po PRESENTATION FORM FOR VERTICAL TWO DOT LEADER +FE31..FE32;W # Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH +FE33..FE34;W # Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE35;W # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37;W # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE39;W # Ps PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET +FE3A;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET +FE3B;W # Ps PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET +FE3C;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET +FE3D;W # Ps PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET +FE3E;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET +FE3F;W # Ps PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET +FE40;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET +FE41;W # Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43;W # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE45..FE46;W # Po [2] SESAME DOT..WHITE SESAME DOT +FE47;W # Ps PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET +FE48;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET +FE49..FE4C;W # Po [4] DASHED OVERLINE..DOUBLE WAVY OVERLINE +FE4D..FE4F;W # Pc [3] DASHED LOW LINE..WAVY LOW LINE +FE50..FE52;W # Po [3] SMALL COMMA..SMALL FULL STOP +FE54..FE57;W # Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK +FE58;W # Pd SMALL EM DASH +FE59;W # Ps SMALL LEFT PARENTHESIS +FE5A;W # Pe SMALL RIGHT PARENTHESIS +FE5B;W # Ps SMALL LEFT CURLY BRACKET +FE5C;W # Pe SMALL RIGHT CURLY BRACKET +FE5D;W # Ps SMALL LEFT TORTOISE SHELL BRACKET +FE5E;W # Pe SMALL RIGHT TORTOISE SHELL BRACKET +FE5F..FE61;W # Po [3] SMALL NUMBER SIGN..SMALL ASTERISK +FE62;W # Sm SMALL PLUS SIGN +FE63;W # Pd SMALL HYPHEN-MINUS +FE64..FE66;W # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68;W # Po SMALL REVERSE SOLIDUS +FE69;W # Sc SMALL DOLLAR SIGN +FE6A..FE6B;W # Po [2] SMALL PERCENT SIGN..SMALL COMMERCIAL AT +FE70..FE74;N # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC;N # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FEFF;N # Cf ZERO WIDTH NO-BREAK SPACE +FF01..FF03;F # Po [3] FULLWIDTH EXCLAMATION MARK..FULLWIDTH NUMBER SIGN +FF04;F # Sc FULLWIDTH DOLLAR SIGN +FF05..FF07;F # Po [3] FULLWIDTH PERCENT SIGN..FULLWIDTH APOSTROPHE +FF08;F # Ps FULLWIDTH LEFT PARENTHESIS +FF09;F # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A;F # Po FULLWIDTH ASTERISK +FF0B;F # Sm FULLWIDTH PLUS SIGN +FF0C;F # Po FULLWIDTH COMMA +FF0D;F # Pd FULLWIDTH HYPHEN-MINUS +FF0E..FF0F;F # Po [2] FULLWIDTH FULL STOP..FULLWIDTH SOLIDUS +FF10..FF19;F # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF1A..FF1B;F # Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON +FF1C..FF1E;F # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF1F..FF20;F # Po [2] FULLWIDTH QUESTION MARK..FULLWIDTH COMMERCIAL AT +FF21..FF3A;F # Lu [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF3B;F # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C;F # Po FULLWIDTH REVERSE SOLIDUS +FF3D;F # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E;F # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF3F;F # Pc FULLWIDTH LOW LINE +FF40;F # Sk FULLWIDTH GRAVE ACCENT +FF41..FF5A;F # Ll [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF5B;F # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C;F # Sm FULLWIDTH VERTICAL LINE +FF5D;F # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E;F # Sm FULLWIDTH TILDE +FF5F;F # Ps FULLWIDTH LEFT WHITE PARENTHESIS +FF60;F # Pe FULLWIDTH RIGHT WHITE PARENTHESIS +FF61;H # Po HALFWIDTH IDEOGRAPHIC FULL STOP +FF62;H # Ps HALFWIDTH LEFT CORNER BRACKET +FF63;H # Pe HALFWIDTH RIGHT CORNER BRACKET +FF64..FF65;H # Po [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDLE DOT +FF66..FF6F;H # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70;H # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D;H # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F;H # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE;H # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7;H # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF;H # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7;H # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC;H # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +FFE0..FFE1;F # Sc [2] FULLWIDTH CENT SIGN..FULLWIDTH POUND SIGN +FFE2;F # Sm FULLWIDTH NOT SIGN +FFE3;F # Sk FULLWIDTH MACRON +FFE4;F # So FULLWIDTH BROKEN BAR +FFE5..FFE6;F # Sc [2] FULLWIDTH YEN SIGN..FULLWIDTH WON SIGN +FFE8;H # So HALFWIDTH FORMS LIGHT VERTICAL +FFE9..FFEC;H # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +FFED..FFEE;H # So [2] HALFWIDTH BLACK SQUARE..HALFWIDTH WHITE CIRCLE +FFF9..FFFB;N # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +FFFC;N # So OBJECT REPLACEMENT CHARACTER +FFFD;A # So REPLACEMENT CHARACTER +10000..1000B;N # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026;N # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A;N # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D;N # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D;N # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D;N # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA;N # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10100..10102;N # Po [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK +10107..10133;N # No [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND +10137..1013F;N # So [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT +10140..10174;N # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10175..10178;N # No [4] GREEK ONE HALF SIGN..GREEK THREE QUARTERS SIGN +10179..10189;N # So [17] GREEK YEAR SIGN..GREEK TRYBLION BASE SIGN +1018A..1018B;N # No [2] GREEK ZERO SIGN..GREEK ONE QUARTER SIGN +1018C..1018E;N # So [3] GREEK SINUSOID SIGN..NOMISMA SIGN +10190..1019C;N # So [13] ROMAN SEXTANS SIGN..ASCIA SYMBOL +101A0;N # So GREEK SYMBOL TAU RHO +101D0..101FC;N # So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND +101FD;N # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10280..1029C;N # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0;N # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +102E0;N # Mn COPTIC EPACT THOUSANDS MARK +102E1..102FB;N # No [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED +10300..1031F;N # Lo [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS +10320..10323;N # No [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY +1032D..1032F;N # Lo [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE +10330..10340;N # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341;N # Nl GOTHIC LETTER NINETY +10342..10349;N # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A;N # Nl GOTHIC LETTER NINE HUNDRED +10350..10375;N # Lo [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA +10376..1037A;N # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10380..1039D;N # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +1039F;N # Po UGARITIC WORD DIVIDER +103A0..103C3;N # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF;N # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D0;N # Po OLD PERSIAN WORD DIVIDER +103D1..103D5;N # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F;N # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1047F;N # Lo [48] SHAVIAN LETTER PEEP..SHAVIAN LETTER YEW +10480..1049D;N # Lo [30] OSMANYA LETTER ALEF..OSMANYA LETTER OO +104A0..104A9;N # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +104B0..104D3;N # Lu [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA +104D8..104FB;N # Ll [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA +10500..10527;N # Lo [40] ELBASAN LETTER A..ELBASAN LETTER KHE +10530..10563;N # Lo [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW +1056F;N # Po CAUCASIAN ALBANIAN CITATION MARK +10570..1057A;N # Lu [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA +1057C..1058A;N # Lu [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE +1058C..10592;N # Lu [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE +10594..10595;N # Lu [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE +10597..105A1;N # Ll [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA +105A3..105B1;N # Ll [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE +105B3..105B9;N # Ll [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE +105BB..105BC;N # Ll [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE +10600..10736;N # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 +10740..10755;N # Lo [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE +10760..10767;N # Lo [8] LINEAR A SIGN A800..LINEAR A SIGN A807 +10780..10785;N # Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0;N # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA;N # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +10800..10805;N # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808;N # Lo CYPRIOT SYLLABLE JO +1080A..10835;N # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838;N # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C;N # Lo CYPRIOT SYLLABLE ZA +1083F;N # Lo CYPRIOT SYLLABLE ZO +10840..10855;N # Lo [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW +10857;N # Po IMPERIAL ARAMAIC SECTION SIGN +10858..1085F;N # No [8] IMPERIAL ARAMAIC NUMBER ONE..IMPERIAL ARAMAIC NUMBER TEN THOUSAND +10860..10876;N # Lo [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW +10877..10878;N # So [2] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE RIGHT-POINTING FLEURON +10879..1087F;N # No [7] PALMYRENE NUMBER ONE..PALMYRENE NUMBER TWENTY +10880..1089E;N # Lo [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW +108A7..108AF;N # No [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED +108E0..108F2;N # Lo [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH +108F4..108F5;N # Lo [2] HATRAN LETTER SHIN..HATRAN LETTER TAW +108FB..108FF;N # No [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED +10900..10915;N # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10916..1091B;N # No [6] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER THREE +1091F;N # Po PHOENICIAN WORD SEPARATOR +10920..10939;N # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +1093F;N # Po LYDIAN TRIANGULAR MARK +10980..1099F;N # Lo [32] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC HIEROGLYPHIC SYMBOL VIDJ-2 +109A0..109B7;N # Lo [24] MEROITIC CURSIVE LETTER A..MEROITIC CURSIVE LETTER DA +109BC..109BD;N # No [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF +109BE..109BF;N # Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN +109C0..109CF;N # No [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY +109D2..109FF;N # No [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS +10A00;N # Lo KHAROSHTHI LETTER A +10A01..10A03;N # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06;N # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F;N # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A10..10A13;N # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17;N # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A35;N # Lo [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA +10A38..10A3A;N # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F;N # Mn KHAROSHTHI VIRAMA +10A40..10A48;N # No [9] KHAROSHTHI DIGIT ONE..KHAROSHTHI FRACTION ONE HALF +10A50..10A58;N # Po [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES +10A60..10A7C;N # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10A7D..10A7E;N # No [2] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMBER FIFTY +10A7F;N # Po OLD SOUTH ARABIAN NUMERIC INDICATOR +10A80..10A9C;N # Lo [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH +10A9D..10A9F;N # No [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY +10AC0..10AC7;N # Lo [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW +10AC8;N # So MANICHAEAN SIGN UD +10AC9..10AE4;N # Lo [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW +10AE5..10AE6;N # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10AEB..10AEF;N # No [5] MANICHAEAN NUMBER ONE..MANICHAEAN NUMBER ONE HUNDRED +10AF0..10AF6;N # Po [7] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION LINE FILLER +10B00..10B35;N # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B39..10B3F;N # Po [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B40..10B55;N # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B58..10B5F;N # No [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND +10B60..10B72;N # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10B78..10B7F;N # No [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND +10B80..10B91;N # Lo [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW +10B99..10B9C;N # Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10BA9..10BAF;N # No [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED +10C00..10C48;N # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +10C80..10CB2;N # Lu [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US +10CC0..10CF2;N # Ll [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US +10CFA..10CFF;N # No [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND +10D00..10D23;N # Lo [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA +10D24..10D27;N # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10D30..10D39;N # Nd [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE +10E60..10E7E;N # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS +10E80..10EA9;N # Lo [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET +10EAB..10EAC;N # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EAD;N # Pd YEZIDI HYPHENATION MARK +10EB0..10EB1;N # Lo [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE +10F00..10F1C;N # Lo [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL +10F1D..10F26;N # No [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF +10F27;N # Lo OLD SOGDIAN LIGATURE AYIN-DALETH +10F30..10F45;N # Lo [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN +10F46..10F50;N # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW +10F51..10F54;N # No [4] SOGDIAN NUMBER ONE..SOGDIAN NUMBER ONE HUNDRED +10F55..10F59;N # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F70..10F81;N # Lo [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH +10F82..10F85;N # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW +10F86..10F89;N # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +10FB0..10FC4;N # Lo [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW +10FC5..10FCB;N # No [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED +10FE0..10FF6;N # Lo [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH +11000;N # Mc BRAHMI SIGN CANDRABINDU +11001;N # Mn BRAHMI SIGN ANUSVARA +11002;N # Mc BRAHMI SIGN VISARGA +11003..11037;N # Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA +11038..11046;N # Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA +11047..1104D;N # Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +11052..11065;N # No [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND +11066..1106F;N # Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE +11070;N # Mn BRAHMI SIGN OLD TAMIL VIRAMA +11071..11072;N # Lo [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O +11073..11074;N # Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O +11075;N # Lo BRAHMI LETTER OLD TAMIL LLA +1107F;N # Mn BRAHMI NUMBER JOINER +11080..11081;N # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +11082;N # Mc KAITHI SIGN VISARGA +11083..110AF;N # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2;N # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6;N # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8;N # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110B9..110BA;N # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +110BB..110BC;N # Po [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN +110BD;N # Cf KAITHI NUMBER SIGN +110BE..110C1;N # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +110C2;N # Mn KAITHI VOWEL SIGN VOCALIC R +110CD;N # Cf KAITHI NUMBER SIGN ABOVE +110D0..110E8;N # Lo [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE +110F0..110F9;N # Nd [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE +11100..11102;N # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA +11103..11126;N # Lo [36] CHAKMA LETTER AA..CHAKMA LETTER HAA +11127..1112B;N # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU +1112C;N # Mc CHAKMA VOWEL SIGN E +1112D..11134;N # Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA +11136..1113F;N # Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE +11140..11143;N # Po [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK +11144;N # Lo CHAKMA LETTER LHAA +11145..11146;N # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI +11147;N # Lo CHAKMA LETTER VAA +11150..11172;N # Lo [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA +11173;N # Mn MAHAJANI SIGN NUKTA +11174..11175;N # Po [2] MAHAJANI ABBREVIATION SIGN..MAHAJANI SECTION MARK +11176;N # Lo MAHAJANI LIGATURE SHRI +11180..11181;N # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA +11182;N # Mc SHARADA SIGN VISARGA +11183..111B2;N # Lo [48] SHARADA LETTER A..SHARADA LETTER HA +111B3..111B5;N # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II +111B6..111BE;N # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O +111BF..111C0;N # Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA +111C1..111C4;N # Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM +111C5..111C8;N # Po [4] SHARADA DANDA..SHARADA SEPARATOR +111C9..111CC;N # Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK +111CD;N # Po SHARADA SUTRA MARK +111CE;N # Mc SHARADA VOWEL SIGN PRISHTHAMATRA E +111CF;N # Mn SHARADA SIGN INVERTED CANDRABINDU +111D0..111D9;N # Nd [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE +111DA;N # Lo SHARADA EKAM +111DB;N # Po SHARADA SIGN SIDDHAM +111DC;N # Lo SHARADA HEADSTROKE +111DD..111DF;N # Po [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +111E1..111F4;N # No [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND +11200..11211;N # Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA +11213..1122B;N # Lo [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA +1122C..1122E;N # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II +1122F..11231;N # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI +11232..11233;N # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU +11234;N # Mn KHOJKI SIGN ANUSVARA +11235;N # Mc KHOJKI SIGN VIRAMA +11236..11237;N # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA +11238..1123D;N # Po [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN +1123E;N # Mn KHOJKI SIGN SUKUN +11280..11286;N # Lo [7] MULTANI LETTER A..MULTANI LETTER GA +11288;N # Lo MULTANI LETTER GHA +1128A..1128D;N # Lo [4] MULTANI LETTER CA..MULTANI LETTER JJA +1128F..1129D;N # Lo [15] MULTANI LETTER NYA..MULTANI LETTER BA +1129F..112A8;N # Lo [10] MULTANI LETTER BHA..MULTANI LETTER RHA +112A9;N # Po MULTANI SECTION MARK +112B0..112DE;N # Lo [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA +112DF;N # Mn KHUDAWADI SIGN ANUSVARA +112E0..112E2;N # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II +112E3..112EA;N # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA +112F0..112F9;N # Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE +11300..11301;N # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU +11302..11303;N # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA +11305..1130C;N # Lo [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L +1130F..11310;N # Lo [2] GRANTHA LETTER EE..GRANTHA LETTER AI +11313..11328;N # Lo [22] GRANTHA LETTER OO..GRANTHA LETTER NA +1132A..11330;N # Lo [7] GRANTHA LETTER PA..GRANTHA LETTER RA +11332..11333;N # Lo [2] GRANTHA LETTER LA..GRANTHA LETTER LLA +11335..11339;N # Lo [5] GRANTHA LETTER VA..GRANTHA LETTER HA +1133B..1133C;N # Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA +1133D;N # Lo GRANTHA SIGN AVAGRAHA +1133E..1133F;N # Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I +11340;N # Mn GRANTHA VOWEL SIGN II +11341..11344;N # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348;N # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134D;N # Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA +11350;N # Lo GRANTHA OM +11357;N # Mc GRANTHA AU LENGTH MARK +1135D..11361;N # Lo [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL +11362..11363;N # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL +11366..1136C;N # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374;N # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +11400..11434;N # Lo [53] NEWA LETTER A..NEWA LETTER HA +11435..11437;N # Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II +11438..1143F;N # Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI +11440..11441;N # Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU +11442..11444;N # Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA +11445;N # Mc NEWA SIGN VISARGA +11446;N # Mn NEWA SIGN NUKTA +11447..1144A;N # Lo [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI +1144B..1144F;N # Po [5] NEWA DANDA..NEWA ABBREVIATION SIGN +11450..11459;N # Nd [10] NEWA DIGIT ZERO..NEWA DIGIT NINE +1145A..1145B;N # Po [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK +1145D;N # Po NEWA INSERTION SIGN +1145E;N # Mn NEWA SANDHI MARK +1145F..11461;N # Lo [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA +11480..114AF;N # Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA +114B0..114B2;N # Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II +114B3..114B8;N # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL +114B9;N # Mc TIRHUTA VOWEL SIGN E +114BA;N # Mn TIRHUTA VOWEL SIGN SHORT E +114BB..114BE;N # Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU +114BF..114C0;N # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA +114C1;N # Mc TIRHUTA SIGN VISARGA +114C2..114C3;N # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA +114C4..114C5;N # Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG +114C6;N # Po TIRHUTA ABBREVIATION SIGN +114C7;N # Lo TIRHUTA OM +114D0..114D9;N # Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE +11580..115AE;N # Lo [47] SIDDHAM LETTER A..SIDDHAM LETTER HA +115AF..115B1;N # Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II +115B2..115B5;N # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115BB;N # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU +115BC..115BD;N # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA +115BE;N # Mc SIDDHAM SIGN VISARGA +115BF..115C0;N # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA +115C1..115D7;N # Po [23] SIDDHAM SIGN SIDDHAM..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +115D8..115DB;N # Lo [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U +115DC..115DD;N # Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU +11600..1162F;N # Lo [48] MODI LETTER A..MODI LETTER LLA +11630..11632;N # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II +11633..1163A;N # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI +1163B..1163C;N # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU +1163D;N # Mn MODI SIGN ANUSVARA +1163E;N # Mc MODI SIGN VISARGA +1163F..11640;N # Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA +11641..11643;N # Po [3] MODI DANDA..MODI ABBREVIATION SIGN +11644;N # Lo MODI SIGN HUVA +11650..11659;N # Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE +11660..1166C;N # Po [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT +11680..116AA;N # Lo [43] TAKRI LETTER A..TAKRI LETTER RRA +116AB;N # Mn TAKRI SIGN ANUSVARA +116AC;N # Mc TAKRI SIGN VISARGA +116AD;N # Mn TAKRI VOWEL SIGN AA +116AE..116AF;N # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II +116B0..116B5;N # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU +116B6;N # Mc TAKRI SIGN VIRAMA +116B7;N # Mn TAKRI SIGN NUKTA +116B8;N # Lo TAKRI LETTER ARCHAIC KHA +116B9;N # Po TAKRI ABBREVIATION SIGN +116C0..116C9;N # Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE +11700..1171A;N # Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA +1171D..1171F;N # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA +11720..11721;N # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA +11722..11725;N # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU +11726;N # Mc AHOM VOWEL SIGN E +11727..1172B;N # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER +11730..11739;N # Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE +1173A..1173B;N # No [2] AHOM NUMBER TEN..AHOM NUMBER TWENTY +1173C..1173E;N # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +1173F;N # So AHOM SYMBOL VI +11740..11746;N # Lo [7] AHOM LETTER CA..AHOM LETTER LLA +11800..1182B;N # Lo [44] DOGRA LETTER A..DOGRA LETTER RRA +1182C..1182E;N # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +1182F..11837;N # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11838;N # Mc DOGRA SIGN VISARGA +11839..1183A;N # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +1183B;N # Po DOGRA ABBREVIATION SIGN +118A0..118DF;N # L& [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO +118E0..118E9;N # Nd [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE +118EA..118F2;N # No [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY +118FF;N # Lo WARANG CITI OM +11900..11906;N # Lo [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E +11909;N # Lo DIVES AKURU LETTER O +1190C..11913;N # Lo [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA +11915..11916;N # Lo [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA +11918..1192F;N # Lo [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA +11930..11935;N # Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E +11937..11938;N # Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..1193C;N # Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU +1193D;N # Mc DIVES AKURU SIGN HALANTA +1193E;N # Mn DIVES AKURU VIRAMA +1193F;N # Lo DIVES AKURU PREFIXED NASAL SIGN +11940;N # Mc DIVES AKURU MEDIAL YA +11941;N # Lo DIVES AKURU INITIAL RA +11942;N # Mc DIVES AKURU MEDIAL RA +11943;N # Mn DIVES AKURU SIGN NUKTA +11944..11946;N # Po [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK +11950..11959;N # Nd [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE +119A0..119A7;N # Lo [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR +119AA..119D0;N # Lo [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA +119D1..119D3;N # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119D4..119D7;N # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB;N # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119DC..119DF;N # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E0;N # Mn NANDINAGARI SIGN VIRAMA +119E1;N # Lo NANDINAGARI SIGN AVAGRAHA +119E2;N # Po NANDINAGARI SIGN SIDDHAM +119E3;N # Lo NANDINAGARI HEADSTROKE +119E4;N # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A00;N # Lo ZANABAZAR SQUARE LETTER A +11A01..11A0A;N # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A0B..11A32;N # Lo [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA +11A33..11A38;N # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA +11A39;N # Mc ZANABAZAR SQUARE SIGN VISARGA +11A3A;N # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA +11A3B..11A3E;N # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A3F..11A46;N # Po [8] ZANABAZAR SQUARE INITIAL HEAD MARK..ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK +11A47;N # Mn ZANABAZAR SQUARE SUBJOINER +11A50;N # Lo SOYOMBO LETTER A +11A51..11A56;N # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A57..11A58;N # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A59..11A5B;N # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A5C..11A89;N # Lo [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA +11A8A..11A96;N # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A97;N # Mc SOYOMBO SIGN VISARGA +11A98..11A99;N # Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER +11A9A..11A9C;N # Po [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD +11A9D;N # Lo SOYOMBO MARK PLUTA +11A9E..11AA2;N # Po [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2 +11AB0..11ABF;N # Lo [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA +11AC0..11AF8;N # Lo [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL +11C00..11C08;N # Lo [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L +11C0A..11C2E;N # Lo [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA +11C2F;N # Mc BHAIKSUKI VOWEL SIGN AA +11C30..11C36;N # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C3D;N # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA +11C3E;N # Mc BHAIKSUKI SIGN VISARGA +11C3F;N # Mn BHAIKSUKI SIGN VIRAMA +11C40;N # Lo BHAIKSUKI SIGN AVAGRAHA +11C41..11C45;N # Po [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 +11C50..11C59;N # Nd [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE +11C5A..11C6C;N # No [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK +11C70..11C71;N # Po [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD +11C72..11C8F;N # Lo [30] MARCHEN LETTER KA..MARCHEN LETTER A +11C92..11CA7;N # Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9;N # Mc MARCHEN SUBJOINED LETTER YA +11CAA..11CB0;N # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA +11CB1;N # Mc MARCHEN VOWEL SIGN I +11CB2..11CB3;N # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E +11CB4;N # Mc MARCHEN VOWEL SIGN O +11CB5..11CB6;N # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D00..11D06;N # Lo [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E +11D08..11D09;N # Lo [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O +11D0B..11D30;N # Lo [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA +11D31..11D36;N # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A;N # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D;N # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D45;N # Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA +11D46;N # Lo MASARAM GONDI REPHA +11D47;N # Mn MASARAM GONDI RA-KARA +11D50..11D59;N # Nd [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE +11D60..11D65;N # Lo [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU +11D67..11D68;N # Lo [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI +11D6A..11D89;N # Lo [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA +11D8A..11D8E;N # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91;N # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D94;N # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D95;N # Mn GUNJALA GONDI SIGN ANUSVARA +11D96;N # Mc GUNJALA GONDI SIGN VISARGA +11D97;N # Mn GUNJALA GONDI VIRAMA +11D98;N # Lo GUNJALA GONDI OM +11DA0..11DA9;N # Nd [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE +11EE0..11EF2;N # Lo [19] MAKASAR LETTER KA..MAKASAR ANGKA +11EF3..11EF4;N # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11EF5..11EF6;N # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +11EF7..11EF8;N # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11FB0;N # Lo LISU LETTER YHA +11FC0..11FD4;N # No [21] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL FRACTION DOWNSCALING FACTOR KIIZH +11FD5..11FDC;N # So [8] TAMIL SIGN NEL..TAMIL SIGN MUKKURUNI +11FDD..11FE0;N # Sc [4] TAMIL SIGN KAACU..TAMIL SIGN VARAAKAN +11FE1..11FF1;N # So [17] TAMIL SIGN PAARAM..TAMIL SIGN VAKAIYARAA +11FFF;N # Po TAMIL PUNCTUATION END OF TEXT +12000..12399;N # Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U +12400..1246E;N # Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM +12470..12474;N # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +12480..12543;N # Lo [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU +12F90..12FF0;N # Lo [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 +12FF1..12FF2;N # Po [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 +13000..1342E;N # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +13430..13438;N # Cf [9] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END SEGMENT +14400..14646;N # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 +16800..16A38;N # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ +16A40..16A5E;N # Lo [31] MRO LETTER TA..MRO LETTER TEK +16A60..16A69;N # Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE +16A6E..16A6F;N # Po [2] MRO DANDA..MRO DOUBLE DANDA +16A70..16ABE;N # Lo [79] TANGSA LETTER OZ..TANGSA LETTER ZA +16AC0..16AC9;N # Nd [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE +16AD0..16AED;N # Lo [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I +16AF0..16AF4;N # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16AF5;N # Po BASSA VAH FULL STOP +16B00..16B2F;N # Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU +16B30..16B36;N # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM +16B37..16B3B;N # Po [5] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS FEEM +16B3C..16B3F;N # So [4] PAHAWH HMONG SIGN XYEEM NTXIV..PAHAWH HMONG SIGN XYEEM FAIB +16B40..16B43;N # Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM +16B44;N # Po PAHAWH HMONG SIGN XAUS +16B45;N # So PAHAWH HMONG SIGN CIM TSOV ROG +16B50..16B59;N # Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE +16B5B..16B61;N # No [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS +16B63..16B77;N # Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS +16B7D..16B8F;N # Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ +16E40..16E7F;N # L& [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y +16E80..16E96;N # No [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM +16E97..16E9A;N # Po [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH +16F00..16F4A;N # Lo [75] MIAO LETTER PA..MIAO LETTER RTE +16F4F;N # Mn MIAO SIGN CONSONANT MODIFIER BAR +16F50;N # Lo MIAO LETTER NASALIZATION +16F51..16F87;N # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI +16F8F..16F92;N # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16F93..16F9F;N # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 +16FE0..16FE1;W # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK +16FE2;W # Po OLD CHINESE HOOK MARK +16FE3;W # Lm OLD CHINESE ITERATION MARK +16FE4;W # Mn KHITAN SMALL SCRIPT FILLER +16FF0..16FF1;W # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +17000..187F7;W # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 +18800..18AFF;W # Lo [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 +18B00..18CD5;W # Lo [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18D00..18D08;W # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +1AFF0..1AFF3;W # Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB;W # Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE;W # Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 +1B000..1B0FF;W # Lo [256] KATAKANA LETTER ARCHAIC E..HENTAIGANA LETTER RE-2 +1B100..1B122;W # Lo [35] HENTAIGANA LETTER RE-3..KATAKANA LETTER ARCHAIC WU +1B150..1B152;W # Lo [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO +1B164..1B167;W # Lo [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N +1B170..1B2FB;W # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +1BC00..1BC6A;N # Lo [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M +1BC70..1BC7C;N # Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK +1BC80..1BC88;N # Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL +1BC90..1BC99;N # Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW +1BC9C;N # So DUPLOYAN SIGN O WITH CROSS +1BC9D..1BC9E;N # Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK +1BC9F;N # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1BCA0..1BCA3;N # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1CF00..1CF2D;N # Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46;N # Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1CF50..1CFC3;N # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK +1D000..1D0F5;N # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO +1D100..1D126;N # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 +1D129..1D164;N # So [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D165..1D166;N # Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D167..1D169;N # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16A..1D16C;N # So [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 +1D16D..1D172;N # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D173..1D17A;N # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +1D17B..1D182;N # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D183..1D184;N # So [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN +1D185..1D18B;N # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D18C..1D1A9;N # So [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH +1D1AA..1D1AD;N # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D1AE..1D1EA;N # So [61] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL KORON +1D200..1D241;N # So [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 +1D242..1D244;N # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +1D245;N # So GREEK MUSICAL LEIMMA +1D2E0..1D2F3;N # No [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN +1D300..1D356;N # So [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING +1D360..1D378;N # No [25] COUNTING ROD UNIT DIGIT ONE..TALLY MARK FIVE +1D400..1D454;N # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C;N # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F;N # Lu [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2;N # Lu MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6;N # Lu [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC;N # Lu [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9;N # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB;N # Ll MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3;N # Ll [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505;N # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A;N # Lu [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514;N # Lu [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C;N # Lu [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539;N # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E;N # Lu [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544;N # Lu [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546;N # Lu MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550;N # Lu [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5;N # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0;N # Lu [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1;N # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA;N # Ll [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB;N # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA;N # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB;N # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714;N # Ll [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715;N # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734;N # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735;N # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E;N # Ll [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F;N # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E;N # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F;N # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788;N # Ll [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789;N # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8;N # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9;N # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2;N # Ll [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3;N # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB;N # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF;N # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1D800..1D9FF;N # So [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD +1DA00..1DA36;N # Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN +1DA37..1DA3A;N # So [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE +1DA3B..1DA6C;N # Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT +1DA6D..1DA74;N # So [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING +1DA75;N # Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS +1DA76..1DA83;N # So [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH +1DA84;N # Mn SIGNWRITING LOCATION HEAD NECK +1DA85..1DA86;N # So [2] SIGNWRITING LOCATION TORSO..SIGNWRITING LOCATION LIMBS DIGITS +1DA87..1DA8B;N # Po [5] SIGNWRITING COMMA..SIGNWRITING PARENTHESIS +1DA9B..1DA9F;N # Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 +1DAA1..1DAAF;N # Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 +1DF00..1DF09;N # Ll [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK +1DF0A;N # Lo LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK +1DF0B..1DF1E;N # Ll [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL +1E000..1E006;N # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018;N # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021;N # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024;N # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A;N # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E100..1E12C;N # Lo [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W +1E130..1E136;N # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E137..1E13D;N # Lm [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E140..1E149;N # Nd [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE +1E14E;N # Lo NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ +1E14F;N # So NYIAKENG PUACHUE HMONG CIRCLED CA +1E290..1E2AD;N # Lo [30] TOTO LETTER PA..TOTO LETTER A +1E2AE;N # Mn TOTO SIGN RISING TONE +1E2C0..1E2EB;N # Lo [44] WANCHO LETTER AA..WANCHO LETTER YIH +1E2EC..1E2EF;N # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI +1E2F0..1E2F9;N # Nd [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE +1E2FF;N # Sc WANCHO NGUN SIGN +1E7E0..1E7E6;N # Lo [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO +1E7E8..1E7EB;N # Lo [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE +1E7ED..1E7EE;N # Lo [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE +1E7F0..1E7FE;N # Lo [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE +1E800..1E8C4;N # Lo [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON +1E8C7..1E8CF;N # No [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE +1E8D0..1E8D6;N # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS +1E900..1E943;N # L& [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA +1E944..1E94A;N # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA +1E94B;N # Lm ADLAM NASALIZATION MARK +1E950..1E959;N # Nd [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE +1E95E..1E95F;N # Po [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK +1EC71..1ECAB;N # No [59] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ NUMBER PREFIXED NINE +1ECAC;N # So INDIC SIYAQ PLACEHOLDER +1ECAD..1ECAF;N # No [3] INDIC SIYAQ FRACTION ONE QUARTER..INDIC SIYAQ FRACTION THREE QUARTERS +1ECB0;N # Sc INDIC SIYAQ RUPEE MARK +1ECB1..1ECB4;N # No [4] INDIC SIYAQ NUMBER ALTERNATE ONE..INDIC SIYAQ ALTERNATE LAKH MARK +1ED01..1ED2D;N # No [45] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ NUMBER NINETY THOUSAND +1ED2E;N # So OTTOMAN SIYAQ MARRATAN +1ED2F..1ED3D;N # No [15] OTTOMAN SIYAQ ALTERNATE NUMBER TWO..OTTOMAN SIYAQ FRACTION ONE SIXTH +1EE00..1EE03;N # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F;N # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22;N # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24;N # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27;N # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32;N # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37;N # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39;N # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B;N # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42;N # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47;N # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49;N # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B;N # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F;N # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52;N # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54;N # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57;N # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59;N # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B;N # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D;N # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F;N # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62;N # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64;N # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A;N # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72;N # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77;N # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C;N # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E;N # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89;N # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B;N # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3;N # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9;N # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB;N # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1EEF0..1EEF1;N # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL +1F000..1F003;N # So [4] MAHJONG TILE EAST WIND..MAHJONG TILE NORTH WIND +1F004;W # So MAHJONG TILE RED DRAGON +1F005..1F02B;N # So [39] MAHJONG TILE GREEN DRAGON..MAHJONG TILE BACK +1F030..1F093;N # So [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F0A0..1F0AE;N # So [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0B1..1F0BF;N # So [15] PLAYING CARD ACE OF HEARTS..PLAYING CARD RED JOKER +1F0C1..1F0CE;N # So [14] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD KING OF DIAMONDS +1F0CF;W # So PLAYING CARD BLACK JOKER +1F0D1..1F0F5;N # So [37] PLAYING CARD ACE OF CLUBS..PLAYING CARD TRUMP-21 +1F100..1F10A;A # No [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA +1F10B..1F10C;N # No [2] DINGBAT CIRCLED SANS-SERIF DIGIT ZERO..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO +1F10D..1F10F;N # So [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH +1F110..1F12D;A # So [30] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED CD +1F12E..1F12F;N # So [2] CIRCLED WZ..COPYLEFT SYMBOL +1F130..1F169;A # So [58] SQUARED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F16A..1F16F;N # So [6] RAISED MC SIGN..CIRCLED HUMAN FIGURE +1F170..1F18D;A # So [30] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED SA +1F18E;W # So NEGATIVE SQUARED AB +1F18F..1F190;A # So [2] NEGATIVE SQUARED WC..SQUARE DJ +1F191..1F19A;W # So [10] SQUARED CL..SQUARED VS +1F19B..1F1AC;A # So [18] SQUARED THREE D..SQUARED VOD +1F1AD;N # So MASK WORK SYMBOL +1F1E6..1F1FF;N # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z +1F200..1F202;W # So [3] SQUARE HIRAGANA HOKA..SQUARED KATAKANA SA +1F210..1F23B;W # So [44] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-914D +1F240..1F248;W # So [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 +1F250..1F251;W # So [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT +1F260..1F265;W # So [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F300..1F320;W # So [33] CYCLONE..SHOOTING STAR +1F321..1F32C;N # So [12] THERMOMETER..WIND BLOWING FACE +1F32D..1F335;W # So [9] HOT DOG..CACTUS +1F336;N # So HOT PEPPER +1F337..1F37C;W # So [70] TULIP..BABY BOTTLE +1F37D;N # So FORK AND KNIFE WITH PLATE +1F37E..1F393;W # So [22] BOTTLE WITH POPPING CORK..GRADUATION CAP +1F394..1F39F;N # So [12] HEART WITH TIP ON THE LEFT..ADMISSION TICKETS +1F3A0..1F3CA;W # So [43] CAROUSEL HORSE..SWIMMER +1F3CB..1F3CE;N # So [4] WEIGHT LIFTER..RACING CAR +1F3CF..1F3D3;W # So [5] CRICKET BAT AND BALL..TABLE TENNIS PADDLE AND BALL +1F3D4..1F3DF;N # So [12] SNOW CAPPED MOUNTAIN..STADIUM +1F3E0..1F3F0;W # So [17] HOUSE BUILDING..EUROPEAN CASTLE +1F3F1..1F3F3;N # So [3] WHITE PENNANT..WAVING WHITE FLAG +1F3F4;W # So WAVING BLACK FLAG +1F3F5..1F3F7;N # So [3] ROSETTE..LABEL +1F3F8..1F3FA;W # So [3] BADMINTON RACQUET AND SHUTTLECOCK..AMPHORA +1F3FB..1F3FF;W # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 +1F400..1F43E;W # So [63] RAT..PAW PRINTS +1F43F;N # So CHIPMUNK +1F440;W # So EYES +1F441;N # So EYE +1F442..1F4FC;W # So [187] EAR..VIDEOCASSETTE +1F4FD..1F4FE;N # So [2] FILM PROJECTOR..PORTABLE STEREO +1F4FF..1F53D;W # So [63] PRAYER BEADS..DOWN-POINTING SMALL RED TRIANGLE +1F53E..1F54A;N # So [13] LOWER RIGHT SHADOWED WHITE CIRCLE..DOVE OF PEACE +1F54B..1F54E;W # So [4] KAABA..MENORAH WITH NINE BRANCHES +1F54F;N # So BOWL OF HYGIEIA +1F550..1F567;W # So [24] CLOCK FACE ONE OCLOCK..CLOCK FACE TWELVE-THIRTY +1F568..1F579;N # So [18] RIGHT SPEAKER..JOYSTICK +1F57A;W # So MAN DANCING +1F57B..1F594;N # So [26] LEFT HAND TELEPHONE RECEIVER..REVERSED VICTORY HAND +1F595..1F596;W # So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS +1F597..1F5A3;N # So [13] WHITE DOWN POINTING LEFT HAND INDEX..BLACK DOWN POINTING BACKHAND INDEX +1F5A4;W # So BLACK HEART +1F5A5..1F5FA;N # So [86] DESKTOP COMPUTER..WORLD MAP +1F5FB..1F5FF;W # So [5] MOUNT FUJI..MOYAI +1F600..1F64F;W # So [80] GRINNING FACE..PERSON WITH FOLDED HANDS +1F650..1F67F;N # So [48] NORTH WEST POINTING LEAF..REVERSE CHECKER BOARD +1F680..1F6C5;W # So [70] ROCKET..LEFT LUGGAGE +1F6C6..1F6CB;N # So [6] TRIANGLE WITH ROUNDED CORNERS..COUCH AND LAMP +1F6CC;W # So SLEEPING ACCOMMODATION +1F6CD..1F6CF;N # So [3] SHOPPING BAGS..BED +1F6D0..1F6D2;W # So [3] PLACE OF WORSHIP..SHOPPING TROLLEY +1F6D3..1F6D4;N # So [2] STUPA..PAGODA +1F6D5..1F6D7;W # So [3] HINDU TEMPLE..ELEVATOR +1F6DD..1F6DF;W # So [3] PLAYGROUND SLIDE..RING BUOY +1F6E0..1F6EA;N # So [11] HAMMER AND WRENCH..NORTHEAST-POINTING AIRPLANE +1F6EB..1F6EC;W # So [2] AIRPLANE DEPARTURE..AIRPLANE ARRIVING +1F6F0..1F6F3;N # So [4] SATELLITE..PASSENGER SHIP +1F6F4..1F6FC;W # So [9] SCOOTER..ROLLER SKATE +1F700..1F773;N # So [116] ALCHEMICAL SYMBOL FOR QUINTESSENCE..ALCHEMICAL SYMBOL FOR HALF OUNCE +1F780..1F7D8;N # So [89] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7E0..1F7EB;W # So [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE +1F7F0;W # So HEAVY EQUALS SIGN +1F800..1F80B;N # So [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD +1F810..1F847;N # So [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW +1F850..1F859;N # So [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW +1F860..1F887;N # So [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW +1F890..1F8AD;N # So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS +1F8B0..1F8B1;N # So [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST +1F900..1F90B;N # So [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT +1F90C..1F93A;W # So [47] PINCHED FINGERS..FENCER +1F93B;N # So MODERN PENTATHLON +1F93C..1F945;W # So [10] WRESTLERS..GOAL NET +1F946;N # So RIFLE +1F947..1F9FF;W # So [185] FIRST PLACE MEDAL..NAZAR AMULET +1FA00..1FA53;N # So [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA60..1FA6D;N # So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA70..1FA74;W # So [5] BALLET SHOES..THONG SANDAL +1FA78..1FA7C;W # So [5] DROP OF BLOOD..CRUTCH +1FA80..1FA86;W # So [7] YO-YO..NESTING DOLLS +1FA90..1FAAC;W # So [29] RINGED PLANET..HAMSA +1FAB0..1FABA;W # So [11] FLY..NEST WITH EGGS +1FAC0..1FAC5;W # So [6] ANATOMICAL HEART..PERSON WITH CROWN +1FAD0..1FAD9;W # So [10] BLUEBERRIES..JAR +1FAE0..1FAE7;W # So [8] MELTING FACE..BUBBLES +1FAF0..1FAF6;W # So [7] HAND WITH INDEX FINGER AND THUMB CROSSED..HEART HANDS +1FB00..1FB92;N # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK +1FB94..1FBCA;N # So [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON +1FBF0..1FBF9;N # Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +20000..2A6DF;W # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A6E0..2A6FF;W # Cn [32] <reserved-2A6E0>..<reserved-2A6FF> +2A700..2B738;W # Lo [4153] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B738 +2B739..2B73F;W # Cn [7] <reserved-2B739>..<reserved-2B73F> +2B740..2B81D;W # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B81E..2B81F;W # Cn [2] <reserved-2B81E>..<reserved-2B81F> +2B820..2CEA1;W # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEA2..2CEAF;W # Cn [14] <reserved-2CEA2>..<reserved-2CEAF> +2CEB0..2EBE0;W # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBE1..2F7FF;W # Cn [3103] <reserved-2EBE1>..<reserved-2F7FF> +2F800..2FA1D;W # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +2FA1E..2FA1F;W # Cn [2] <reserved-2FA1E>..<reserved-2FA1F> +2FA20..2FFFD;W # Cn [1502] <reserved-2FA20>..<reserved-2FFFD> +30000..3134A;W # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +3134B..3FFFD;W # Cn [60595] <reserved-3134B>..<reserved-3FFFD> +E0001;N # Cf LANGUAGE TAG +E0020..E007F;N # Cf [96] TAG SPACE..CANCEL TAG +E0100..E01EF;A # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 +F0000..FFFFD;A # Co [65534] <private-use-F0000>..<private-use-FFFFD> +100000..10FFFD;A # Co [65534] <private-use-100000>..<private-use-10FFFD> + +# EOF diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index ecc30b40c6..6f55f6da45 100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -23,55 +23,64 @@ -mode(compile). --record(cp, {name, class, dec, comp, cs}). +-record(cp, {name, class, dec, comp, cs, cat}). -define(MOD, "unicode_util"). main(_) -> %% Parse main table - {ok, UD} = file:open("../uc_spec/UnicodeData.txt", [read, raw, {read_ahead, 1000000}]), + UD = file_open("../uc_spec/UnicodeData.txt"), Data0 = foldl(fun parse_unicode_data/2, [], UD), Data1 = array:from_orddict(lists:reverse(Data0)), ok = file:close(UD), %% Special Casing table - {ok, SC} = file:open("../uc_spec/SpecialCasing.txt", [read, raw, {read_ahead, 1000000}]), + SC = file_open("../uc_spec/SpecialCasing.txt"), Data2 = foldl(fun parse_special_casing/2, Data1, SC), ok = file:close(SC), %% Casing Folding table - {ok, CF} = file:open("../uc_spec/CaseFolding.txt", [read, raw, {read_ahead, 1000000}]), + CF = file_open("../uc_spec/CaseFolding.txt"), Data = foldl(fun parse_case_folding/2, Data2, CF), ok = file:close(CF), %% Normalization - {ok, ExclF} = file:open("../uc_spec/CompositionExclusions.txt", [read, raw, {read_ahead, 1000000}]), + ExclF = file_open("../uc_spec/CompositionExclusions.txt"), ExclData = foldl(fun parse_comp_excl/2, Data, ExclF), ok = file:close(ExclF), %% GraphemeBreakProperty table - {ok, Emoji} = file:open("../uc_spec/emoji-data.txt", [read, raw, {read_ahead, 1000000}]), + Emoji = file_open("../uc_spec/emoji-data.txt"), Props00 = foldl(fun parse_properties/2, [], Emoji), %% Filter Extended_Pictographic class which we are interested in. Props0 = [EP || {extended_pictographic, _} = EP <- Props00], ok = file:close(Emoji), - {ok, GBPF} = file:open("../uc_spec/GraphemeBreakProperty.txt", [read, raw, {read_ahead, 1000000}]), + GBPF = file_open("../uc_spec/GraphemeBreakProperty.txt"), Props1 = foldl(fun parse_properties/2, Props0, GBPF), ok = file:close(GBPF), - {ok, PropF} = file:open("../uc_spec/PropList.txt", [read, raw, {read_ahead, 1000000}]), + PropF = file_open("../uc_spec/PropList.txt"), Props2 = foldl(fun parse_properties/2, Props1, PropF), ok = file:close(PropF), Props = sofs:to_external(sofs:relation_to_family(sofs:relation(Props2))), + WidthF = file_open("../uc_spec/EastAsianWidth.txt"), + WideCs = foldl(fun parse_widths/2, [], WidthF), + ok = file:close(WidthF), + %% Make module {ok, Out} = file:open(?MOD++".erl", [write]), - gen_file(Out, Data, ExclData, maps:from_list(Props)), + gen_file(Out, Data, ExclData, maps:from_list(Props), WideCs), ok = file:close(Out), ok. +file_open(File) -> + {ok, Fd} = file:open(File, [read, raw, {read_ahead, 1000000}]), + Fd. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% parse_unicode_data(Line0, Acc) -> Line = string:chomp(Line0), - [CodePoint,Name,_Cat,Class,_BiDi,Decomp, + [CodePoint,Name,Cat,Class,_BiDi,Decomp, _N1,_N2,_N3,_BDMirror,_Uni1,_Iso|Case] = tokens(Line, ";"), {Dec,Comp} = case to_decomp(Decomp) of {_, _} = Compabil -> {[], Compabil}; @@ -79,7 +88,7 @@ parse_unicode_data(Line0, Acc) -> end, [{hex_to_int(CodePoint), #cp{name=list_to_binary(Name),class=to_class(Class), - dec=Dec, comp=Comp, cs=to_case(Case)}} + dec=Dec, comp=Comp, cs=to_case(Case), cat=Cat}} |Acc]. to_class(String) -> @@ -152,7 +161,62 @@ parse_properties(Line0, Acc) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -gen_file(Fd, Data, ExclData, Props) -> +%% Pick ranges that are wide when seen from a non East Asian context, +%% That way we can minimize the data, every other code point is considered narrow. +%% We loose information but hopefully keep the important width for a standard +%% terminal. +parse_widths(Line0, Acc) -> + [{WidthClass, {From, _To}=Range}] = parse_properties(Line0, []), + case is_default_width(From, WidthClass) of + {true, narrow} -> + Acc; + {false, narrow} -> + [Range|Acc]; + {true, RuleRange} -> + [RuleRange|Acc] +%%% {false, rule_execption} -> i.e. narrow codepoint in wide range +%%% Should not happen in current specs + end. + +is_default_width(Index, WD) -> + if + 16#3400 =< Index, Index =< 16#4DBF -> + if WD =:= w orelse WD =:= f -> + {true, {16#3400, 16#4DBF}}; + true -> + {false, rule_execption} + end; + 16#4E00 =< Index, Index =< 16#9FFF -> + if WD =:= w orelse WD =:= f -> + {true, {16#4E00, 16#9FFF}}; + true -> + {false, rule_execption} + end; + 16#F900 =< Index, Index =< 16#FAFF -> + if WD =:= w orelse WD =:= f -> + {true, {16#F900, 16#FAFF}}; + true -> + {false, rule_execption} + end; + 16#20000 =< Index, Index =< 16#2FFFD -> + if WD =:= w orelse WD =:= f -> + {true, {16#20000, 16#2FFFD}}; + true -> + {false, rule_execption} + end; + 16#30000 =< Index, Index =< 16#3FFFD -> + if WD =:= w orelse WD =:= f -> + {true, {16#30000, 16#3FFFD}}; + true -> + {false, rule_execption} + end; + true -> + {WD =:= n orelse WD =:= na orelse WD == h orelse WD =:= a, narrow} + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +gen_file(Fd, Data, ExclData, Props, WideCs) -> gen_header(Fd), gen_static(Fd), gen_norm(Fd), @@ -162,6 +226,7 @@ gen_file(Fd, Data, ExclData, Props) -> gen_compose_pairs(Fd, ExclData, Data), gen_case_table(Fd, Data), gen_unicode_table(Fd, Data), + gen_width_table(Fd, WideCs), ok. gen_header(Fd) -> @@ -173,26 +238,32 @@ gen_header(Fd) -> io:put_chars(Fd, "-export([whitespace/0, is_whitespace/1]).\n"), io:put_chars(Fd, "-export([uppercase/1, lowercase/1, titlecase/1, casefold/1]).\n\n"), io:put_chars(Fd, "-export([spec_version/0, lookup/1, get_case/1]).\n"), + io:put_chars(Fd, "-export([is_wide/1]).\n"), io:put_chars(Fd, "-compile({inline, [class/1]}).\n"), io:put_chars(Fd, "-compile(nowarn_unused_vars).\n"), io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc/1, gc_prepend/2]}).\n"), - io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n\n"), + io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n"), + io:put_chars(Fd, "-define(IS_CP(CP), (is_integer(CP) andalso 0 =< CP andalso CP < 16#110000)).\n\n\n"), ok. gen_static(Fd) -> io:put_chars(Fd, "-spec lookup(char()) -> #{'canon':=[{byte(),char()}], 'ccc':=byte(), " - "'compat':=[] | {atom(),[{byte(),char()}]}}.\n"), - io:put_chars(Fd, "lookup(Codepoint) ->\n" - " {CCC,Can,Comp} = unicode_table(Codepoint),\n" - " #{ccc=>CCC, canon=>Can, compat=>Comp}.\n\n"), + "'compat':=[] | {atom(),[{byte(),char()}]}, 'category':={atom(),atom()}}.\n"), + io:put_chars(Fd, "lookup(Codepoint) when ?IS_CP(Codepoint) ->\n" + " {CCC,Can,Comp,Cat} = unicode_table(Codepoint),\n" + " #{ccc=>CCC, canon=>Can, compat=>Comp, category=>category(Codepoint,Cat)}.\n\n"), + io:put_chars(Fd, "-spec get_case(char()) -> #{'fold':=gc(), 'lower':=gc(), 'title':=gc(), 'upper':=gc()}.\n"), - io:put_chars(Fd, "get_case(Codepoint) ->\n" + io:put_chars(Fd, "get_case(Codepoint) when ?IS_CP(Codepoint) ->\n" " case case_table(Codepoint) of\n" " {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n" " {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n" " end.\n\n"), + io:put_chars(Fd, "spec_version() -> {14,0}.\n\n\n"), - io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), + io:put_chars(Fd, "class(Codepoint) when ?IS_CP(Codepoint) -> \n" + " {CCC,_,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), + io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "uppercase(Str0) ->\n"), @@ -217,6 +288,7 @@ gen_static(Fd) -> io:put_chars(Fd, " [] -> [];\n"), io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "-spec titlecase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "titlecase(Str0) ->\n"), @@ -229,6 +301,7 @@ gen_static(Fd) -> io:put_chars(Fd, " [] -> [];\n"), io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "-spec casefold(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "casefold(Str0) ->\n"), @@ -242,6 +315,19 @@ gen_static(Fd) -> io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "%% Returns true if the character is considered wide in non east asian context.\n"), + io:put_chars(Fd, "-spec is_wide(gc()) -> boolean().\n"), + io:put_chars(Fd, "is_wide(C) when ?IS_CP(C) ->\n"), + io:put_chars(Fd, " is_wide_cp(C);\n"), + io:put_chars(Fd, "is_wide([_, 16#FE0E|Cs]) -> true; %% Presentation sequence\n"), + io:put_chars(Fd, "is_wide([_, 16#FE0F|Cs]) -> true; %% Presentation sequence\n"), + io:put_chars(Fd, "is_wide([C|Cs]) when ?IS_CP(C) ->\n"), + io:put_chars(Fd, " is_wide_cp(C) orelse is_wide(Cs);\n"), + io:put_chars(Fd, "is_wide([]) ->\n false.\n\n"), + + io:put_chars(Fd, "category(CP, lookup_category) ->\n" + " cat_translate(lookup_category(CP));\n" + "category(_, Def) -> cat_translate(Def).\n\n"), ok. gen_norm(Fd) -> @@ -249,7 +335,7 @@ gen_norm(Fd) -> "-spec nfd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -259,7 +345,7 @@ gen_norm(Fd) -> "-spec nfkd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose_compat(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -269,7 +355,7 @@ gen_norm(Fd) -> "-spec nfc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 256 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 256 -> [GC|R];\n" " [GC|Str] -> [compose(decompose(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -279,7 +365,7 @@ gen_norm(Fd) -> "-spec nfkc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [compose_compat_0(decompose_compat(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -288,13 +374,13 @@ gen_norm(Fd) -> io:put_chars(Fd, "decompose(CP) when is_integer(CP), CP < 16#AC00, 16#D7A3 > CP ->\n" " case unicode_table(CP) of\n" - " {_,[],_} -> CP;\n" - " {_,CPs,_} -> canonical_order(CPs)\n" + " {_,[],_,_} -> CP;\n" + " {_,CPs,_,_} -> canonical_order(CPs)\n" " end;\n" "decompose(CP) ->\n" " canonical_order(decompose_1(CP)).\n" "\n" - "decompose_1(CP) when 16#AC00 =< CP, CP =< 16#D7A3 ->\n" + "decompose_1(CP) when is_integer(CP), 16#AC00 =< CP, CP =< 16#D7A3 ->\n" " Syll = CP-16#AC00,\n" " T = 28,\n" " N = 588,\n" @@ -306,8 +392,8 @@ gen_norm(Fd) -> " end;\n" "decompose_1(CP) when is_integer(CP) ->\n" " case unicode_table(CP) of\n" - " {CCC, [],_} -> [{CCC,CP}];\n" - " {_, CPs, _} -> CPs\n" + " {CCC, [],_,_} -> [{CCC,CP}];\n" + " {_,CPs,_,_} -> CPs\n" " end;\n" "decompose_1([CP|CPs]) ->\n" " decompose_1(CP) ++ decompose_1(CPs);\n" @@ -331,14 +417,14 @@ gen_norm(Fd) -> io:put_chars(Fd, "decompose_compat(CP) when is_integer(CP), CP < 16#AC00, 16#D7A3 > CP ->\n" " case unicode_table(CP) of\n" - " {_, [], []} -> CP;\n" - " {_, _, {_,CPs}} -> canonical_order(CPs);\n" - " {_, CPs, _} -> canonical_order(CPs)\n" + " {_, [], [], _} -> CP;\n" + " {_, _, {_,CPs}, _} -> canonical_order(CPs);\n" + " {_, CPs, _, _} -> canonical_order(CPs)\n" " end;\n" "decompose_compat(CP) ->\n" " canonical_order(decompose_compat_1(CP)).\n" "\n" - "decompose_compat_1(CP) when 16#AC00 =< CP, CP =< 16#D7A3 ->\n" + "decompose_compat_1(CP) when is_integer(CP), 16#AC00 =< CP, CP =< 16#D7A3 ->\n" " Syll = CP-16#AC00,\n" " T = 28,\n" " N = 588,\n" @@ -350,23 +436,24 @@ gen_norm(Fd) -> " end;\n" "decompose_compat_1(CP) when is_integer(CP) ->\n" " case unicode_table(CP) of\n" - " {CCC, [], []} -> [{CCC,CP}];\n" - " {_, _, {_,CPs}} -> CPs;\n" - " {_, CPs, _} -> CPs\n" + " {CCC, [], [], _} -> [{CCC,CP}];\n" + " {_, _, {_,CPs}, _} -> CPs;\n" + " {_, CPs, _, _} -> CPs\n" " end;\n" "decompose_compat_1([CP|CPs]) ->\n" " decompose_compat_1(CP) ++ decompose_compat_1(CPs);\n" - "decompose_compat_1([]) -> [].\n"), + "decompose_compat_1([]) -> [].\n\n"), io:put_chars(Fd, "compose(CP) when is_integer(CP) -> CP;\n" "compose([Lead,Vowel|Trail]) %% Hangul\n" - " when 16#1100 =< Lead, Lead =< 16#1112 ->\n" + " when is_integer(Lead), 16#1100 =< Lead, Lead =< 16#1112, is_integer(Vowel) ->\n" " if 16#1161 =< Vowel, Vowel =< 16#1175 ->\n" " CP = 16#AC00 + ((Lead - 16#1100) * 588) + ((Vowel - 16#1161) * 28),\n" " case Trail of\n" - " [T|Acc] when 16#11A7 =< T, T =< 16#11C2 -> nolist(CP+T-16#11A7,Acc);\n" + " [T|Acc] when is_integer(T), 16#11A7 =< T, T =< 16#11C2 ->" + " nolist(CP+T-16#11A7,Acc);\n" " Acc -> nolist(CP,Acc)\n" " end;\n" " true ->\n" @@ -408,11 +495,12 @@ gen_norm(Fd) -> " end.\n\n" "compose_compat(CP) when is_integer(CP) -> CP;\n" "compose_compat([Lead,Vowel|Trail]) %% Hangul\n" - " when 16#1100 =< Lead, Lead =< 16#1112 ->\n" + " when is_integer(Lead), 16#1100 =< Lead, Lead =< 16#1112, is_integer(Vowel) ->\n" " if 16#1161 =< Vowel, Vowel =< 16#1175 ->\n" " CP = 16#AC00 + ((Lead - 16#1100) * 588) + ((Vowel - 16#1161) * 28),\n" " case Trail of\n" - " [T|Acc] when 16#11A7 =< T, T =< 16#11C2 -> nolist(CP+T-16#11A7,Acc);\n" + " [T|Acc] when is_integer(T), 16#11A7 =< T, T =< 16#11C2 ->" + " nolist(CP+T-16#11A7,Acc);\n" " Acc -> nolist(CP,Acc)\n" " end;\n" " true ->\n" @@ -462,7 +550,7 @@ gen_ws(Fd, Props) -> gen_cp(Fd) -> io:put_chars(Fd, "-spec cp(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), - io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp([C|_]=L) when ?IS_CP(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), @@ -470,8 +558,8 @@ gen_cp(Fd) -> io:put_chars(Fd, "cp(<<>>) -> [];\n"), io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), io:put_chars(Fd, "\n"), - io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), - io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([C], R) when ?IS_CP(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when ?IS_CP(C) -> [C|cpl_cont(T, R)];\n"), io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), @@ -542,18 +630,18 @@ gen_gc(Fd, GBP) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "gc([]=R) -> R;\n" - "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([CP]=R) when ?IS_CP(CP) -> R;\n" "gc([$\\r=CP|R0]) ->\n" " case cp(R0) of % Don't break CRLF\n" " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" " T -> [CP|T]\n" " end;\n" - "gc([CP1|T1]=T) when CP1 < 256 ->\n" + "gc([CP1|T1]=T) when ?IS_CP(CP1), CP1 < 256 ->\n" " case T1 of\n" - " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " [CP2|_] when is_integer(CP2), 0 =< CP2, CP2 < 256 -> T; %% Ascii Fast path\n" " _ -> %% Keep the tail binary.\n" " case cp_no_bin(T1) of\n" - " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " [CP2|_]=T3 when is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" " binary_found -> gc_1(T);\n" " T4 -> gc_1([CP1|T4])\n" " end\n" @@ -568,7 +656,7 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" - "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" + "gc([CP|_]=T) when ?IS_CP(CP) -> gc_1(T);\n" "gc(Str) ->\n" " case cp(Str) of\n" " {error,_}=Error -> Error;\n" @@ -596,13 +684,14 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, - "gc_1([CP|R]=R0) when CP < 256 ->\n" - " case R of\n" - " [CP2|_] when CP2 < 256 -> R0;\n" - " _ -> gc_extend(cp(R), R, CP)\n" - " end;\n", - []), + io:put_chars(Fd, + "gc_1([CP|R]=R0) when is_integer(CP), 0 =< CP, CP < 256 ->\n" + " case R of\n" + " [CP2|_] when is_integer(CP2), 0 =< CP2, CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n" + "gc_1([CP|_]) when not ?IS_CP(CP) ->\n" + " error({badarg,CP});\n"), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -623,7 +712,7 @@ gen_gc(Fd, GBP) -> GenHangulT = fun(Range) -> io:format(Fd, "gc_1~s gc_h_T(R1,[CP]);\n", [gen_clause(Range)]) end, [GenHangulT(CP) || CP <- merge_ranges(maps:get(t,GBP))], io:put_chars(Fd, "%% Handle Hangul LV and LVT special, since they are large\n"), - io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), + io:put_chars(Fd, "gc_1([CP|_]=R0) when is_integer(CP), 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, @@ -739,7 +828,9 @@ gen_gc(Fd, GBP) -> [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:format(Fd, " [CP|R1] when is_integer(CP), ~w =< CP, CP =< ~w ->\n" + " gc_extend2(cp(R1),R1,[CP,CP0]);~n", + [RLess, RLarge]), io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), @@ -780,6 +871,7 @@ gen_gc(Fd, GBP) -> " _ -> gc_extend2(R1, R0, Acc)\n" " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|_], _R0, _Acc) when not ?IS_CP(CP) -> error(badarg);\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLV(CP) || CP <- merge_ranges(maps:get(lv,GBP))], @@ -806,50 +898,234 @@ gen_compose_pairs(Fd, ExclData, Data) -> [io:format(Fd, "compose_pair(~w,~w) -> ~w;~n", [A,B,CP]) || {[A,B],CP} <- lists:sort(DeCmp2)], io:put_chars(Fd, "compose_pair(_,_) -> false.\n\n"), - io:put_chars(Fd, "nolist(CP, []) -> CP;\nnolist(CP,L) -> [CP|L].\n\n"), + io:put_chars(Fd, "nolist(CP, []) when ?IS_CP(CP) -> CP;\n" + "nolist(CP, L) when ?IS_CP(CP) -> [CP|L].\n\n"), ok. gen_case_table(Fd, Data) -> - Case = array:foldr(fun(CP, #cp{cs={U0,L0,T0,F0}}, Acc) -> - U = def_cp(U0,CP), - L = def_cp(L0,CP), - T = def_cp(T0,CP), - F = def_cp(F0,CP), - case T =:= U andalso F =:= L of - true -> - [{CP,{U,L}}|Acc]; - false -> - [{CP,{U,L,T,F}}|Acc] - end; - (_CP, _, Acc) -> Acc - end, [], Data), + HC = fun(CP, #cp{cs=Cs}, Acc) -> + case case_data(CP, Cs) of + default -> Acc; + CaseData -> [{CP,CaseData}|Acc] + end + end, + Case = array:sparse_foldr(HC, [], Data), [io:format(Fd, "case_table(~w) -> ~w;\n", [CP, Map])|| {CP,Map} <- Case], io:format(Fd, "case_table(CP) -> {CP, CP}.\n\n",[]), ok. +case_data(CP, {U0,L0,T0,F0}) -> + U = def_cp(U0,CP), + L = def_cp(L0,CP), + T = def_cp(T0,CP), + F = def_cp(F0,CP), + case T =:= U andalso F =:= L of + true -> {U,L}; + false -> {U,L,T,F} + end; +case_data(_, _) -> + default. + def_cp([], CP) -> CP; def_cp(CP, _) -> CP. gen_unicode_table(Fd, Data) -> - FixCanon = fun(_, #cp{class=CCC, dec=Dec, comp=Comp}) -> + FixCanon = fun(_, #cp{class=CCC, dec=Dec, comp=Comp, cat=Cat}) -> Canon = decompose(Dec,Data), - #{ccc=>CCC, canonical=>Canon, compat=>Comp} + #{ccc=>CCC, canonical=>Canon, compat=>Comp, cat=>Cat} end, AofMaps0 = array:sparse_map(FixCanon, Data), - FixCompat = fun(_, #{ccc:=CCC, canonical:=Canon, compat:=Comp}) -> + FixCompat = fun(_, #{ccc:=CCC, canonical:=Canon, compat:=Comp, cat:=Cat}) -> Compat = decompose_compat(Canon, Comp, AofMaps0), - {CCC, Canon, Compat} + {CCC, Canon, Compat, category(Cat)} end, AofMaps1 = array:sparse_map(FixCompat, AofMaps0), Dict0 = array:sparse_to_orddict(AofMaps1), - Def = {0, [], []}, - Dict = lists:filter(fun({_, Map}) -> Map =/= Def end, Dict0), - - [io:format(Fd, "unicode_table(~w) -> ~w;~n", [CP, Map]) || {CP,Map} <- Dict], + Def = {0, [], [], lookup_category}, + {NonDef, CatTable} = lists:partition(fun({_, {0,[],[],_Cat}}) -> false; + (_) -> true + end, Dict0), + + %% Export testfile + %% Dict1 = lists:map(fun({Id,{CCC, Canon, Compat, Cat}}) -> + %% {_, ECat} = lists:keyfind(Cat, 1, category_translate()), + %% {Id, {CCC, Canon, Compat, ECat}} + %% end, Dict0), + %% file:write_file("../test/unicode_util_SUITE_data/unicode_table.bin", term_to_binary(Dict1, [compressed])), + + [io:format(Fd, "unicode_table(~w) -> ~w;~n", [CP, Map]) || {CP,Map} <- NonDef], io:format(Fd, "unicode_table(_) -> ~w.~n~n",[Def]), + + [io:format(Fd, "cat_translate(~w) -> ~w;~n", [Cat, EC]) || {Cat,EC} <- category_translate()], + io:format(Fd, "cat_translate(Cat) -> error({internal_error, Cat}).~n~n",[]), + gen_category(Fd, CatTable, Data), + ok. + +category([C,Sub]) -> + list_to_atom([C-$A+$a, Sub]). + +category_translate() -> + [{lu, {letter, uppercase}}, % Letter, Uppercase + {ll, {letter, lowercase}}, % Letter, Lowercase + {lt, {letter, titlecase}}, % Letter, Titlecase + {mn, {mark, non_spacing}}, % Mark, Non-Spacing + {mc, {mark, spacing_combining}}, % Mark, Spacing Combining + {me, {mark, enclosing}}, % Mark, Enclosing + {nd, {number, decimal}}, % Number, Decimal Digit + {nl, {number, letter}}, % Number, Letter + {no, {number, other}}, % Number, Other + {zs, {separator, space}}, % Separator, Space + {zl, {separator, line}}, % Separator, Line + {zp, {separator, paragraph}}, % Separator, Paragraph + {cc, {other, control}}, % Other, Control + {cf, {other, format}}, % Other, Format + {cs, {other, surrogate}}, % Other, Surrogate + {co, {other, private}}, % Other, Private Use + {cn, {other, not_assigned}}, % Other, Not Assigned (no characters in the file have this property) + {lm, {letter, modifier}}, % Letter, Modifier + {lo, {letter, other}}, % Letter, Other + {pc, {punctuation, connector}}, % Punctuation, Connector + {pd, {punctuation, dash}}, % Punctuation, Dash + {ps, {punctuation, open}}, % Punctuation, Open + {pe, {punctuation, close}}, % Punctuation, Close + {pi, {punctuation, initial}}, % Punctuation, Initial quote (may behave like Ps or Pe depending on usage) + {pf, {punctuation, final}}, % Punctuation, Final quote (may behave like Ps or Pe depending on usage) + {po, {punctuation, other}}, % Punctuation, Other + {sm, {symbol, math}}, % Symbol, Math + {sc, {symbol, currency}}, % Symbol, Currency + {sk, {symbol, modifier}}, % Symbol, Modifier + {so, {symbol, other}}]. % Symbol, Other + +gen_category(Fd, [{CP, {_, _, _, Cat}}|Rest], All) -> + gen_category(Fd, Rest, Cat, CP, CP, All, []). + +gen_category(Fd, [{CP, {_, _, _, NextCat}}|Rest], Cat, Start, End, All, Acc) + when End+1 =:= CP -> + IsLetterCat = letter_cat(NextCat, Cat), + if NextCat =:= Cat -> + gen_category(Fd, Rest, Cat, Start, CP, All, Acc); + IsLetterCat -> + gen_category(Fd, Rest, letter, Start, CP, All, Acc); + Start =:= End -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]), + gen_category(Fd, Rest, NextCat, CP, CP, All, Acc); + true -> + case Cat of + letter -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> subcat_letter(CP);~n", + [Start, End]), + gen_category(Fd, Rest, NextCat, CP, CP, All, + lists:reverse(lists:seq(Start, End)) ++ Acc); + _ -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> ~w;~n", [Start, End, Cat]), + gen_category(Fd, Rest, NextCat, CP, CP, All, Acc) + end + end; +gen_category(Fd, [{CP, {_, _, _, NewCat}}|Rest]=Cont, Cat, Start, End, All, Acc) -> + case array:get(End+1, All) of + undefined -> + if Start =:= End -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]), + gen_category(Fd, Rest, NewCat, CP, CP, All, Acc); + true -> + case Cat of + letter -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> subcat_letter(CP);~n", + [Start, End]), + gen_category(Fd, Rest, NewCat, CP, CP, All, + lists:reverse(lists:seq(Start, End)) ++ Acc); + _ -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w -> ~w;~n", + [Start, End, Cat]), + gen_category(Fd, Rest, NewCat, CP, CP, All, Acc) + end + end; + _ -> %% We can make ranges larger by setting already assigned category + gen_category(Fd, Cont, Cat, Start, End+1, All, Acc) + end; +gen_category(Fd, [], Cat, Start, End, All, Acc) -> + case Start =:= End of + true -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]); + false -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w -> ~w;~n", [Start, End, Cat]) + end, + io:put_chars(Fd, "lookup_category(Cp) -> cn.\n\n"), + gen_letter(Fd, lists:reverse(Acc), All), ok. +letter_cat(lm, _) -> + false; +letter_cat(_, lm) -> + false; +letter_cat(L1, L2) -> + is_letter(L1) andalso (L2 =:= letter orelse is_letter(L2)). + +is_letter(LC) -> + lists:member(LC, [lu,ll,lt,lo,lm]). + +gen_letter(Fd, Letters, All) -> + gen_letter(Fd, Letters, All, []). +gen_letter(Fd, [CP|Rest], All, Acc) -> + case array:get(CP, All) of + undefined -> + gen_letter(Fd, Rest, All, Acc); + #cp{cat=Cat0, cs=Cs} -> + case {category(Cat0), case_table(CP,case_data(CP, Cs))} of + {Sub,Sub} -> + gen_letter(Fd, Rest, All, Acc); + {lm,_} -> + gen_letter(Fd, Rest, All, Acc); + {Cat, _Dbg} -> + case is_letter(Cat) of + true -> + gen_letter(Fd, Rest, All, [{CP, Cat}|Acc]); + false -> + gen_letter(Fd, Rest, All, Acc) + end + end + end; +gen_letter(Fd, [], _, Acc) -> + [{Start, Cat}|SCletters] = lists:reverse(Acc), + subcat_letter(Fd, SCletters, Start, Start, Cat), + io:put_chars(Fd, + "subcat_letter(CP) ->\n" + " case case_table(CP) of\n" + " {CP, CP} -> lo; %{letter,other};\n" + " {CP, _} -> lu; %{letter,uppercase};\n" + " {_, CP} -> ll; %{letter,lowercase};\n" + " {_, _, CP, _} -> lt; %{letter,titlecase};\n" + " {CP, _, _, _} -> lu; %{letter,uppercase};\n" + " {_,CP,_,_} -> ll %{letter,lowercase}\n" + " end.\n\n"). + +subcat_letter(Fd, [{CP, Cat}|R], Start, End, Cat) when End+1 =:= CP -> + subcat_letter(Fd, R, Start, CP, Cat); +subcat_letter(Fd, Rest, Start, Start, Cat) -> + io:format(Fd, "subcat_letter(~w) -> ~w;\n",[Start,Cat]), + case Rest of + [] -> ok; + [{CP, NewCat}|R] -> subcat_letter(Fd, R, CP, CP, NewCat) + end; +subcat_letter(Fd, Rest, Start, End, Cat) -> + io:format(Fd, "subcat_letter(CP) when ~w =< CP, CP =< ~w -> ~w;\n",[Start,End,Cat]), + case Rest of + [] -> ok; + [{CP, NewCat}|R] -> subcat_letter(Fd, R, CP, CP, NewCat) + end. + +case_table(CP, CaseData) -> + case CaseData of + {CP, CP} -> lo; + {CP, _} -> lu; + {_, CP} -> ll; + {_, _, CP, _} -> lt; + {CP, _, _, _} -> lu; + {_,CP,_,_} -> ll; + default -> lo + end. + decompose([], _Data) -> []; decompose([CP|CPs], Data) when is_integer(CP) -> case array:get(CP,Data) of @@ -883,35 +1159,41 @@ decompose_compat([{_,CP}|CPs], Data) -> decompose_compat([CP|CPs], Data). +gen_width_table(Fd, WideChars) -> + MergedWCs = merge_ranges(WideChars), + Write = fun(Range) -> io:format(Fd, "is_wide_cp~s true;~n", [gen_single_clause(Range)]) end, + [Write(Range) || Range <- MergedWCs], + io:format(Fd, "is_wide_cp(_) -> false.~n", []). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% gen_clause({R0, undefined}) -> io_lib:format("([~w=CP|R1]=R0) ->", [R0]); gen_clause({R0, R1}) -> - io_lib:format("([CP|R1]=R0) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1]=R0) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_clause2({R0, undefined}) -> io_lib:format("([~w=CP|R1], R0, Acc) ->", [R0]); gen_clause2({R0, R1}) -> - io_lib:format("([CP|R1], R0, Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1], R0, Acc) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_case_clause({R0, undefined}) -> io_lib:format("[~w=CP|R1] ->", [R0]); gen_case_clause({R0, R1}) -> - io_lib:format("[CP|R1] when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("[CP|R1] when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_single_clause({R0, undefined}) -> io_lib:format("(~w) ->", [R0]); gen_single_clause({R0, R1}) -> - io_lib:format("(CP) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("(CP) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). merge_ranges(List) -> merge_ranges(List, true). merge_ranges(List, Opt) -> - Res0 = merge_ranges_1(lists:sort(List), []), + Res0 = merge_ranges_1(lists:usort(List), []), case Opt of split -> split_ranges(Res0,[]); % One clause per CP diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 065e180804..e47185869d 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -928,7 +928,6 @@ resulting regexp is surrounded by \\_< and \\_>." "delay_trap" "delete_element" "display" - "display_nl" "display_string" "dist_get_stat" "dist_ctrl_get_data" diff --git a/lib/wx/.gitignore b/lib/wx/.gitignore index 09564b499e..aa95f9ad4c 100644 --- a/lib/wx/.gitignore +++ b/lib/wx/.gitignore @@ -3,6 +3,7 @@ wx_test_case_info doc/html/* api_gen/wx_doxygen.log* api_gen/wx_*_api.dump +priv/WebView2Loader.dll %% Don't delete links to man src when git clean -dfX %% api_gen/gl_man? diff --git a/lib/wx/Makefile b/lib/wx/Makefile index f471824a14..b77a7808c5 100644 --- a/lib/wx/Makefile +++ b/lib/wx/Makefile @@ -21,7 +21,7 @@ include ./vsn.mk include ./config.mk -ifdef TERTIARY_BOOTSTRAP +ifdef BOOTSTRAP SUBDIRS = src else # Normal build SUBDIRS = src @@ -29,7 +29,7 @@ else # Normal build SUBDIRS += c_src endif SUBDIRS += examples doc/src -endif #TERTIARY_BOOTSTRAP +endif #BOOTSTRAP CLEANDIRS = $(SUBDIRS) api_gen diff --git a/lib/wx/src/Makefile b/lib/wx/src/Makefile index ce14c0b6df..b055dfed4a 100644 --- a/lib/wx/src/Makefile +++ b/lib/wx/src/Makefile @@ -19,7 +19,7 @@ # include ../vsn.mk -ifdef TERTIARY_BOOTSTRAP +ifdef BOOTSTRAP VSN = $(WX_VSN) include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk diff --git a/make/autoconf/otp.m4 b/make/autoconf/otp.m4 index 7b2e81328d..2f06fc92cc 100644 --- a/make/autoconf/otp.m4 +++ b/make/autoconf/otp.m4 @@ -1480,27 +1480,30 @@ AC_DEFUN(ETHR_CHK_GCC_ATOMIC_OPS, AC_DEFUN(ETHR_CHK_INTERLOCKED, [ ilckd="$1" - AC_MSG_CHECKING([for ${ilckd}()]) case "$2" in "1") ilckd_call="${ilckd}(var);";; "2") ilckd_call="${ilckd}(var, ($3) 0);";; "3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";; "4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";; esac - have_interlocked_op=no - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #define WIN32_LEAN_AND_MEAN - #include <windows.h> - #include <intrin.h> - ]], [[ - volatile $3 *var; - volatile $3 arr[2]; - - $ilckd_call - return 0; - ]])],[have_interlocked_op=yes],[]) - test $have_interlocked_op = yes && $4 - AC_MSG_RESULT([$have_interlocked_op]) + AC_CACHE_CHECK([for ${ilckd}()],ethr_cv_have_$1, + [ethr_cv_have_$1=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #include <intrin.h> + ]], [[ + volatile $3 *var; + volatile $3 arr[2]; + + $ilckd_call + return 0; + ]])],[ethr_cv_have_$1=yes],[])]) + if [[ "${ethr_cv_have_$1}" = "yes" ]]; then + $4 + else + m4_default([$5], [:]) + fi ]) dnl ---------------------------------------------------------------------- @@ -1687,13 +1690,15 @@ AS_CASE( ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()])) ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()])) ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()])) - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()]) + ethr_have_native_atomics=yes]) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()]) + ethr_have_native_atomics=yes]) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()]) + ethr_have_native_atomics=yes]) ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])) ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()])) ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])) @@ -1703,13 +1708,15 @@ AS_CASE( ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])) ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])) ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])) - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]) + ethr_have_native_atomics=yes]) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()]) + ethr_have_native_atomics=yes]) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], + [AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()]) + ethr_have_native_atomics=yes]) ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()])) fi if test "$ethr_have_native_atomics" = "yes"; then diff --git a/make/autoconf/win32.config.cache.static b/make/autoconf/win32.config.cache.static index 790ea9ab00..8aa795cdc0 100755 --- a/make/autoconf/win32.config.cache.static +++ b/make/autoconf/win32.config.cache.static @@ -11,9 +11,12 @@ # loading this file, other *unset* `ac_cv_foo' will be assigned the # following values. +# ac_cv_build=${ac_cv_build=local-x86-pc-windows} ac_cv_c_bigendian=${ac_cv_c_bigendian=no} ac_cv_c_compiler_gnu=${ac_cv_c_compiler_gnu=no} ac_cv_c_const=${ac_cv_c_const=yes} +# ac_cv_c_double_middle_endian=${ac_cv_c_double_middle_endian=no} +ac_cv_c_undeclared_builtin_options=${ac_cv_c_undeclared_builtin_options='none needed'} ac_cv_cxx_compiler_gnu=${ac_cv_cxx_compiler_gnu=no} ac_cv_decl_h_errno=${ac_cv_decl_h_errno=no} ac_cv_decl_inaddr_loopback=${ac_cv_decl_inaddr_loopback=no} @@ -21,10 +24,16 @@ ac_cv_decl_inaddr_loopback_rpc=${ac_cv_decl_inaddr_loopback_rpc=no} ac_cv_decl_inaddr_loopback_winsock2=${ac_cv_decl_inaddr_loopback_winsock2=yes} ac_cv_decl_so_bsdcompat=${ac_cv_decl_so_bsdcompat=no} ac_cv_decl_sys_errlist=${ac_cv_decl_sys_errlist=no} +ac_cv_env_AR_set=set +ac_cv_env_AR_value=ar.sh +ac_cv_env_CCC_set= +ac_cv_env_CCC_value= ac_cv_env_CC_set=set ac_cv_env_CC_value=cc.sh ac_cv_env_CFLAGS_set= ac_cv_env_CFLAGS_value= +# ac_cv_env_CFLAG_RUNTIME_LIBRARY_PATH_set= +# ac_cv_env_CFLAG_RUNTIME_LIBRARY_PATH_value= ac_cv_env_CPPFLAGS_set= ac_cv_env_CPPFLAGS_value= ac_cv_env_CPP_set= @@ -33,10 +42,70 @@ ac_cv_env_CXXFLAGS_set= ac_cv_env_CXXFLAGS_value= ac_cv_env_CXX_set=set ac_cv_env_CXX_value=cc.sh -ac_cv_env_LDFLAGS_set= -ac_cv_env_LDFLAGS_value= +# ac_cv_env_DED_LDFLAGS_set= +# ac_cv_env_DED_LDFLAGS_value= +# ac_cv_env_DED_LD_FLAG_RUNTIME_LIBRARY_PATH_set= +# ac_cv_env_DED_LD_FLAG_RUNTIME_LIBRARY_PATH_value= +# ac_cv_env_DED_LD_set= +# ac_cv_env_DED_LD_value= +# ac_cv_env_ERL_TOP_set=set +# ac_cv_env_ERL_TOP_value=$ERL_TOP +# ac_cv_env_GETCONF_set= +# ac_cv_env_GETCONF_value= +# ac_cv_env_LDFLAGS_set= +# ac_cv_env_LDFLAGS_value= +# ac_cv_env_LD_set= +# ac_cv_env_LD_value= +# ac_cv_env_LFS_CFLAGS_set= +# ac_cv_env_LFS_CFLAGS_value= +# ac_cv_env_LFS_LDFLAGS_set= +# ac_cv_env_LFS_LDFLAGS_value= +# ac_cv_env_LFS_LIBS_set= +# ac_cv_env_LFS_LIBS_value= +# ac_cv_env_LIBS_set= +# ac_cv_env_LIBS_value= +# ac_cv_env_RANLIB_set=set +# ac_cv_env_RANLIB_value=true +# ac_cv_env_STATIC_CFLAGS_set= +# ac_cv_env_STATIC_CFLAGS_value= +# ac_cv_env_YACC_set= +# ac_cv_env_YACC_value= +# ac_cv_env_YFLAGS_set= +# ac_cv_env_YFLAGS_value= ac_cv_env_build_alias_set=set ac_cv_env_build_alias_value=local-x86-pc-windows +# ac_cv_env_erl_xcomp_after_morecore_hook_set= +# ac_cv_env_erl_xcomp_after_morecore_hook_value= +# ac_cv_env_erl_xcomp_bigendian_set= +# ac_cv_env_erl_xcomp_bigendian_value= +# ac_cv_env_erl_xcomp_clock_gettime_cpu_time_set= +# ac_cv_env_erl_xcomp_clock_gettime_cpu_time_value= +# ac_cv_env_erl_xcomp_dlsym_brk_wrappers_set= +# ac_cv_env_erl_xcomp_dlsym_brk_wrappers_value= +# ac_cv_env_erl_xcomp_double_middle_endian_set= +# ac_cv_env_erl_xcomp_double_middle_endian_value= +# ac_cv_env_erl_xcomp_getaddrinfo_set= +# ac_cv_env_erl_xcomp_getaddrinfo_value= +# ac_cv_env_erl_xcomp_gethrvtime_procfs_ioctl_set= +# ac_cv_env_erl_xcomp_gethrvtime_procfs_ioctl_value= +# ac_cv_env_erl_xcomp_isysroot_set= +# ac_cv_env_erl_xcomp_isysroot_value= +# ac_cv_env_erl_xcomp_kqueue_set= +# ac_cv_env_erl_xcomp_kqueue_value= +# ac_cv_env_erl_xcomp_linux_nptl_set= +# ac_cv_env_erl_xcomp_linux_nptl_value= +# ac_cv_env_erl_xcomp_linux_usable_sigaltstack_set= +# ac_cv_env_erl_xcomp_linux_usable_sigaltstack_value= +# ac_cv_env_erl_xcomp_linux_usable_sigusrx_set= +# ac_cv_env_erl_xcomp_linux_usable_sigusrx_value= +# ac_cv_env_erl_xcomp_poll_set= +# ac_cv_env_erl_xcomp_poll_value= +# ac_cv_env_erl_xcomp_putenv_copy_set= +# ac_cv_env_erl_xcomp_putenv_copy_value= +# ac_cv_env_erl_xcomp_reliable_fpe_set= +# ac_cv_env_erl_xcomp_reliable_fpe_value= +# ac_cv_env_erl_xcomp_sysroot_set= +# ac_cv_env_erl_xcomp_sysroot_value= ac_cv_env_host_alias_set=set ac_cv_env_host_alias_value=local-x86-pc-windows ac_cv_env_target_alias_set=set @@ -47,91 +116,110 @@ ac_cv_func___sbrk=${ac_cv_func___sbrk=no} ac_cv_func__brk=${ac_cv_func__brk=no} ac_cv_func__doprnt=${ac_cv_func__doprnt=no} ac_cv_func__sbrk=${ac_cv_func__sbrk=no} -ac_cv_func_accept=${ac_cv_func_accept=no} -ac_cv_func_alloca_works=${ac_cv_func_alloca_works=yes} ac_cv_func_brk=${ac_cv_func_brk=no} -ac_cv_func_clock_gettime=${ac_cv_func_clock_gettime=no} +ac_cv_func_clock_get_attributes=${ac_cv_func_clock_get_attributes=no} +ac_cv_func_clock_getres=${ac_cv_func_clock_getres=no} +ac_cv_func_closefrom=${ac_cv_func_closefrom=no} ac_cv_func_connect=${ac_cv_func_connect=no} ac_cv_func_decl_fread=${ac_cv_func_decl_fread=no} ac_cv_func_dlopen=${ac_cv_func_dlopen=no} -ac_cv_func_dup2=${ac_cv_func_dup2=yes} +ac_cv_func_dlvsym=${ac_cv_func_dlvsym=no} +ac_cv_func_endprotoent=${ac_cv_func_endprotoent=no} +ac_cv_func_fdatasync=${ac_cv_func_fdatasync=no} ac_cv_func_finite=${ac_cv_func_finite=no} ac_cv_func_flockfile=${ac_cv_func_flockfile=no} -ac_cv_func_fork=${ac_cv_func_fork=no} -ac_cv_func_fork_works=${ac_cv_func_fork_works=no} ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no} ac_cv_func_fstat=${ac_cv_func_fstat=yes} -ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no} -ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no} -ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no} ac_cv_func_gethostbyname2=${ac_cv_func_gethostbyname2=no} +ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no} ac_cv_func_gethostbyname_r=${ac_cv_func_gethostbyname_r=no} ac_cv_func_gethostname=${ac_cv_func_gethostname=no} ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no} +ac_cv_func_getifaddrs=${ac_cv_func_getifaddrs=no} ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no} ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no} -ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no} +ac_cv_func_getprotoent=${ac_cv_func_getprotoent=no} +ac_cv_func_getrusage=${ac_cv_func_getrusage=no} ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no} ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no} ac_cv_func_ieee_handler=${ac_cv_func_ieee_handler=no} -ac_cv_func_inet_ntoa=${ac_cv_func_inet_ntoa=no} +ac_cv_func_if_freenameindex=${ac_cv_func_if_freenameindex=no} +ac_cv_func_if_indextoname=${ac_cv_func_if_indextoname=no} +ac_cv_func_if_nameindex=${ac_cv_func_if_nameindex=no} +ac_cv_func_if_nametoindex=${ac_cv_func_if_nametoindex=no} ac_cv_func_isinf=${ac_cv_func_isinf=no} ac_cv_func_isnan=${ac_cv_func_isnan=no} ac_cv_func_localtime_r=${ac_cv_func_localtime_r=no} +ac_cv_func_log2=${ac_cv_func_log2=yes} +ac_cv_func_madvise=${ac_cv_func_madvise=no} ac_cv_func_mallopt=${ac_cv_func_mallopt=no} -ac_cv_func_memchr=${ac_cv_func_memchr=yes} -ac_cv_func_memcmp_working=${ac_cv_func_memcmp_working=yes} -ac_cv_func_memcpy=${ac_cv_func_memcpy=yes} -ac_cv_func_memmove=${ac_cv_func_memmove=yes} -ac_cv_func_memset=${ac_cv_func_memset=yes} -ac_cv_func_mmap_fixed_mapped=${ac_cv_func_mmap_fixed_mapped=no} +ac_cv_func_memcpy=${ac_cv_func_memcpy=no} +ac_cv_func_memmove=${ac_cv_func_memmove=no} +ac_cv_func_mlockall=${ac_cv_func_mlockall=no} +ac_cv_func_mmap=${ac_cv_func_mmap=no} +ac_cv_func_mprotect=${ac_cv_func_mprotect=no} ac_cv_func_mremap=${ac_cv_func_mremap=no} ac_cv_func_nl_langinfo=${ac_cv_func_nl_langinfo=no} ac_cv_func_openpty=${ac_cv_func_openpty=no} +ac_cv_func_poll=${ac_cv_func_poll=no} ac_cv_func_posix2time=${ac_cv_func_posix2time=no} +ac_cv_func_posix_fadvise=${ac_cv_func_posix_fadvise=no} +ac_cv_func_posix_madvise=${ac_cv_func_posix_madvise=no} +ac_cv_func_posix_memalign=${ac_cv_func_posix_memalign=no} +ac_cv_func_ppoll=${ac_cv_func_ppoll=no} ac_cv_func_pread=${ac_cv_func_pread=no} ac_cv_func_pwrite=${ac_cv_func_pwrite=no} ac_cv_func_res_gethostbyname=${ac_cv_func_res_gethostbyname=no} ac_cv_func_sbrk=${ac_cv_func_sbrk=no} -ac_cv_func_select=${ac_cv_func_select=no} ac_cv_func_setlocale=${ac_cv_func_setlocale=yes} +ac_cv_func_setns=${ac_cv_func_setns=no} +ac_cv_func_setprotoent=${ac_cv_func_setprotoent=no} ac_cv_func_setsid=${ac_cv_func_setsid=no} -ac_cv_func_socket=${ac_cv_func_socket=no} -ac_cv_func_strchr=${ac_cv_func_strchr=yes} ac_cv_func_strerror=${ac_cv_func_strerror=yes} ac_cv_func_strerror_r=${ac_cv_func_strerror_r=no} +ac_cv_func_strftime=${ac_cv_func_strftime=yes} ac_cv_func_strlcat=${ac_cv_func_strlcat=no} ac_cv_func_strlcpy=${ac_cv_func_strlcpy=no} ac_cv_func_strncasecmp=${ac_cv_func_strncasecmp=no} -ac_cv_func_strrchr=${ac_cv_func_strrchr=yes} -ac_cv_func_strstr=${ac_cv_func_strstr=yes} -ac_cv_func_uname=${ac_cv_func_uname=no} -ac_cv_func_vfork=${ac_cv_func_vfork=no} -ac_cv_func_vfork_works=${ac_cv_func_vfork_works=no} -ac_cv_func_vprintf=${ac_cv_func_vprintf=yes} +ac_cv_func_time2posix=${ac_cv_func_time2posix=no} +ac_cv_func_vprintf=${ac_cv_func_vprintf=no} +ac_cv_func_vsyslog=${ac_cv_func_vsyslog=no} ac_cv_func_writev=${ac_cv_func_writev=no} -ac_cv_header_arpa_inet_h=${ac_cv_header_arpa_inet_h=no} +ac_cv_have_decl_IN6ADDR_ANY_INIT=${ac_cv_have_decl_IN6ADDR_ANY_INIT=no} +ac_cv_have_decl_IN6ADDR_LOOPBACK_INIT=${ac_cv_have_decl_IN6ADDR_LOOPBACK_INIT=no} +ac_cv_have_decl_IPV6_V6ONLY=${ac_cv_have_decl_IPV6_V6ONLY=no} +ac_cv_have_decl_posix2time=${ac_cv_have_decl_posix2time=no} +ac_cv_have_decl_time2posix=${ac_cv_have_decl_time2posix=no} ac_cv_header_arpa_nameser_h=${ac_cv_header_arpa_nameser_h=no} ac_cv_header_dirent_dirent_h=${ac_cv_header_dirent_dirent_h=no} ac_cv_header_dirent_ndir_h=${ac_cv_header_dirent_ndir_h=no} ac_cv_header_dirent_sys_dir_h=${ac_cv_header_dirent_sys_dir_h=no} ac_cv_header_dirent_sys_ndir_h=${ac_cv_header_dirent_sys_ndir_h=no} ac_cv_header_dlfcn_h=${ac_cv_header_dlfcn_h=no} +ac_cv_header_elf_h=${ac_cv_header_elf_h=no} ac_cv_header_fcntl_h=${ac_cv_header_fcntl_h=yes} -ac_cv_header_gl_gl_h=${ac_cv_header_gl_gl_h=yes} ac_cv_header_ieeefp_h=${ac_cv_header_ieeefp_h=no} -ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=no} +ac_cv_header_ifaddrs_h=${ac_cv_header_ifaddrs_h=no} +ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=yes} ac_cv_header_langinfo_h=${ac_cv_header_langinfo_h=no} +ac_cv_header_libdlpi_h=${ac_cv_header_libdlpi_h=no} +ac_cv_header_libutil_h=${ac_cv_header_libutil_h=no} ac_cv_header_limits_h=${ac_cv_header_limits_h=yes} +ac_cv_header_linux_errqueue_h=${ac_cv_header_linux_errqueue_h=no} +ac_cv_header_linux_falloc_h=${ac_cv_header_linux_falloc_h=no} +ac_cv_header_linux_types_h=${ac_cv_header_linux_types_h=no} ac_cv_header_malloc_h=${ac_cv_header_malloc_h=yes} -ac_cv_header_memory_h=${ac_cv_header_memory_h=yes} ac_cv_header_net_errno_h=${ac_cv_header_net_errno_h=no} -ac_cv_header_netdb_h=${ac_cv_header_netdb_h=no} -ac_cv_header_netinet_in_h=${ac_cv_header_netinet_in_h=no} +ac_cv_header_net_if_dl_h=${ac_cv_header_net_if_dl_h=no} +ac_cv_header_netinet_sctp_h=${ac_cv_header_netinet_sctp_h=no} +ac_cv_header_netpacket_packet_h=${ac_cv_header_netpacket_packet_h=no} +ac_cv_header_poll_h=${ac_cv_header_poll_h=no} ac_cv_header_pty_h=${ac_cv_header_pty_h=no} -ac_cv_header_stdc=${ac_cv_header_stdc=yes} -ac_cv_header_stddef_h=${ac_cv_header_stddef_h=yes} -ac_cv_header_stdint_h=${ac_cv_header_stdint_h=no} +ac_cv_header_sched_h=${ac_cv_header_sched_h=no} +ac_cv_header_sdkddkver_h=${ac_cv_header_sdkddkver_h=yes} +ac_cv_header_setns_h=${ac_cv_header_setns_h=no} +ac_cv_header_stdint_h=${ac_cv_header_stdint_h=yes} +ac_cv_header_stdio_h=${ac_cv_header_stdio_h=yes} ac_cv_header_stdlib_h=${ac_cv_header_stdlib_h=yes} ac_cv_header_string_h=${ac_cv_header_string_h=yes} ac_cv_header_strings_h=${ac_cv_header_strings_h=no} @@ -139,8 +227,8 @@ ac_cv_header_sys_devpoll_h=${ac_cv_header_sys_devpoll_h=no} ac_cv_header_sys_epoll_h=${ac_cv_header_sys_epoll_h=no} ac_cv_header_sys_event_h=${ac_cv_header_sys_event_h=no} ac_cv_header_sys_ioctl_h=${ac_cv_header_sys_ioctl_h=no} -ac_cv_header_sys_param_h=${ac_cv_header_sys_param_h=no} -ac_cv_header_sys_select_h=${ac_cv_header_sys_select_h=no} +ac_cv_header_sys_mman_h=${ac_cv_header_sys_mman_h=no} +ac_cv_header_sys_resource_h=${ac_cv_header_sys_resource_h=no} ac_cv_header_sys_socket_h=${ac_cv_header_sys_socket_h=no} ac_cv_header_sys_socketio_h=${ac_cv_header_sys_socketio_h=no} ac_cv_header_sys_sockio_h=${ac_cv_header_sys_sockio_h=no} @@ -148,55 +236,67 @@ ac_cv_header_sys_stat_h=${ac_cv_header_sys_stat_h=yes} ac_cv_header_sys_stropts_h=${ac_cv_header_sys_stropts_h=no} ac_cv_header_sys_sysctl_h=${ac_cv_header_sys_sysctl_h=no} ac_cv_header_sys_time_h=${ac_cv_header_sys_time_h=no} +ac_cv_header_sys_timerfd_h=${ac_cv_header_sys_timerfd_h=no} ac_cv_header_sys_types_h=${ac_cv_header_sys_types_h=yes} ac_cv_header_sys_uio_h=${ac_cv_header_sys_uio_h=no} +ac_cv_header_sys_un_h=${ac_cv_header_sys_un_h=no} ac_cv_header_sys_wait_h=${ac_cv_header_sys_wait_h=no} ac_cv_header_syslog_h=${ac_cv_header_syslog_h=no} -ac_cv_header_time=${ac_cv_header_time=no} ac_cv_header_unistd_h=${ac_cv_header_unistd_h=no} ac_cv_header_util_h=${ac_cv_header_util_h=no} ac_cv_header_utmp_h=${ac_cv_header_utmp_h=no} ac_cv_header_valgrind_valgrind_h=${ac_cv_header_valgrind_valgrind_h=no} -ac_cv_header_vfork_h=${ac_cv_header_vfork_h=no} +ac_cv_header_windows_h=${ac_cv_header_windows_h=yes} +ac_cv_header_winsock2_h=${ac_cv_header_winsock2_h=yes} +ac_cv_header_ws2tcpip_h=${ac_cv_header_ws2tcpip_h=yes} +ac_cv_host=${ac_cv_host=local-x86-pc-windows} ac_cv_lib_dl_dlopen=${ac_cv_lib_dl_dlopen=no} +ac_cv_lib_dl_dlvsym=${ac_cv_lib_dl_dlvsym=no} +ac_cv_lib_dlpi_dlpi_open=${ac_cv_lib_dlpi_dlpi_open=no} ac_cv_lib_inet_main=${ac_cv_lib_inet_main=no} ac_cv_lib_kstat_kstat_open=${ac_cv_lib_kstat_kstat_open=no} +ac_cv_lib_kvm_kvm_open=${ac_cv_lib_kvm_kvm_open=no} ac_cv_lib_m_sin=${ac_cv_lib_m_sin=no} -ac_cv_lib_nsl_gethostbyname=${ac_cv_lib_nsl_gethostbyname=no} ac_cv_lib_nsl_main=${ac_cv_lib_nsl_main=no} -ac_cv_lib_resolv_res_gethostbyname=${ac_cv_lib_resolv_res_gethostbyname=no} ac_cv_lib_rt_clock_gettime=${ac_cv_lib_rt_clock_gettime=no} -ac_cv_lib_socket_getpeername=${ac_cv_lib_socket_getpeername=no} ac_cv_lib_socket_main=${ac_cv_lib_socket_main=yes} -ac_cv_lib_socket_socket=${ac_cv_lib_socket_socket=no} ac_cv_lib_util_openpty=${ac_cv_lib_util_openpty=no} -ac_cv_lib_ws2_32_main=${ac_cv_lib_ws2_32_main=yes} -ac_cv_member_struct_ErlDrvEntry_stop_select=${ac_cv_member_struct_ErlDrvEntry_stop_select=no} +ac_cv_member_struct_ifreq_ifr_enaddr=${ac_cv_member_struct_ifreq_ifr_enaddr=no} +ac_cv_member_struct_ifreq_ifr_hwaddr=${ac_cv_member_struct_ifreq_ifr_hwaddr=no} +ac_cv_member_struct_ifreq_ifr_ifindex=${ac_cv_member_struct_ifreq_ifr_ifindex=no} +ac_cv_member_struct_ifreq_ifr_index=${ac_cv_member_struct_ifreq_ifr_index=no} +ac_cv_member_struct_ifreq_ifr_map=${ac_cv_member_struct_ifreq_ifr_map=no} +ac_cv_member_struct_sockaddr_dl_sdl_len=${ac_cv_member_struct_sockaddr_dl_sdl_len=no} +ac_cv_member_struct_sockaddr_un_sun_path=${ac_cv_member_struct_sockaddr_un_sun_path=no} ac_cv_objext=${ac_cv_objext=o} -ac_cv_path_MKDIR=${ac_cv_path_MKDIR=/bin/mkdir} -ac_cv_path_PERL=${ac_cv_path_PERL=/usr/bin/perl} -ac_cv_path_RM=${ac_cv_path_RM=/bin/rm} -ac_cv_path_install=${ac_cv_path_install='/usr/bin/install -c'} +# ac_cv_path_CP=${ac_cv_path_CP=/bin/cp} +# ac_cv_path_EGREP=${ac_cv_path_EGREP='/usr/bin/grep -E'} +# ac_cv_path_GREP=${ac_cv_path_GREP=/usr/bin/grep} +# ac_cv_path_MKDIR=${ac_cv_path_MKDIR=/bin/mkdir} +# ac_cv_path_PERL=${ac_cv_path_PERL=/usr/bin/perl} +# ac_cv_path_install=${ac_cv_path_install='/usr/bin/install -c'} ac_cv_prog_AR=${ac_cv_prog_AR=ar.sh} ac_cv_prog_CC=${ac_cv_prog_CC=cc.sh} ac_cv_prog_CPP=${ac_cv_prog_CPP='cc.sh -E'} -ac_cv_prog_CXX=${ac_cv_prog_CXX=cc.sh} -ac_cv_prog_DED_LD=${ac_cv_prog_DED_LD=ld.sh} -ac_cv_prog_ac_ct_DED_LD=${ac_cv_prog_ac_ct_DED_LD=ld.sh} -ac_cv_prog_M4=${ac_cv_prog_M4=m4} -ac_cv_prog_PERL=${ac_cv_prog_PERL=perl} +ac_cv_prog_GETCONF=${ac_cv_prog_GETCONF=getconf} +ac_cv_prog_JAVAC=${ac_cv_prog_JAVAC=javac.sh} ac_cv_prog_RANLIB=${ac_cv_prog_RANLIB=true} -ac_cv_prog_LD=${ac_cv_prog_LD=ld.sh} -ac_cv_prog_ac_ct_LD=${ac_cv_prog_ac_ct_LD=ld.sh} +ac_cv_prog_cc_c11=${ac_cv_prog_cc_c11=no} +ac_cv_prog_cc_c89=${ac_cv_prog_cc_c89=no} +ac_cv_prog_cc_c99=${ac_cv_prog_cc_c99=no} ac_cv_prog_cc_g=${ac_cv_prog_cc_g=yes} -ac_cv_prog_cc_stdc=${ac_cv_prog_cc_stdc=} -ac_cv_prog_cxx_g=${ac_cv_prog_cxx_g=no} -ac_cv_prog_egrep=${ac_cv_prog_egrep='grep -E'} -ac_cv_prog_emu_cc=${ac_cv_prog_emu_cc=emu_cc.sh} -ac_cv_prog_make_make_set=${ac_cv_prog_make_make_set=yes} -ac_cv_prog_mkdir_p=${ac_cv_prog_mkdir_p='/usr/bin/install -c -d'} +ac_cv_prog_cxx_11=${ac_cv_prog_cxx_11=no} +ac_cv_prog_cxx_g=${ac_cv_prog_cxx_g=yes} +ac_cv_prog_cxx_stdcxx=${ac_cv_prog_cxx_stdcxx=} +# ac_cv_prog_emu_cc=${ac_cv_prog_emu_cc=$ERL_TOP/erts/etc/win32/wsl_tools/vc/emu_cc.sh} +# ac_cv_prog_javac_ver_1_6=${ac_cv_prog_javac_ver_1_6=no} +# ac_cv_prog_mkdir_p=${ac_cv_prog_mkdir_p='/usr/bin/install -c -d'} +ac_cv_search_fdatasync=${ac_cv_search_fdatasync=no} ac_cv_search_opendir=${ac_cv_search_opendir=no} ac_cv_search_strerror=${ac_cv_search_strerror='none required'} +ac_cv_sizeof__Float16=${ac_cv_sizeof__Float16=0} +ac_cv_sizeof___int128_t=${ac_cv_sizeof___int128_t=0} +ac_cv_sizeof___int64=${ac_cv_sizeof___int64=8} ac_cv_sizeof_char=${ac_cv_sizeof_char=1} ac_cv_sizeof_int=${ac_cv_sizeof_int=4} ac_cv_sizeof_long=${ac_cv_sizeof_long=4} @@ -204,26 +304,56 @@ ac_cv_sizeof_long_long=${ac_cv_sizeof_long_long=8} ac_cv_sizeof_off_t=${ac_cv_sizeof_off_t=4} ac_cv_sizeof_short=${ac_cv_sizeof_short=2} ac_cv_sizeof_size_t=${ac_cv_sizeof_size_t=4} +ac_cv_sizeof_suseconds_t=${ac_cv_sizeof_suseconds_t=0} +ac_cv_sizeof_time_t=${ac_cv_sizeof_time_t=8} ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=4} -ac_cv_struct_exception=${ac_cv_struct_exception=no} ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no} ac_cv_struct_tm=${ac_cv_struct_tm=time.h} +# ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=yes} ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no} -ac_cv_type_char=${ac_cv_type_char=yes} -ac_cv_type_int=${ac_cv_type_int=yes} -ac_cv_type_long=${ac_cv_type_long=yes} -ac_cv_type_long_long=${ac_cv_type_long_long=yes} +ac_cv_target=${ac_cv_target=local-x86-pc-windows} ac_cv_type_off_t=${ac_cv_type_off_t=yes} ac_cv_type_pid_t=${ac_cv_type_pid_t=no} -ac_cv_type_short=${ac_cv_type_short=yes} -ac_cv_type_signal=${ac_cv_type_signal=void} ac_cv_type_size_t=${ac_cv_type_size_t=yes} -ac_cv_type_uid_t=${ac_cv_type_uid_t=no} -ac_cv_type_void_p=${ac_cv_type_void_p=yes} -ac_cv_working_alloca_h=${ac_cv_working_alloca_h=no} +erl_cv_clock_gettime_monotonic_default_resolution=${erl_cv_clock_gettime_monotonic_default_resolution=no} +erl_cv_clock_gettime_monotonic_high_resolution=${erl_cv_clock_gettime_monotonic_high_resolution=no} +erl_cv_clock_gettime_monotonic_raw=${erl_cv_clock_gettime_monotonic_raw=no} +erl_cv_clock_gettime_monotonic_try_find_pthread_compatible=${erl_cv_clock_gettime_monotonic_try_find_pthread_compatible=no} +erl_cv_clock_gettime_wall_default_resolution=${erl_cv_clock_gettime_wall_default_resolution=no} +erl_cv_mach_clock_get_time_monotonic=${erl_cv_mach_clock_get_time_monotonic=no} +erl_cv_mach_clock_get_time_wall=${erl_cv_mach_clock_get_time_wall=no} erts_cv___after_morecore_hook_can_track_malloc=${erts_cv___after_morecore_hook_can_track_malloc=no} erts_cv_fwrite_unlocked=${erts_cv_fwrite_unlocked=no} erts_cv_have__end_symbol=${erts_cv_have__end_symbol=no} erts_cv_have_end_symbol=${erts_cv_have_end_symbol=no} +erts_cv_have_in6addr_any=${erts_cv_have_in6addr_any=no} +erts_cv_have_in6addr_loopback=${erts_cv_have_in6addr_loopback=no} erts_cv_putc_unlocked=${erts_cv_putc_unlocked=no} erts_cv_windows_h_includes_winsock2_h=${erts_cv_windows_h_includes_winsock2_h=no} +ethr_cv_have__InterlockedAnd64=${ethr_cv_have__InterlockedAnd64=no} +ethr_cv_have__InterlockedAnd=${ethr_cv_have__InterlockedAnd=yes} +ethr_cv_have__InterlockedCompareExchange128=${ethr_cv_have__InterlockedCompareExchange128=no} +ethr_cv_have__InterlockedCompareExchange64=${ethr_cv_have__InterlockedCompareExchange64=yes} +# ethr_cv_have__InterlockedCompareExchange64_acq=${ethr_cv_have__InterlockedCompareExchange64_acq=no} +# ethr_cv_have__InterlockedCompareExchange64_rel=${ethr_cv_have__InterlockedCompareExchange64_rel=no} +ethr_cv_have__InterlockedCompareExchange=${ethr_cv_have__InterlockedCompareExchange=yes} +# ethr_cv_have__InterlockedCompareExchange_acq=${ethr_cv_have__InterlockedCompareExchange_acq=no} +# ethr_cv_have__InterlockedCompareExchange_rel=${ethr_cv_have__InterlockedCompareExchange_rel=no} +ethr_cv_have__InterlockedDecrement64=${ethr_cv_have__InterlockedDecrement64=no} +# ethr_cv_have__InterlockedDecrement64_rel=${ethr_cv_have__InterlockedDecrement64_rel=no} +ethr_cv_have__InterlockedDecrement=${ethr_cv_have__InterlockedDecrement=yes} +# ethr_cv_have__InterlockedDecrement_rel=${ethr_cv_have__InterlockedDecrement_rel=no} +ethr_cv_have__InterlockedExchange64=${ethr_cv_have__InterlockedExchange64=no} +ethr_cv_have__InterlockedExchange=${ethr_cv_have__InterlockedExchange=yes} +ethr_cv_have__InterlockedExchangeAdd64=${ethr_cv_have__InterlockedExchangeAdd64=no} +# ethr_cv_have__InterlockedExchangeAdd64_acq=${ethr_cv_have__InterlockedExchangeAdd64_acq=no} +ethr_cv_have__InterlockedExchangeAdd=${ethr_cv_have__InterlockedExchangeAdd=yes} +# ethr_cv_have__InterlockedExchangeAdd_acq=${ethr_cv_have__InterlockedExchangeAdd_acq=no} +ethr_cv_have__InterlockedIncrement64=${ethr_cv_have__InterlockedIncrement64=no} +# ethr_cv_have__InterlockedIncrement64_acq=${ethr_cv_have__InterlockedIncrement64_acq=no} +ethr_cv_have__InterlockedIncrement=${ethr_cv_have__InterlockedIncrement=yes} +# ethr_cv_have__InterlockedIncrement_acq=${ethr_cv_have__InterlockedIncrement_acq=no} +ethr_cv_have__InterlockedOr64=${ethr_cv_have__InterlockedOr64=no} +ethr_cv_have__InterlockedOr=${ethr_cv_have__InterlockedOr=yes} +i_cv_fallocate_works=${i_cv_fallocate_works=no} +i_cv_posix_fallocate_works=${i_cv_posix_fallocate_works=no} diff --git a/make/autoconf/win64.config.cache.static b/make/autoconf/win64.config.cache.static index 73146cbb86..a151aa3452 100755 --- a/make/autoconf/win64.config.cache.static +++ b/make/autoconf/win64.config.cache.static @@ -11,9 +11,12 @@ # loading this file, other *unset* `ac_cv_foo' will be assigned the # following values. +# ac_cv_build=${ac_cv_build=local-x86_64-pc-windows} ac_cv_c_bigendian=${ac_cv_c_bigendian=no} ac_cv_c_compiler_gnu=${ac_cv_c_compiler_gnu=no} ac_cv_c_const=${ac_cv_c_const=yes} +# ac_cv_c_double_middle_endian=${ac_cv_c_double_middle_endian=no} +ac_cv_c_undeclared_builtin_options=${ac_cv_c_undeclared_builtin_options='none needed'} ac_cv_cxx_compiler_gnu=${ac_cv_cxx_compiler_gnu=no} ac_cv_decl_h_errno=${ac_cv_decl_h_errno=no} ac_cv_decl_inaddr_loopback=${ac_cv_decl_inaddr_loopback=no} @@ -29,6 +32,8 @@ ac_cv_env_CC_set=set ac_cv_env_CC_value=cc.sh ac_cv_env_CFLAGS_set= ac_cv_env_CFLAGS_value= +# ac_cv_env_CFLAG_RUNTIME_LIBRARY_PATH_set= +# ac_cv_env_CFLAG_RUNTIME_LIBRARY_PATH_value= ac_cv_env_CPPFLAGS_set= ac_cv_env_CPPFLAGS_value= ac_cv_env_CPP_set= @@ -37,10 +42,70 @@ ac_cv_env_CXXFLAGS_set= ac_cv_env_CXXFLAGS_value= ac_cv_env_CXX_set=set ac_cv_env_CXX_value=cc.sh -ac_cv_env_LDFLAGS_set= -ac_cv_env_LDFLAGS_value= +# ac_cv_env_DED_LDFLAGS_set= +# ac_cv_env_DED_LDFLAGS_value= +# ac_cv_env_DED_LD_FLAG_RUNTIME_LIBRARY_PATH_set= +# ac_cv_env_DED_LD_FLAG_RUNTIME_LIBRARY_PATH_value= +# ac_cv_env_DED_LD_set= +# ac_cv_env_DED_LD_value= +# ac_cv_env_ERL_TOP_set=set +# ac_cv_env_ERL_TOP_value=$ERL_TOP +# ac_cv_env_GETCONF_set= +# ac_cv_env_GETCONF_value= +# ac_cv_env_LDFLAGS_set= +# ac_cv_env_LDFLAGS_value= +# ac_cv_env_LD_set= +# ac_cv_env_LD_value= +# ac_cv_env_LFS_CFLAGS_set= +# ac_cv_env_LFS_CFLAGS_value= +# ac_cv_env_LFS_LDFLAGS_set= +# ac_cv_env_LFS_LDFLAGS_value= +# ac_cv_env_LFS_LIBS_set= +# ac_cv_env_LFS_LIBS_value= +# ac_cv_env_LIBS_set= +# ac_cv_env_LIBS_value= +# ac_cv_env_RANLIB_set=set +# ac_cv_env_RANLIB_value=true +# ac_cv_env_STATIC_CFLAGS_set= +# ac_cv_env_STATIC_CFLAGS_value= +# ac_cv_env_YACC_set= +# ac_cv_env_YACC_value= +# ac_cv_env_YFLAGS_set= +# ac_cv_env_YFLAGS_value= ac_cv_env_build_alias_set=set ac_cv_env_build_alias_value=local-x86_64-pc-windows +# ac_cv_env_erl_xcomp_after_morecore_hook_set= +# ac_cv_env_erl_xcomp_after_morecore_hook_value= +# ac_cv_env_erl_xcomp_bigendian_set= +# ac_cv_env_erl_xcomp_bigendian_value= +# ac_cv_env_erl_xcomp_clock_gettime_cpu_time_set= +# ac_cv_env_erl_xcomp_clock_gettime_cpu_time_value= +# ac_cv_env_erl_xcomp_dlsym_brk_wrappers_set= +# ac_cv_env_erl_xcomp_dlsym_brk_wrappers_value= +# ac_cv_env_erl_xcomp_double_middle_endian_set= +# ac_cv_env_erl_xcomp_double_middle_endian_value= +# ac_cv_env_erl_xcomp_getaddrinfo_set= +# ac_cv_env_erl_xcomp_getaddrinfo_value= +# ac_cv_env_erl_xcomp_gethrvtime_procfs_ioctl_set= +# ac_cv_env_erl_xcomp_gethrvtime_procfs_ioctl_value= +# ac_cv_env_erl_xcomp_isysroot_set= +# ac_cv_env_erl_xcomp_isysroot_value= +# ac_cv_env_erl_xcomp_kqueue_set= +# ac_cv_env_erl_xcomp_kqueue_value= +# ac_cv_env_erl_xcomp_linux_nptl_set= +# ac_cv_env_erl_xcomp_linux_nptl_value= +# ac_cv_env_erl_xcomp_linux_usable_sigaltstack_set= +# ac_cv_env_erl_xcomp_linux_usable_sigaltstack_value= +# ac_cv_env_erl_xcomp_linux_usable_sigusrx_set= +# ac_cv_env_erl_xcomp_linux_usable_sigusrx_value= +# ac_cv_env_erl_xcomp_poll_set= +# ac_cv_env_erl_xcomp_poll_value= +# ac_cv_env_erl_xcomp_putenv_copy_set= +# ac_cv_env_erl_xcomp_putenv_copy_value= +# ac_cv_env_erl_xcomp_reliable_fpe_set= +# ac_cv_env_erl_xcomp_reliable_fpe_value= +# ac_cv_env_erl_xcomp_sysroot_set= +# ac_cv_env_erl_xcomp_sysroot_value= ac_cv_env_host_alias_set=set ac_cv_env_host_alias_value=local-x86_64-pc-windows ac_cv_env_target_alias_set=set @@ -51,134 +116,110 @@ ac_cv_func___sbrk=${ac_cv_func___sbrk=no} ac_cv_func__brk=${ac_cv_func__brk=no} ac_cv_func__doprnt=${ac_cv_func__doprnt=no} ac_cv_func__sbrk=${ac_cv_func__sbrk=no} -ac_cv_func_accept=${ac_cv_func_accept=no} -ac_cv_func_alloca_works=${ac_cv_func_alloca_works=yes} ac_cv_func_brk=${ac_cv_func_brk=no} -ac_cv_func_clock_gettime=${ac_cv_func_clock_gettime=no} +ac_cv_func_clock_get_attributes=${ac_cv_func_clock_get_attributes=no} +ac_cv_func_clock_getres=${ac_cv_func_clock_getres=no} +ac_cv_func_closefrom=${ac_cv_func_closefrom=no} ac_cv_func_connect=${ac_cv_func_connect=no} ac_cv_func_decl_fread=${ac_cv_func_decl_fread=yes} ac_cv_func_dlopen=${ac_cv_func_dlopen=no} -ac_cv_func_dup2=${ac_cv_func_dup2=yes} +ac_cv_func_dlvsym=${ac_cv_func_dlvsym=no} +ac_cv_func_endprotoent=${ac_cv_func_endprotoent=no} ac_cv_func_fdatasync=${ac_cv_func_fdatasync=no} ac_cv_func_finite=${ac_cv_func_finite=no} ac_cv_func_flockfile=${ac_cv_func_flockfile=no} -ac_cv_func_fork=${ac_cv_func_fork=no} -ac_cv_func_fork_works=${ac_cv_func_fork_works=no} ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no} ac_cv_func_fstat=${ac_cv_func_fstat=yes} -ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no} -ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no} -ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=yes} ac_cv_func_gethostbyname2=${ac_cv_func_gethostbyname2=no} +ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=yes} ac_cv_func_gethostbyname_r=${ac_cv_func_gethostbyname_r=no} ac_cv_func_gethostname=${ac_cv_func_gethostname=no} ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no} ac_cv_func_getifaddrs=${ac_cv_func_getifaddrs=no} ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no} ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no} -ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no} +ac_cv_func_getprotoent=${ac_cv_func_getprotoent=no} +ac_cv_func_getrusage=${ac_cv_func_getrusage=no} ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no} ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no} ac_cv_func_ieee_handler=${ac_cv_func_ieee_handler=no} -ac_cv_func_inet_ntoa=${ac_cv_func_inet_ntoa=no} -ac_cv_func_inet_pton=${ac_cv_func_inet_pton=yes} +ac_cv_func_if_freenameindex=${ac_cv_func_if_freenameindex=no} +ac_cv_func_if_indextoname=${ac_cv_func_if_indextoname=no} +ac_cv_func_if_nameindex=${ac_cv_func_if_nameindex=no} +ac_cv_func_if_nametoindex=${ac_cv_func_if_nametoindex=no} ac_cv_func_isinf=${ac_cv_func_isinf=no} ac_cv_func_isnan=${ac_cv_func_isnan=no} ac_cv_func_localtime_r=${ac_cv_func_localtime_r=no} +ac_cv_func_log2=${ac_cv_func_log2=no} +ac_cv_func_madvise=${ac_cv_func_madvise=no} ac_cv_func_mallopt=${ac_cv_func_mallopt=no} -ac_cv_func_memchr=${ac_cv_func_memchr=yes} -ac_cv_func_memcmp_working=${ac_cv_func_memcmp_working=yes} ac_cv_func_memcpy=${ac_cv_func_memcpy=no} -ac_cv_func_memmove=${ac_cv_func_memmove=yes} -ac_cv_func_memset=${ac_cv_func_memset=yes} +ac_cv_func_memmove=${ac_cv_func_memmove=no} +ac_cv_func_mlockall=${ac_cv_func_mlockall=no} ac_cv_func_mmap=${ac_cv_func_mmap=no} -ac_cv_func_mmap_fixed_mapped=${ac_cv_func_mmap_fixed_mapped=no} +ac_cv_func_mprotect=${ac_cv_func_mprotect=no} ac_cv_func_mremap=${ac_cv_func_mremap=no} ac_cv_func_nl_langinfo=${ac_cv_func_nl_langinfo=no} ac_cv_func_openpty=${ac_cv_func_openpty=no} ac_cv_func_poll=${ac_cv_func_poll=no} ac_cv_func_posix2time=${ac_cv_func_posix2time=no} ac_cv_func_posix_fadvise=${ac_cv_func_posix_fadvise=no} +ac_cv_func_posix_madvise=${ac_cv_func_posix_madvise=no} +ac_cv_func_posix_memalign=${ac_cv_func_posix_memalign=no} +ac_cv_func_ppoll=${ac_cv_func_ppoll=no} ac_cv_func_pread=${ac_cv_func_pread=no} ac_cv_func_pwrite=${ac_cv_func_pwrite=no} ac_cv_func_res_gethostbyname=${ac_cv_func_res_gethostbyname=no} ac_cv_func_sbrk=${ac_cv_func_sbrk=no} -ac_cv_func_sctp_bindx=${ac_cv_func_sctp_bindx=no} -ac_cv_func_sctp_peeloff=${ac_cv_func_sctp_peeloff=no} -ac_cv_func_select=${ac_cv_func_select=no} ac_cv_func_setlocale=${ac_cv_func_setlocale=yes} +ac_cv_func_setns=${ac_cv_func_setns=no} +ac_cv_func_setprotoent=${ac_cv_func_setprotoent=no} ac_cv_func_setsid=${ac_cv_func_setsid=no} -ac_cv_func_setvbuf_reversed=${ac_cv_func_setvbuf_reversed=yes} -ac_cv_func_socket=${ac_cv_func_socket=no} -ac_cv_func_strchr=${ac_cv_func_strchr=yes} ac_cv_func_strerror=${ac_cv_func_strerror=yes} ac_cv_func_strerror_r=${ac_cv_func_strerror_r=no} +ac_cv_func_strftime=${ac_cv_func_strftime=yes} ac_cv_func_strlcat=${ac_cv_func_strlcat=no} ac_cv_func_strlcpy=${ac_cv_func_strlcpy=no} ac_cv_func_strncasecmp=${ac_cv_func_strncasecmp=no} -ac_cv_func_strrchr=${ac_cv_func_strrchr=yes} -ac_cv_func_strstr=${ac_cv_func_strstr=yes} -ac_cv_func_uname=${ac_cv_func_uname=no} -ac_cv_func_vfork=${ac_cv_func_vfork=no} -ac_cv_func_vfork_works=${ac_cv_func_vfork_works=no} -ac_cv_func_vprintf=${ac_cv_func_vprintf=yes} +ac_cv_func_time2posix=${ac_cv_func_time2posix=no} +ac_cv_func_vprintf=${ac_cv_func_vprintf=no} +ac_cv_func_vsyslog=${ac_cv_func_vsyslog=no} ac_cv_func_writev=${ac_cv_func_writev=no} -ac_cv_have_decl_SCTPS_BOUND=${ac_cv_have_decl_SCTPS_BOUND=no} -ac_cv_have_decl_SCTPS_COOKIE_ECHOED=${ac_cv_have_decl_SCTPS_COOKIE_ECHOED=no} -ac_cv_have_decl_SCTPS_COOKIE_WAIT=${ac_cv_have_decl_SCTPS_COOKIE_WAIT=no} -ac_cv_have_decl_SCTPS_ESTABLISHED=${ac_cv_have_decl_SCTPS_ESTABLISHED=no} -ac_cv_have_decl_SCTPS_IDLE=${ac_cv_have_decl_SCTPS_IDLE=no} -ac_cv_have_decl_SCTPS_LISTEN=${ac_cv_have_decl_SCTPS_LISTEN=no} -ac_cv_have_decl_SCTPS_SHUTDOWN_ACK_SENT=${ac_cv_have_decl_SCTPS_SHUTDOWN_ACK_SENT=no} -ac_cv_have_decl_SCTPS_SHUTDOWN_PENDING=${ac_cv_have_decl_SCTPS_SHUTDOWN_PENDING=no} -ac_cv_have_decl_SCTPS_SHUTDOWN_RECEIVED=${ac_cv_have_decl_SCTPS_SHUTDOWN_RECEIVED=no} -ac_cv_have_decl_SCTPS_SHUTDOWN_SENT=${ac_cv_have_decl_SCTPS_SHUTDOWN_SENT=no} -ac_cv_have_decl_SCTP_ABORT=${ac_cv_have_decl_SCTP_ABORT=no} -ac_cv_have_decl_SCTP_ADDR_CONFIRMED=${ac_cv_have_decl_SCTP_ADDR_CONFIRMED=no} -ac_cv_have_decl_SCTP_ADDR_OVER=${ac_cv_have_decl_SCTP_ADDR_OVER=no} -ac_cv_have_decl_SCTP_BOUND=${ac_cv_have_decl_SCTP_BOUND=no} -ac_cv_have_decl_SCTP_CLOSED=${ac_cv_have_decl_SCTP_CLOSED=no} -ac_cv_have_decl_SCTP_COOKIE_ECHOED=${ac_cv_have_decl_SCTP_COOKIE_ECHOED=no} -ac_cv_have_decl_SCTP_COOKIE_WAIT=${ac_cv_have_decl_SCTP_COOKIE_WAIT=no} -ac_cv_have_decl_SCTP_DELAYED_ACK_TIME=${ac_cv_have_decl_SCTP_DELAYED_ACK_TIME=no} -ac_cv_have_decl_SCTP_EMPTY=${ac_cv_have_decl_SCTP_EMPTY=no} -ac_cv_have_decl_SCTP_EOF=${ac_cv_have_decl_SCTP_EOF=no} -ac_cv_have_decl_SCTP_ESTABLISHED=${ac_cv_have_decl_SCTP_ESTABLISHED=no} -ac_cv_have_decl_SCTP_LISTEN=${ac_cv_have_decl_SCTP_LISTEN=no} -ac_cv_have_decl_SCTP_SENDALL=${ac_cv_have_decl_SCTP_SENDALL=no} -ac_cv_have_decl_SCTP_SHUTDOWN_ACK_SENT=${ac_cv_have_decl_SCTP_SHUTDOWN_ACK_SENT=no} -ac_cv_have_decl_SCTP_SHUTDOWN_PENDING=${ac_cv_have_decl_SCTP_SHUTDOWN_PENDING=no} -ac_cv_have_decl_SCTP_SHUTDOWN_RECEIVED=${ac_cv_have_decl_SCTP_SHUTDOWN_RECEIVED=no} -ac_cv_have_decl_SCTP_SHUTDOWN_SENT=${ac_cv_have_decl_SCTP_SHUTDOWN_SENT=no} -ac_cv_have_decl_SCTP_UNORDERED=${ac_cv_have_decl_SCTP_UNORDERED=no} +ac_cv_have_decl_IN6ADDR_ANY_INIT=${ac_cv_have_decl_IN6ADDR_ANY_INIT=no} +ac_cv_have_decl_IN6ADDR_LOOPBACK_INIT=${ac_cv_have_decl_IN6ADDR_LOOPBACK_INIT=no} +ac_cv_have_decl_IPV6_V6ONLY=${ac_cv_have_decl_IPV6_V6ONLY=no} ac_cv_have_decl_posix2time=${ac_cv_have_decl_posix2time=no} -ac_cv_header_arpa_inet_h=${ac_cv_header_arpa_inet_h=no} +ac_cv_have_decl_time2posix=${ac_cv_have_decl_time2posix=no} ac_cv_header_arpa_nameser_h=${ac_cv_header_arpa_nameser_h=no} ac_cv_header_dirent_dirent_h=${ac_cv_header_dirent_dirent_h=no} ac_cv_header_dirent_ndir_h=${ac_cv_header_dirent_ndir_h=no} ac_cv_header_dirent_sys_dir_h=${ac_cv_header_dirent_sys_dir_h=no} ac_cv_header_dirent_sys_ndir_h=${ac_cv_header_dirent_sys_ndir_h=no} ac_cv_header_dlfcn_h=${ac_cv_header_dlfcn_h=no} +ac_cv_header_elf_h=${ac_cv_header_elf_h=no} ac_cv_header_fcntl_h=${ac_cv_header_fcntl_h=yes} -ac_cv_header_gl_gl_h=${ac_cv_header_gl_gl_h=yes} ac_cv_header_ieeefp_h=${ac_cv_header_ieeefp_h=no} ac_cv_header_ifaddrs_h=${ac_cv_header_ifaddrs_h=no} -ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=no} +ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=yes} ac_cv_header_langinfo_h=${ac_cv_header_langinfo_h=no} +ac_cv_header_libdlpi_h=${ac_cv_header_libdlpi_h=no} +ac_cv_header_libutil_h=${ac_cv_header_libutil_h=no} ac_cv_header_limits_h=${ac_cv_header_limits_h=yes} -ac_cv_header_mach_o_dyld_h=${ac_cv_header_mach_o_dyld_h=no} +ac_cv_header_linux_errqueue_h=${ac_cv_header_linux_errqueue_h=no} +ac_cv_header_linux_falloc_h=${ac_cv_header_linux_falloc_h=no} +ac_cv_header_linux_types_h=${ac_cv_header_linux_types_h=no} ac_cv_header_malloc_h=${ac_cv_header_malloc_h=yes} -ac_cv_header_memory_h=${ac_cv_header_memory_h=yes} ac_cv_header_net_errno_h=${ac_cv_header_net_errno_h=no} ac_cv_header_net_if_dl_h=${ac_cv_header_net_if_dl_h=no} -ac_cv_header_netdb_h=${ac_cv_header_netdb_h=no} -ac_cv_header_netinet_in_h=${ac_cv_header_netinet_in_h=no} +ac_cv_header_netinet_sctp_h=${ac_cv_header_netinet_sctp_h=no} ac_cv_header_netpacket_packet_h=${ac_cv_header_netpacket_packet_h=no} ac_cv_header_poll_h=${ac_cv_header_poll_h=no} ac_cv_header_pty_h=${ac_cv_header_pty_h=no} -ac_cv_header_stdc=${ac_cv_header_stdc=yes} -ac_cv_header_stddef_h=${ac_cv_header_stddef_h=yes} +ac_cv_header_sched_h=${ac_cv_header_sched_h=no} +ac_cv_header_sdkddkver_h=${ac_cv_header_sdkddkver_h=yes} +ac_cv_header_setns_h=${ac_cv_header_setns_h=no} ac_cv_header_stdint_h=${ac_cv_header_stdint_h=yes} +ac_cv_header_stdio_h=${ac_cv_header_stdio_h=yes} ac_cv_header_stdlib_h=${ac_cv_header_stdlib_h=yes} ac_cv_header_string_h=${ac_cv_header_string_h=yes} ac_cv_header_strings_h=${ac_cv_header_strings_h=no} @@ -186,9 +227,8 @@ ac_cv_header_sys_devpoll_h=${ac_cv_header_sys_devpoll_h=no} ac_cv_header_sys_epoll_h=${ac_cv_header_sys_epoll_h=no} ac_cv_header_sys_event_h=${ac_cv_header_sys_event_h=no} ac_cv_header_sys_ioctl_h=${ac_cv_header_sys_ioctl_h=no} -ac_cv_header_sys_param_h=${ac_cv_header_sys_param_h=no} +ac_cv_header_sys_mman_h=${ac_cv_header_sys_mman_h=no} ac_cv_header_sys_resource_h=${ac_cv_header_sys_resource_h=no} -ac_cv_header_sys_select_h=${ac_cv_header_sys_select_h=no} ac_cv_header_sys_socket_h=${ac_cv_header_sys_socket_h=no} ac_cv_header_sys_socketio_h=${ac_cv_header_sys_socketio_h=no} ac_cv_header_sys_sockio_h=${ac_cv_header_sys_sockio_h=no} @@ -196,54 +236,65 @@ ac_cv_header_sys_stat_h=${ac_cv_header_sys_stat_h=yes} ac_cv_header_sys_stropts_h=${ac_cv_header_sys_stropts_h=no} ac_cv_header_sys_sysctl_h=${ac_cv_header_sys_sysctl_h=no} ac_cv_header_sys_time_h=${ac_cv_header_sys_time_h=no} +ac_cv_header_sys_timerfd_h=${ac_cv_header_sys_timerfd_h=no} ac_cv_header_sys_types_h=${ac_cv_header_sys_types_h=yes} ac_cv_header_sys_uio_h=${ac_cv_header_sys_uio_h=no} +ac_cv_header_sys_un_h=${ac_cv_header_sys_un_h=no} ac_cv_header_sys_wait_h=${ac_cv_header_sys_wait_h=no} ac_cv_header_syslog_h=${ac_cv_header_syslog_h=no} -ac_cv_header_time=${ac_cv_header_time=no} ac_cv_header_unistd_h=${ac_cv_header_unistd_h=no} ac_cv_header_util_h=${ac_cv_header_util_h=no} ac_cv_header_utmp_h=${ac_cv_header_utmp_h=no} ac_cv_header_valgrind_valgrind_h=${ac_cv_header_valgrind_valgrind_h=no} -ac_cv_header_vfork_h=${ac_cv_header_vfork_h=no} ac_cv_header_windows_h=${ac_cv_header_windows_h=yes} ac_cv_header_winsock2_h=${ac_cv_header_winsock2_h=yes} ac_cv_header_ws2tcpip_h=${ac_cv_header_ws2tcpip_h=yes} +ac_cv_host=${ac_cv_host=local-x86_64-pc-windows} ac_cv_lib_dl_dlopen=${ac_cv_lib_dl_dlopen=no} +ac_cv_lib_dl_dlvsym=${ac_cv_lib_dl_dlvsym=no} +ac_cv_lib_dlpi_dlpi_open=${ac_cv_lib_dlpi_dlpi_open=no} ac_cv_lib_inet_main=${ac_cv_lib_inet_main=no} ac_cv_lib_kstat_kstat_open=${ac_cv_lib_kstat_kstat_open=no} +ac_cv_lib_kvm_kvm_open=${ac_cv_lib_kvm_kvm_open=no} ac_cv_lib_m_sin=${ac_cv_lib_m_sin=no} -ac_cv_lib_nsl_gethostbyname=${ac_cv_lib_nsl_gethostbyname=no} -ac_cv_lib_nsl_main=${ac_cv_lib_nsl_main=no} -ac_cv_lib_resolv_res_gethostbyname=${ac_cv_lib_resolv_res_gethostbyname=no} ac_cv_lib_rt_clock_gettime=${ac_cv_lib_rt_clock_gettime=no} -ac_cv_lib_socket_getpeername=${ac_cv_lib_socket_getpeername=no} ac_cv_lib_socket_main=${ac_cv_lib_socket_main=yes} -ac_cv_lib_socket_socket=${ac_cv_lib_socket_socket=no} ac_cv_lib_util_openpty=${ac_cv_lib_util_openpty=no} -ac_cv_lib_ws2_32_main=${ac_cv_lib_ws2_32_main=yes} -ac_cv_member_struct_ErlDrvEntry_stop_select=${ac_cv_member_struct_ErlDrvEntry_stop_select=no} -ac_cv_member_struct_sctp_paddrparams_spp_flags=${ac_cv_member_struct_sctp_paddrparams_spp_flags=no} -ac_cv_member_struct_sctp_paddrparams_spp_pathmtu=${ac_cv_member_struct_sctp_paddrparams_spp_pathmtu=no} -ac_cv_member_struct_sctp_paddrparams_spp_sackdelay=${ac_cv_member_struct_sctp_paddrparams_spp_sackdelay=no} -ac_cv_member_struct_sctp_remote_error_sre_data=${ac_cv_member_struct_sctp_remote_error_sre_data=no} -ac_cv_member_struct_sctp_send_failed_ssf_data=${ac_cv_member_struct_sctp_send_failed_ssf_data=no} +ac_cv_member_struct_ifreq_ifr_enaddr=${ac_cv_member_struct_ifreq_ifr_enaddr=no} +ac_cv_member_struct_ifreq_ifr_hwaddr=${ac_cv_member_struct_ifreq_ifr_hwaddr=no} +ac_cv_member_struct_ifreq_ifr_ifindex=${ac_cv_member_struct_ifreq_ifr_ifindex=no} +ac_cv_member_struct_ifreq_ifr_index=${ac_cv_member_struct_ifreq_ifr_index=no} +ac_cv_member_struct_ifreq_ifr_map=${ac_cv_member_struct_ifreq_ifr_map=no} +ac_cv_member_struct_sockaddr_dl_sdl_len=${ac_cv_member_struct_sockaddr_dl_sdl_len=no} +ac_cv_member_struct_sockaddr_un_sun_path=${ac_cv_member_struct_sockaddr_un_sun_path=no} ac_cv_objext=${ac_cv_objext=o} -ac_cv_path_MKDIR=${ac_cv_path_MKDIR=/bin/mkdir} -ac_cv_path_RM=${ac_cv_path_RM=/bin/rm} +# ac_cv_path_CP=${ac_cv_path_CP=/bin/cp} +# ac_cv_path_EGREP=${ac_cv_path_EGREP='/usr/bin/grep -E'} +# ac_cv_path_GREP=${ac_cv_path_GREP=/usr/bin/grep} +# ac_cv_path_MKDIR=${ac_cv_path_MKDIR=/bin/mkdir} +# ac_cv_path_PERL=${ac_cv_path_PERL=/usr/bin/perl} +# ac_cv_path_install=${ac_cv_path_install='/usr/bin/install -c'} ac_cv_prog_AR=${ac_cv_prog_AR=ar.sh} ac_cv_prog_CC=${ac_cv_prog_CC=cc.sh} ac_cv_prog_CPP=${ac_cv_prog_CPP='cc.sh -E'} -ac_cv_prog_CXX=${ac_cv_prog_CXX=cc.sh} -ac_cv_prog_DED_LD=${ac_cv_prog_DED_LD=ld.sh} +ac_cv_prog_GETCONF=${ac_cv_prog_GETCONF=getconf} ac_cv_prog_JAVAC=${ac_cv_prog_JAVAC=javac.sh} -ac_cv_prog_M4=${ac_cv_prog_M4=m4} ac_cv_prog_RANLIB=${ac_cv_prog_RANLIB=true} -ac_cv_prog_cc_c89=${ac_cv_prog_cc_c89=} +ac_cv_prog_cc_c11=${ac_cv_prog_cc_c11=no} +ac_cv_prog_cc_c89=${ac_cv_prog_cc_c89=no} +ac_cv_prog_cc_c99=${ac_cv_prog_cc_c99=no} ac_cv_prog_cc_g=${ac_cv_prog_cc_g=yes} +ac_cv_prog_cxx_11=${ac_cv_prog_cxx_11=no} +ac_cv_prog_cxx_g=${ac_cv_prog_cxx_g=yes} +ac_cv_prog_cxx_stdcxx=${ac_cv_prog_cxx_stdcxx=} +# ac_cv_prog_emu_cc=${ac_cv_prog_emu_cc=$ERL_TOP/git/otp/erts/etc/win32/wsl_tools/vc/emu_cc.sh} +# ac_cv_prog_javac_ver_1_6=${ac_cv_prog_javac_ver_1_6=no} +# ac_cv_prog_mkdir_p=${ac_cv_prog_mkdir_p='/usr/bin/install -c -d'} ac_cv_search_fdatasync=${ac_cv_search_fdatasync=no} ac_cv_search_opendir=${ac_cv_search_opendir=no} ac_cv_search_strerror=${ac_cv_search_strerror='none required'} +ac_cv_sizeof__Float16=${ac_cv_sizeof__Float16=0} +ac_cv_sizeof___int128_t=${ac_cv_sizeof___int128_t=0} ac_cv_sizeof___int64=${ac_cv_sizeof___int64=8} ac_cv_sizeof_char=${ac_cv_sizeof_char=1} ac_cv_sizeof_int=${ac_cv_sizeof_int=4} @@ -252,19 +303,56 @@ ac_cv_sizeof_long_long=${ac_cv_sizeof_long_long=8} ac_cv_sizeof_off_t=${ac_cv_sizeof_off_t=4} ac_cv_sizeof_short=${ac_cv_sizeof_short=2} ac_cv_sizeof_size_t=${ac_cv_sizeof_size_t=8} +ac_cv_sizeof_suseconds_t=${ac_cv_sizeof_suseconds_t=0} +ac_cv_sizeof_time_t=${ac_cv_sizeof_time_t=8} ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=8} -ac_cv_struct_exception=${ac_cv_struct_exception=no} ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no} ac_cv_struct_tm=${ac_cv_struct_tm=time.h} +# ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=yes} +ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no} +ac_cv_target=${ac_cv_target=local-x86_64-pc-windows} ac_cv_type_off_t=${ac_cv_type_off_t=yes} ac_cv_type_pid_t=${ac_cv_type_pid_t=no} -ac_cv_type_signal=${ac_cv_type_signal=void} ac_cv_type_size_t=${ac_cv_type_size_t=yes} -ac_cv_type_uid_t=${ac_cv_type_uid_t=no} -ac_cv_working_alloca_h=${ac_cv_working_alloca_h=no} +erl_cv_clock_gettime_monotonic_default_resolution=${erl_cv_clock_gettime_monotonic_default_resolution=no} +erl_cv_clock_gettime_monotonic_high_resolution=${erl_cv_clock_gettime_monotonic_high_resolution=no} +erl_cv_clock_gettime_monotonic_raw=${erl_cv_clock_gettime_monotonic_raw=no} +erl_cv_clock_gettime_monotonic_try_find_pthread_compatible=${erl_cv_clock_gettime_monotonic_try_find_pthread_compatible=no} +erl_cv_clock_gettime_wall_default_resolution=${erl_cv_clock_gettime_wall_default_resolution=no} +erl_cv_mach_clock_get_time_monotonic=${erl_cv_mach_clock_get_time_monotonic=no} +erl_cv_mach_clock_get_time_wall=${erl_cv_mach_clock_get_time_wall=no} erts_cv___after_morecore_hook_can_track_malloc=${erts_cv___after_morecore_hook_can_track_malloc=no} erts_cv_fwrite_unlocked=${erts_cv_fwrite_unlocked=no} erts_cv_have__end_symbol=${erts_cv_have__end_symbol=no} erts_cv_have_end_symbol=${erts_cv_have_end_symbol=no} +erts_cv_have_in6addr_any=${erts_cv_have_in6addr_any=no} +erts_cv_have_in6addr_loopback=${erts_cv_have_in6addr_loopback=no} erts_cv_putc_unlocked=${erts_cv_putc_unlocked=no} erts_cv_windows_h_includes_winsock2_h=${erts_cv_windows_h_includes_winsock2_h=no} +ethr_cv_have__InterlockedAnd64=${ethr_cv_have__InterlockedAnd64=yes} +ethr_cv_have__InterlockedAnd=${ethr_cv_have__InterlockedAnd=yes} +ethr_cv_have__InterlockedCompareExchange128=${ethr_cv_have__InterlockedCompareExchange128=yes} +ethr_cv_have__InterlockedCompareExchange64=${ethr_cv_have__InterlockedCompareExchange64=yes} +# ethr_cv_have__InterlockedCompareExchange64_acq=${ethr_cv_have__InterlockedCompareExchange64_acq=no} +# ethr_cv_have__InterlockedCompareExchange64_rel=${ethr_cv_have__InterlockedCompareExchange64_rel=no} +ethr_cv_have__InterlockedCompareExchange=${ethr_cv_have__InterlockedCompareExchange=yes} +# ethr_cv_have__InterlockedCompareExchange_acq=${ethr_cv_have__InterlockedCompareExchange_acq=no} +# ethr_cv_have__InterlockedCompareExchange_rel=${ethr_cv_have__InterlockedCompareExchange_rel=no} +ethr_cv_have__InterlockedDecrement64=${ethr_cv_have__InterlockedDecrement64=yes} +# ethr_cv_have__InterlockedDecrement64_rel=${ethr_cv_have__InterlockedDecrement64_rel=no} +ethr_cv_have__InterlockedDecrement=${ethr_cv_have__InterlockedDecrement=yes} +# ethr_cv_have__InterlockedDecrement_rel=${ethr_cv_have__InterlockedDecrement_rel=no} +ethr_cv_have__InterlockedExchange64=${ethr_cv_have__InterlockedExchange64=yes} +ethr_cv_have__InterlockedExchange=${ethr_cv_have__InterlockedExchange=yes} +ethr_cv_have__InterlockedExchangeAdd64=${ethr_cv_have__InterlockedExchangeAdd64=yes} +# ethr_cv_have__InterlockedExchangeAdd64_acq=${ethr_cv_have__InterlockedExchangeAdd64_acq=no} +ethr_cv_have__InterlockedExchangeAdd=${ethr_cv_have__InterlockedExchangeAdd=yes} +# ethr_cv_have__InterlockedExchangeAdd_acq=${ethr_cv_have__InterlockedExchangeAdd_acq=no} +ethr_cv_have__InterlockedIncrement64=${ethr_cv_have__InterlockedIncrement64=yes} +# ethr_cv_have__InterlockedIncrement64_acq=${ethr_cv_have__InterlockedIncrement64_acq=no} +ethr_cv_have__InterlockedIncrement=${ethr_cv_have__InterlockedIncrement=yes} +# ethr_cv_have__InterlockedIncrement_acq=${ethr_cv_have__InterlockedIncrement_acq=no} +ethr_cv_have__InterlockedOr64=${ethr_cv_have__InterlockedOr64=yes} +ethr_cv_have__InterlockedOr=${ethr_cv_have__InterlockedOr=yes} +i_cv_fallocate_works=${i_cv_fallocate_works=no} +i_cv_posix_fallocate_works=${i_cv_posix_fallocate_works=no} diff --git a/make/configure b/make/configure index 169ae15fe6..8c048b9325 100755 --- a/make/configure +++ b/make/configure @@ -773,7 +773,6 @@ enable_option_checking enable_bootstrap_only enable_parallel_configure enable_dirty_schedulers -enable_plain_emulator with_termcap enable_kernel_poll enable_sctp @@ -1466,9 +1465,6 @@ Optional Features: disable parallel execution of configure scripts --enable-dirty-schedulers enable dirty scheduler support - --enable-plain-emulator enable threaded non-smp emulator - --disable-plain-emulator - disable threaded non-smp emulator --enable-kernel-poll enable kernel poll support --disable-kernel-poll disable kernel poll support --enable-sctp enable sctp support (default) to on demand load the @@ -5557,13 +5553,6 @@ then : fi -# Check whether --enable-plain-emulator was given. -if test ${enable_plain_emulator+y} -then : - enableval=$enable_plain_emulator; -fi - - # Check whether --with-termcap was given. if test ${with_termcap+y} diff --git a/make/configure.ac b/make/configure.ac index e0d4103b6e..1276caecbc 100644 --- a/make/configure.ac +++ b/make/configure.ac @@ -212,10 +212,6 @@ AS_HELP_STRING([--disable-parallel-configure], [disable parallel execution of co AC_ARG_ENABLE(dirty-schedulers, AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support])) -AC_ARG_ENABLE(plain-emulator, -AS_HELP_STRING([--enable-plain-emulator], [enable threaded non-smp emulator]) -AS_HELP_STRING([--disable-plain-emulator], [disable threaded non-smp emulator])) - AC_ARG_WITH(termcap, AS_HELP_STRING([--with-termcap], [use termcap (default)]) AS_HELP_STRING([--without-termcap], diff --git a/make/doc.mk b/make/doc.mk index 119dc75971..247ae4b36e 100644 --- a/make/doc.mk +++ b/make/doc.mk @@ -156,9 +156,6 @@ clean_chunks: # ---------------------------------------------------- include $(ERL_TOP)/make/otp_release_targets.mk -$(RELSYSDIR) $(RELSYSDIR)/doc: - $(INSTALL_DIR) "$@" - release_pdf_spec: pdf $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" @@ -206,8 +203,8 @@ ifneq ($(MAN7_FILES),) $(INSTALL_DATA) $(MAN7_FILES) "$(RELEASE_PATH)/man/man7" endif -release_docs_spec: $(RELSYSDIR)/doc $(INFO_FILE) $(DOC_TARGETS:%=release_%_spec) - $(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR) +release_docs_spec: $(INFO_FILE) $(DOC_TARGETS:%=release_%_spec) + $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" ifneq ($(STANDARDS),) $(INSTALL_DIR) "$(RELEASE_PATH)/doc/standard" $(INSTALL_DATA) $(STANDARDS) "$(RELEASE_PATH)/doc/standard" diff --git a/make/otp.mk.in b/make/otp.mk.in index 61654ad5cb..1ed554ee76 100644 --- a/make/otp.mk.in +++ b/make/otp.mk.in @@ -101,7 +101,7 @@ EMULATOR = beam ifeq ($(ERL_COMPILE_WARNINGS_AS_ERRORS),yes) ERL_COMPILE_FLAGS += -Werror endif -ifdef BOOTSTRAP +ifdef PRIMARY_BOOTSTRAP ERL_COMPILE_FLAGS += +slim else ERL_COMPILE_FLAGS += +debug_info diff --git a/make/otp_version_tickets_in_merge b/make/otp_version_tickets_in_merge index 1437e042af..e69de29bb2 100644 --- a/make/otp_version_tickets_in_merge +++ b/make/otp_version_tickets_in_merge @@ -1,2 +0,0 @@ -OTP-18288 -OTP-18290 diff --git a/make/test_target_script.sh b/make/test_target_script.sh index b21185fd06..6f854a55d4 100755 --- a/make/test_target_script.sh +++ b/make/test_target_script.sh @@ -114,13 +114,19 @@ EOM } release_erlang () { - local RELEASE_ROOT=${1} - if ! (cd $ERL_TOP && make release RELEASE_ROOT="${RELEASE_ROOT}"); then + local RELEASE_ROOT="${1}" + if ! (cd $ERL_TOP && make release TYPE= release_docs DOC_TARGETS=chunks RELEASE_ROOT="${RELEASE_ROOT}"); then return 1 fi if ! (cd "$RELEASE_ROOT" && ./Install -minimal "`pwd`"); then return 1 fi + ## Need to release both TYPE= and TYPE=$TYPE for tests to work + if [ "$TYPE" != "" ]; then + if ! (cd $ERL_TOP && make release TYPE=$TYPE RELEASE_ROOT="${RELEASE_ROOT}"); then + return 1 + fi + fi export PATH="${RELEASE_ROOT}/bin:$PATH" return 0 } @@ -192,7 +198,7 @@ MAKE_TEST_CT_LOGS="$MAKE_TEST_DIR/ct_logs" RELEASE_TEST_SPEC_LOG="$MAKE_TEST_DIR/release_tests_spec_log" INSTALL_TEST_LOG="$MAKE_TEST_DIR/install_tests_log" COMPILE_TEST_LOG="$MAKE_TEST_DIR/compile_tests_log" -RELEASE_ROOT="${MAKE_TEST_DIR}/otp" +RELEASE_ROOT=${RELEASE_ROOT:-"${MAKE_TEST_DIR}/Erlang ∅⊤℞"} RELEASE_LOG="$MAKE_TEST_DIR/release_tests_log" cd test @@ -216,10 +222,10 @@ EOF exit 1 fi CT_RUN="${RELEASE_ROOT}/bin/ct_run" - PATH=${RELEASE_ROOT}/bin/:${PATH} + PATH="${RELEASE_ROOT}/bin/":${PATH} fi -echo "The tests in test directory for $APPLICATION will be executed with ct_run" +echo "The tests in test directory for $APPLICATION will be executed with ${CT_RUN}" if [ -z "${ARGS}" ] then if [ ! -d "$MAKE_TEST_DIR" ] @@ -299,28 +305,28 @@ then CTRUN_TIMEOUT="timeout -s ABRT --foreground --preserve-status $((${CTRUN_TIMEOUT}+5))m timeout -s USR1 --foreground --preserve-status ${CTRUN_TIMEOUT}m" fi ERL_AFLAGS="${ERL_AFLAGS}" $CTRUN_TIMEOUT \ - $CT_RUN -logdir $MAKE_TEST_CT_LOGS\ - -pa "$ERL_TOP/lib/common_test/test_server"\ - -config "$ERL_TOP/lib/common_test/test_server/ts.config"\ - -config "$ERL_TOP/lib/common_test/test_server/ts.unix.config"\ + "${CT_RUN}" -logdir $MAKE_TEST_CT_LOGS \ + -pa "$ERL_TOP/lib/common_test/test_server" \ + -config "$ERL_TOP/lib/common_test/test_server/ts.config" \ + -config "$ERL_TOP/lib/common_test/test_server/ts.unix.config" \ -exit_status ignore_config \ - ${ARGS}\ - -erl_args\ - -env ERL_CRASH_DUMP "$MAKE_TEST_DIR/${APPLICATION}_erl_crash.dump"\ - -boot start_sasl\ - -sasl errlog_type error\ - -pz "$ERL_TOP/lib/common_test/test_server"\ - -pz "."\ - -ct_test_vars "{net_dir,\"\"}"\ - -noshell\ + ${ARGS} \ + -erl_args \ + -env ERL_CRASH_DUMP "$MAKE_TEST_DIR/${APPLICATION}_erl_crash.dump" \ + -boot start_sasl \ + -sasl errlog_type error \ + -pz "$ERL_TOP/lib/common_test/test_server" \ + -pz "." \ + -ct_test_vars "{net_dir,\"\"}" \ + -noinput \ -sname ${CT_NODENAME}\ - -rsh ssh\ + -rsh ssh \ ${ERL_ARGS} else WIN_MAKE_TEST_CT_LOGS=`w32_path.sh -m "$MAKE_TEST_CT_LOGS"` WIN_MAKE_TEST_DIR=`w32_path.sh -m "$MAKE_TEST_DIR"` WIN_ERL_TOP=`w32_path.sh -m "$ERL_TOP"` - $CT_RUN.exe -logdir $WIN_MAKE_TEST_CT_LOGS\ + "$CT_RUN.exe" -logdir $WIN_MAKE_TEST_CT_LOGS\ -pa "$WIN_ERL_TOP/lib/common_test/test_server"\ -config "$WIN_ERL_TOP/lib/common_test/test_server/ts.config"\ -config "$WIN_ERL_TOP/lib/common_test/test_server/ts.win32.config"\ @@ -333,7 +339,7 @@ else -pz "$WIN_ERL_TOP/lib/common_test/test_server"\ -pz "."\ -ct_test_vars "{net_dir,\"\"}"\ - -noshell\ + -noinput\ -sname ${CT_NODENAME}\ -rsh ssh\ ${ERL_ARGS} @@ -1001,8 +1001,7 @@ do_tests () do_debuginfo_win32 () { setup_make - (cd erts/emulator && $MAKE MAKE="$MAKE" TARGET=$TARGET FLAVOR=smp debug &&\ - $MAKE MAKE="$MAKE" TARGET=$TARGET FLAVOR=plain debug) || exit 1 + (cd erts/emulator && $MAKE MAKE="$MAKE" TARGET=$TARGET debug) || exit 1 if [ -z "$1" ]; then RELDIR="$ERL_TOP/release/$TARGET" else @@ -1010,7 +1009,7 @@ do_debuginfo_win32 () fi BINDIR="$ERL_TOP/bin/$TARGET" EVSN=`grep '^VSN' erts/vsn.mk | sed 's,^VSN.*=[^0-9]*\([0-9].*\)$,@\1,g;s,^[^@].*,,g;s,^@,,g'` - for f in beam.debug.smp.dll beam.smp.pdb beam.debug.smp.dll.pdb erl.pdb werl.pdb erlexec.pdb; do + for f in beam.debug.smp.dll beam.smp.pdb beam.debug.smp.dll.pdb erl.pdb erlexec.pdb; do if [ -f $BINDIR/$f ]; then rm -f $RELDIR/erts-$EVSN/bin/$f cp $BINDIR/$f $RELDIR/erts-$EVSN/bin/$f @@ -1218,7 +1217,7 @@ case "$1" in do_configure "$@";; opt) do_boot;; - plain|smp) + smp) if [ $minus_x_flag = false ]; then TYPE=opt fi; diff --git a/prebuild.delete b/prebuild.delete index 17efc89229..d9f7040428 100644 --- a/prebuild.delete +++ b/prebuild.delete @@ -1,4 +1,5 @@ bootstrap/lib bootstrap/target .git +.github scripts diff --git a/scripts/build-otp-tar b/scripts/build-otp-tar index fc8344c76a..6fb251f6e6 100755 --- a/scripts/build-otp-tar +++ b/scripts/build-otp-tar @@ -541,7 +541,13 @@ if [ ! -d $src_dir -o ! -f $src_dir/otp_build ]; then fi progress "Checking target directory name" -target_dirname=`$prebld_dir/make/autoconf/config.guess` +if [ -f "$prebld_dir/make/autoconf/config.guess" ]; then + target_dirname=`$prebld_dir/make/autoconf/config.guess` +elif [ -f "$prebld_dir/erts/autoconf/config.guess" ]; then + target_dirname=`$prebld_dir/erts/autoconf/config.guess` +else + error "Failed to find config.guess" +fi if [ $? -ne 0 ]; then error "Failed to check target directory name" fi @@ -651,7 +657,7 @@ done for kobj in $kobjs; do progress "Keeping $kobj in pre-build-directory" - copy $build_dir/../$kobj $prebld_root/$kobj + copy $build_dir/../$kobj $prebld_root/$(dirname $kobj) done cd $prebld_dir diff --git a/scripts/pre-push b/scripts/pre-push index 5759406182..8005ef80ee 100755 --- a/scripts/pre-push +++ b/scripts/pre-push @@ -23,14 +23,14 @@ # # Bump this version to give users an update notification. -PRE_PUSH_SCRIPT_VERSION=3 +PRE_PUSH_SCRIPT_VERSION=4 -NEW_RELEASES="24 23 22 21 20 19 18 17" +NEW_RELEASES="25 24 23 22 21 20 19 18 17" OLD_RELEASES="r16 r15 r14 r13" RELEASES="$NEW_RELEASES $OLD_RELEASES" # First commit on master, not allowed in other branches -MASTER_ONLY=a20c39812082068a8b9e3b73276de41fbb0338af +MASTER_ONLY=b0c30263a0ef698244d898a95c9b9a40e67eac9c # Number of commits and files allowed in one push by this script NCOMMITS_MAX=100 diff --git a/system/doc/design_principles/Makefile b/system/doc/design_principles/Makefile index 44d5992a2e..eb26f3527e 100644 --- a/system/doc/design_principles/Makefile +++ b/system/doc/design_principles/Makefile @@ -33,7 +33,7 @@ XMLDIR := $(XMLDIR)/design_principles # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/design_principles" +RELSYSDIR = $(RELEASE_PATH)/doc/design_principles # ---------------------------------------------------- # Target Specs @@ -122,7 +122,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(IMAGE_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/design_principles/statem.xml b/system/doc/design_principles/statem.xml index c53d569c99..928996613a 100644 --- a/system/doc/design_principles/statem.xml +++ b/system/doc/design_principles/statem.xml @@ -1912,14 +1912,20 @@ do_unlock() -> to implicitly postpone any events to the <c>locked</c> state. </p> <p> - A selective receive cannot be used from a <c>gen_statem</c> + A catch-all receive should never be used from a <c>gen_statem</c> behaviour (or from any <c>gen_*</c> behaviour), as the receive statement is within the <c>gen_*</c> engine itself. - It must be there because all <seeerl marker="stdlib:sys"><c>sys</c></seeerl> compatible behaviours must respond to system messages and therefore do that in their engine receive loop, passing non-system messages to the <em>callback module</em>. + Using a catch-all receive may result in system messages + being discarded which in turn may lead to unexpected behaviour. + If a selective receive must be used then great care should be taken to + ensure that only messages pertinent to the operation are received. + Likewise, a callback must return in due time to let the engine + receive loop handle system messages, or they might time out also + leading to unexpected behaviour. </p> <p> The diff --git a/system/doc/design_principles/sup_princ.xml b/system/doc/design_principles/sup_princ.xml index 406bea1ad1..554eeeed91 100644 --- a/system/doc/design_principles/sup_princ.xml +++ b/system/doc/design_principles/sup_princ.xml @@ -69,7 +69,7 @@ init(_Args) -> restart => permanent, shutdown => brutal_kill, type => worker, - modules => [cg3]}], + modules => [ch3]}], {ok, {SupFlags, ChildSpecs}}.</code> <p>The <c>SupFlags</c> variable in the return value from <c>init/1</c> represents diff --git a/system/doc/efficiency_guide/Makefile b/system/doc/efficiency_guide/Makefile index 50ed66019a..8b210d38e8 100644 --- a/system/doc/efficiency_guide/Makefile +++ b/system/doc/efficiency_guide/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/efficiency_guide # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/efficiency_guide" +RELSYSDIR = $(RELEASE_PATH)/doc/efficiency_guide # ---------------------------------------------------- # Target Specs @@ -112,7 +112,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/embedded/Makefile b/system/doc/embedded/Makefile index 29f3f74564..775cb4b23b 100644 --- a/system/doc/embedded/Makefile +++ b/system/doc/embedded/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/embedded # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/embedded" +RELSYSDIR = $(RELEASE_PATH)/doc/embedded # ---------------------------------------------------- # Target Specs @@ -100,7 +100,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/general_info/Makefile b/system/doc/general_info/Makefile index 55cac49c1f..3c79601a41 100644 --- a/system/doc/general_info/Makefile +++ b/system/doc/general_info/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/general_info # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/general_info" +RELSYSDIR = $(RELEASE_PATH)/doc/general_info # ---------------------------------------------------- # Target Specs @@ -108,7 +108,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/general_info/upcoming_incompatibilities.xml b/system/doc/general_info/upcoming_incompatibilities.xml index 59003d2243..17fbd135b3 100644 --- a/system/doc/general_info/upcoming_incompatibilities.xml +++ b/system/doc/general_info/upcoming_incompatibilities.xml @@ -68,29 +68,6 @@ <section> <title>OTP 26</title> - <section> - <title>The distribution flag DFLAG_V4_NC will become mandatory</title> - <p>As of OTP 26, the distribution flag <seeguide - marker="erts:erl_dist_protocol#DFLAG_V4_NC">DFLAG_V4_NC</seeguide> - will become mandatory. If you implement the Erlang distribution - protocol yourself, you will need to implement support for - <c>DFLAG_V4_NC</c> in order to communicate with Erlang nodes - running OTP 26.</p> - </section> - - <section> - <title>The new link protocol will become mandatory</title> - <p> - As of OTP 26, the <seeguide - marker="erts:erl_dist_protocol#new_link_protocol">new link - protocol</seeguide> will become mandatory. That is, Erlang - nodes will then refuse to connect to nodes not implementing - the new link protocol. If you implement the Erlang - distribution yourself, you are, however, encouraged to - implement the new link protocol as soon as possible since the - old protocol can cause links to enter an inconsistent state. - </p> - </section> <section> <marker id="atoms_be_utf8"/> @@ -132,18 +109,5 @@ </list> </section> - <section> - <title>The default timewarp mode will change to multi-time warp mode</title> - <p> - The default <seeguide marker="erts:time_correction#Time_Warp_Modes"> - Time Warp Mode</seeguide> will be changed from - <seeguide marker="erts:time_correction#No_Time_Warp_Mode"> - no time warp mode</seeguide> to <seeguide marker="erts:time_correction#Multi_Time_Warp_Mode"> - multi-time warp mode</seeguide>. See <seeguide marker="erts:time_correction"> - Time and Time Correction in Erlang</seeguide> for details on how this will - effect your system. - </p> - </section> - </section> </chapter> diff --git a/system/doc/getting_started/Makefile b/system/doc/getting_started/Makefile index 6a286d6df6..0d8c5c0c72 100644 --- a/system/doc/getting_started/Makefile +++ b/system/doc/getting_started/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/getting_started # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/getting_started" +RELSYSDIR = $(RELEASE_PATH)/doc/getting_started # ---------------------------------------------------- # Target Specs @@ -99,7 +99,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/installation_guide/Makefile b/system/doc/installation_guide/Makefile index 53a1cad319..1d7f29253c 100644 --- a/system/doc/installation_guide/Makefile +++ b/system/doc/installation_guide/Makefile @@ -33,7 +33,7 @@ XMLDIR := $(XMLDIR)/installation_guide # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/installation_guide" +RELSYSDIR = $(RELEASE_PATH)/doc/installation_guide REDIRECT_HTML_RELSYSDIR = $(RELSYSDIR)/source # ---------------------------------------------------- @@ -127,9 +127,9 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" - $(INSTALL_DATA) $(GIF_FILES) $(HTMLDIR)/*.html $(RELSYSDIR) - $(INSTALL_DIR) $(REDIRECT_HTML_RELSYSDIR) - $(INSTALL_DATA) $(REDIRECT_HTML_FILES) $(REDIRECT_HTML_RELSYSDIR) + $(INSTALL_DATA) $(GIF_FILES) $(HTMLDIR)/*.html "$(RELSYSDIR)" + $(INSTALL_DIR) "$(REDIRECT_HTML_RELSYSDIR)" + $(INSTALL_DATA) $(REDIRECT_HTML_FILES) "$(REDIRECT_HTML_RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/oam/Makefile b/system/doc/oam/Makefile index 40b14485da..47cac62ea6 100644 --- a/system/doc/oam/Makefile +++ b/system/doc/oam/Makefile @@ -31,7 +31,7 @@ XMLDIR := $(XMLDIR)/oam # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/oam" +RELSYSDIR = $(RELEASE_PATH)/doc/oam # ---------------------------------------------------- # Target Specs @@ -101,7 +101,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/programming_examples/Makefile b/system/doc/programming_examples/Makefile index 7099d88ebc..fefef96c0a 100644 --- a/system/doc/programming_examples/Makefile +++ b/system/doc/programming_examples/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/programming_examples # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/programming_examples" +RELSYSDIR = $(RELEASE_PATH)/doc/programming_examples # ---------------------------------------------------- # Target Specs @@ -99,7 +99,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/reference_manual/Makefile b/system/doc/reference_manual/Makefile index 2e0cb4fbc2..9c5c9666db 100644 --- a/system/doc/reference_manual/Makefile +++ b/system/doc/reference_manual/Makefile @@ -33,7 +33,7 @@ XMLDIR := $(XMLDIR)/reference_manual # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/reference_manual" +RELSYSDIR = $(RELEASE_PATH)/doc/reference_manual # ---------------------------------------------------- # Target Specs @@ -102,7 +102,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/reference_manual/typespec.xml b/system/doc/reference_manual/typespec.xml index 75482e5be1..fbcc3315a7 100644 --- a/system/doc/reference_manual/typespec.xml +++ b/system/doc/reference_manual/typespec.xml @@ -314,11 +314,6 @@ <tcaption>Additional built-in types</tcaption> </table> - <p> - Users are not allowed to define types with the same names as the - predefined or built-in ones. This is checked by the compiler and - its violation results in a compilation error. - </p> <note> <p> The following built-in list types also exist, @@ -345,7 +340,34 @@ This is described in <seeguide marker="#typeinrecords"> Type Information in Record Declarations</seeguide>. </p> + + <section> + <title>Redefining built-in types</title> + <p> + Starting from Erlang/OTP 26, is is permitted to define a type + having the same name as a built-in type. It is recommended to + avoid deliberately reusing built-in names because it can be + confusing. However, when an Erlang/OTP release introduces a new + type, code that happened to define its own type having the same + name will continue to work. + </p> + + <p>As an example, imagine that the Erlang/OTP 42 release introduces + a new type <c>gadget()</c> defined like this:</p> + + <pre> + -type gadget() :: {'gadget', reference()}.</pre> + + <p>Further imagine that some code has its own (different) + definition of <c>gadget()</c>, for example:</p> + + <pre> + -type gadget() :: #{}.</pre> + + <p>Since redefinitions are allowed, the code will still compile (but + with a warning), and Dialyzer will not emit any additional warnings.</p> </section> +</section> <section> <title>Type Declarations of User-Defined Types</title> diff --git a/system/doc/system_architecture_intro/Makefile b/system/doc/system_architecture_intro/Makefile index 8657da0e2c..57b60a80fe 100644 --- a/system/doc/system_architecture_intro/Makefile +++ b/system/doc/system_architecture_intro/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/system_architecture_intro # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/system_architecture_intro" +RELSYSDIR = $(RELEASE_PATH)/doc/system_architecture_intro # ---------------------------------------------------- # Target Specs @@ -94,7 +94,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/system_principles/Makefile b/system/doc/system_principles/Makefile index 00b2203394..dca8f54cc5 100644 --- a/system/doc/system_principles/Makefile +++ b/system/doc/system_principles/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/system_principles # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/system_principles" +RELSYSDIR = $(RELEASE_PATH)/doc/system_principles # ---------------------------------------------------- # Target Specs @@ -95,7 +95,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) diff --git a/system/doc/top/Makefile b/system/doc/top/Makefile index e294d0b0a7..c01614252c 100644 --- a/system/doc/top/Makefile +++ b/system/doc/top/Makefile @@ -31,7 +31,7 @@ APPLICATION=otp-system-documentation # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc" +RELSYSDIR = $(RELEASE_PATH)/doc GIF_FILES = @@ -102,14 +102,33 @@ PDFREFDIR= pdf TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf TOPDOC=true +sp := + +sp += + +## qs translates ' ' to '\ ', sq translates '\ ' to ' ' +## These function are used when the make target is a path that can +## contain spaces. This is needed a the target 'foo bar:' needs to be +## written as 'foo\ bar:' in order for make to interpret it as a single +## target. Unfortunately, when this is done $@ will contain '\ ', which +## means that we have to use sq to remove the escape again. A small example +## looks like this: +## +## $(call qs, /path/with space/file.o): $(call qs, /path/with space/file.c) +## gcc -c -o "$(call sq, $@)" "$(call sq, $^)" +qs = $(subst $(sp),\$(sp),$1) +sq = $(subst \$(sp),$(sp),$1) + ifdef RELEASE_PATH INST_TYPE=rel INST_TYPE_SRC_DIR=$(RELEASE_PATH) # We build to the 'temporary' dir in order to be able to install # results using INSTALL_DATA (in order to get correct access # rights on installed files) +# The temporary folder needs to be in the released system as we do +# not know if we have write permissions in the source release. INST_TYPE_DEST_DIR=$(RELSYSDIR)/temporary -INST_TYPE_DEST_DIR_DEP=$(INST_TYPE_DEST_DIR) +INST_TYPE_DEST_DIR_DEP=$(call qs,$(INST_TYPE_DEST_DIR)) INST_TYPE_JS_DEST_DIR=$(INST_TYPE_DEST_DIR) INST_TYPE_VSN_FILE=$(INST_TYPE_DEST_DIR)/OTP_VERSION else @@ -131,17 +150,17 @@ EBIN = ebin INDEX_SCRIPT = $(EBIN)/erl_html_tools.$(EMULATOR) INDEX_SRC = src/erl_html_tools.erl -INDEX_HTML=$(INST_TYPE_DEST_DIR)/index.html -APPLICATIONS_HTML=$(INST_TYPE_DEST_DIR)/applications.html +INDEX_HTML=$(call qs,$(INST_TYPE_DEST_DIR)/index.html) +APPLICATIONS_HTML=$(call qs,$(INST_TYPE_DEST_DIR)/applications.html) INDEX_FILES = $(INDEX_HTML) $(APPLICATIONS_HTML) -JAVASCRIPT = $(INST_TYPE_JS_DEST_DIR)/erlresolvelinks.js +JAVASCRIPT = $(call qs,$(INST_TYPE_JS_DEST_DIR)/erlresolvelinks.js) JAVASCRIPT_BUILD_SCRIPT = $(EBIN)/erlresolvelinks.$(EMULATOR) JAVASCRIPT_BUILD_SCRIPT_SRC = src/erlresolvelinks.erl MAN_INDEX_SCRIPT = $(EBIN)/otp_man_index.$(EMULATOR) MAN_INDEX_SRC = src/otp_man_index.erl -MAN_INDEX = $(INST_TYPE_DEST_DIR)/man_index.html +MAN_INDEX = $(call qs,$(INST_TYPE_DEST_DIR)/man_index.html) GLOSSARY = $(HTMLDIR)/glossary.html GLOSSARY_SRC = $(ERL_TOP)/system/internal_tools/doctools/src/glossary.erl @@ -156,18 +175,18 @@ TEMPLATES = \ $(INDEX_SCRIPT): $(INDEX_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< -$(INST_TYPE_DEST_DIR)/OTP_VERSION: $(INST_TYPE_DEST_DIR_DEP) +$(call qs,$(INST_TYPE_DEST_DIR)/OTP_VERSION): $(INST_TYPE_DEST_DIR_DEP) if test -f "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/OTP_VERSION"; then \ - $(CP) "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/OTP_VERSION" $@; \ + $(CP) "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/OTP_VERSION" "$(call sq,$@)"; \ else \ - $(CP) $(ERL_TOP)/OTP_VERSION $@; \ + $(CP) "$(ERL_TOP)/OTP_VERSION" "$(call sq,$@)"; \ fi # We don't list toc_*.html as targets because we don't know -$(INDEX_HTML) + $(APPLICATIONS_HTML): $(INST_TYPE_DEST_DIR_DEP) $(INDEX_SCRIPT) $(TEMPLATES) $(INST_TYPE_VSN_FILE) +$(INDEX_HTML) + $(APPLICATIONS_HTML): $(INST_TYPE_DEST_DIR_DEP) $(INDEX_SCRIPT) $(TEMPLATES) $(call qs,$(INST_TYPE_VSN_FILE)) echo "Generating index $@" - $(ERL) -noshell -pa $(EBIN) -s erl_html_tools top_index $(INST_TYPE) \ - $(INST_TYPE_SRC_DIR) $(INST_TYPE_DEST_DIR) \ + $(ERL) +pc unicode -noshell -pa $(EBIN) -s erl_html_tools top_index $(INST_TYPE) \ + "$(INST_TYPE_SRC_DIR)" "$(INST_TYPE_DEST_DIR)" \ `cat "$(INST_TYPE_VSN_FILE)"` -s erlang halt @@ -177,8 +196,8 @@ $(JAVASCRIPT_BUILD_SCRIPT): $(JAVASCRIPT_BUILD_SCRIPT_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< $(JAVASCRIPT): $(INST_TYPE_DEST_DIR_DEP) $(JAVASCRIPT_BUILD_SCRIPT) - erl -noshell -pa $(EBIN) -run erlresolvelinks make $(ERL_TOP) \ - $(INST_TYPE_SRC_DIR) $(INST_TYPE_JS_DEST_DIR) -s erlang halt + $(ERL) +pc unicode -noshell -pa $(EBIN) -run erlresolvelinks make $(ERL_TOP) \ + "$(INST_TYPE_SRC_DIR)" "$(INST_TYPE_JS_DEST_DIR)" -s erlang halt #-------------------------------------------------------------------------- @@ -186,8 +205,8 @@ $(MAN_INDEX_SCRIPT): $(MAN_INDEX_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< $(MAN_INDEX): $(INST_TYPE_DEST_DIR_DEP) $(MAN_INDEX_SCRIPT) - $(ERL) -noshell -pa $(EBIN) -s otp_man_index gen $(INST_TYPE) \ - $(INST_TYPE_SRC_DIR) $@ -s erlang halt + $(ERL) +pc unicode -noshell -pa $(EBIN) -s otp_man_index gen $(INST_TYPE) \ + "$(INST_TYPE_SRC_DIR)" "$(call sq,$@)" -s erlang halt #-------------------------------------------------------------------------- @@ -280,31 +299,31 @@ clean: # ---------------------------------------------------- include $(ERL_TOP)/make/otp_release_targets.mk -$(RELSYSDIR)/temporary: - $(INSTALL_DIR) $(RELSYSDIR)/temporary +$(call qs,$(RELSYSDIR)/temporary): + $(INSTALL_DIR) "$(RELSYSDIR)/temporary" -$(RELSYSDIR)/docbuild: - $(INSTALL_DIR) $(RELSYSDIR)/docbuild +$(call qs,$(RELSYSDIR)/docbuild): + $(INSTALL_DIR) "$(RELSYSDIR)/docbuild" -release_man_spec: man $(RELSYSDIR)/docbuild +release_man_spec: man $(call qs,$(RELSYSDIR)/docbuild) -release_html_spec: html $(RELSYSDIR)/docbuild - $(INSTALL_DATA) $(MAN_INDEX) $(RELSYSDIR) - $(INSTALL_DATA) $(MAN_INDEX_SRC) $(MAN_INDEX_SCRIPT) $(RELSYSDIR)/docbuild - $(INSTALL_DIR) $(RELSYSDIR)/js - $(INSTALL_DATA) $(JAVASCRIPT) $(RELSYSDIR)/js - $(INSTALL_DATA) $(INDEX_FILES) $(RELSYSDIR) +release_html_spec: html $(call qs,$(RELSYSDIR)/docbuild) + $(INSTALL_DATA) $(MAN_INDEX) "$(RELSYSDIR)" + $(INSTALL_DATA) $(MAN_INDEX_SRC) $(MAN_INDEX_SCRIPT) "$(RELSYSDIR)/docbuild" + $(INSTALL_DIR) "$(RELSYSDIR)/js" + $(INSTALL_DATA) $(JAVASCRIPT) "$(RELSYSDIR)/js" + $(INSTALL_DATA) $(INDEX_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(INDEX_SCRIPT) $(JAVASCRIPT_BUILD_SCRIPT) \ $(INDEX_SRC) $(JAVASCRIPT_BUILD_SCRIPT_SRC) \ - $(TEMPLATES) $(RELSYSDIR)/docbuild + $(TEMPLATES) "$(RELSYSDIR)/docbuild" release_pdf_spec: pdf - $(INSTALL_DIR) $(RELSYSDIR)/pdf + $(INSTALL_DIR) "$(RELSYSDIR)/pdf" $(INSTALL_DATA) \ - $(TOP_PDF_FILE) $(RELSYSDIR)/pdf + $(TOP_PDF_FILE) "$(RELSYSDIR)/pdf" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) $(INFO_FILES) $(INSTALL_DATA) $(INFO_FILES) "$(RELEASE_PATH)" - $(RM) -r $(RELSYSDIR)/temporary + $(RM) -r "$(RELSYSDIR)/temporary" release_spec: diff --git a/system/doc/top/src/erl_html_tools.erl b/system/doc/top/src/erl_html_tools.erl index d609f35380..a6df7f523e 100644 --- a/system/doc/top/src/erl_html_tools.erl +++ b/system/doc/top/src/erl_html_tools.erl @@ -67,18 +67,18 @@ top_index(RootDir) when is_atom(RootDir) -> top_index(Source, RootDir, DestDir, OtpBaseVsn) -> - report("****\nRootDir: ~p", [RootDir]), - report("****\nDestDir: ~p", [DestDir]), - report("****\nOtpBaseVsn: ~p", [OtpBaseVsn]), + report("****\nRootDir: ~tp", [RootDir]), + report("****\nDestDir: ~tp", [DestDir]), + report("****\nOtpBaseVsn: ~tp", [OtpBaseVsn]), put(otp_base_vsn, OtpBaseVsn), Templates = find_templates(["","templates",DestDir]), - report("****\nTemplates: ~p", [Templates]), + report("****\nTemplates: ~tp", [Templates]), Bases = [{"../lib/", filename:join(RootDir,"lib")}, {"../", RootDir}], Groups = find_information(Source, Bases), - report("****\nGroups: ~p", [Groups]), + report("****\nGroups: ~tp", [Groups]), process_templates(Templates, DestDir, Groups). top_index_silent(RootDir, DestDir, OtpBaseVsn) -> @@ -97,7 +97,7 @@ top_index_silent(RootDir, DestDir, OtpBaseVsn) -> process_templates([], _DestDir, _Groups) -> report("\n", []); process_templates([Template | Templates], DestDir, Groups) -> - report("****\nIN-FILE: ~s", [Template]), + report("****\nIN-FILE: ~ts", [Template]), BaseName = filename:basename(Template, ".src"), case lists:reverse(filename:rootname(BaseName)) of "_"++_ -> @@ -127,7 +127,7 @@ process_multi_template_1([{Suffix,Group}|Gs], BaseName, Ext, Template, DestDir, process_multi_template_1([], _, _, _, _, _) -> ok. subst_file(Group, OutFile, Template, Info) -> - report("\nOUTFILE: ~s", [OutFile]), + report("\nOUTFILE: ~ts", [OutFile]), case subst_template(Group, Template, Info) of {ok,Text,_NewInfo} -> case file:open(OutFile, [write]) of @@ -135,10 +135,10 @@ subst_file(Group, OutFile, Template, Info) -> file:write(Stream, Text), file:close(Stream); Error -> - local_error("Can't write to file ~s: ~w", [OutFile,Error]) + local_error("Can't write to file ~ts: ~w", [OutFile,Error]) end; Error -> - local_error("Can't write to file ~s: ~w", [OutFile,Error]) + local_error("Can't write to file ~ts: ~w", [OutFile,Error]) end. @@ -203,11 +203,11 @@ get_app_paths(src, AppDirs, URL) -> {match, [V]} -> V; nomatch -> - exit(io_lib:format("No VSN variable found in ~s\n", + exit(io_lib:format("No VSN variable found in ~ts\n", [VsnFile])) end; {error, Reason} -> - exit(io_lib:format("~p : ~s\n", [Reason, VsnFile])) + exit(io_lib:format("~p : ~ts\n", [Reason, VsnFile])) end, AppURL = URL ++ App ++ "-" ++ VsnStr, {App, VsnStr, AppPath, AppURL ++ "/" ++ Sub1} diff --git a/system/doc/tutorial/Makefile b/system/doc/tutorial/Makefile index faf06b3878..e715080253 100644 --- a/system/doc/tutorial/Makefile +++ b/system/doc/tutorial/Makefile @@ -32,7 +32,7 @@ XMLDIR := $(XMLDIR)/tutorial # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = "$(RELEASE_PATH)/doc/tutorial" +RELSYSDIR = $(RELEASE_PATH)/doc/tutorial # ---------------------------------------------------- # Target Specs @@ -123,7 +123,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_html_spec: html $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTMLDIR)/*.html \ - $(RELSYSDIR) + "$(RELSYSDIR)" release_docs_spec: $(DOC_TARGETS:%=release_%_spec) |